map_github_forks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# github api helper function\n",
    "\n",
    "import requests\n",
    "import time\n",
    "import itertools\n",
    "\n",
    "from requests.auth import HTTPBasicAuth\n",
    "\n",
    "\n",
    "def make_api_call(url):\n",
    "    auth = ('ThePrecious', 'XXX')\n",
    "    headers = {\"Accept\": \"application/vnd.github.v3.star+json\"}\n",
    "    response = requests.get(url, auth=auth, headers=headers)\n",
    "    return response\n",
    "\n",
    "def get_dict_subset(d):\n",
    "    return dict(itertools.islice(d.iteritems(), 0, 10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1673\n",
      "https://api.github.com/repositories/2126244/forks\n"
     ]
    }
   ],
   "source": [
    "## get fork details for a sample repo\n",
    "\n",
    "url = \"https://api.github.com/repos/twbs/bootstrap/forks\"\n",
    "response = make_api_call(url)\n",
    "total_pages = int(response.headers['Link'].split(',')[1].split('page=')[1].split('>;')[0])\n",
    "base_url = response.headers['Link'].split(',')[1].split('?')[0][2:]\n",
    "\n",
    "print total_pages\n",
    "print base_url"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get the required fork and user details page by page and store them in a dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "720\n",
      "{29728800: [u'songafeng', u'2017-06-28T07:22:07Z', 'NOLOCATION'], 5212162: [u'bulinan', u'2017-06-28T05:37:45Z', u'Beijing'], 29624324: [u'HongDanni', u'2017-06-23T12:34:14Z', 'NOLOCATION'], 23920648: [u'coolzpl', u'2017-06-16T11:41:59Z', 'NOLOCATION'], 13555746: [u'zhangao15', u'2017-07-02T09:52:55Z', 'NOLOCATION'], 12838927: [u'xrj3000', u'2017-07-02T01:19:59Z', 'NOLOCATION'], 20639760: [u'bessii', u'2017-06-26T19:11:18Z', u'Buea | Cameroon'], 25606164: [u'Zlth', u'2017-06-28T01:50:52Z', 'NOLOCATION'], 1794074: [u'ciela', u'2017-06-19T11:21:57Z', u'Tokyo'], 11409435: [u'yangyangxie', u'2017-06-28T05:49:02Z', 'NOLOCATION']}\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Function to get the location of the user\n",
    "def get_user_location(login):\n",
    "    url = \"https://api.github.com/users/%s\" % login\n",
    "    response = make_api_call(url)\n",
    "    return response.json()['location']\n",
    "    \n",
    "fork_details = {}\n",
    "total_pages = 25 # Temp; comment this line for all pages\n",
    "\n",
    "# Get the login, forked date and location of all those who have forked the repo\n",
    "for i in range(1, total_pages):\n",
    "    url = base_url + \"?page=%s\" %i\n",
    "    response = make_api_call(url);\n",
    "    result = response.json()\n",
    "\n",
    "    for obj in result:\n",
    "        f_date = obj['created_at']\n",
    "        owner_id = obj['owner']['id']\n",
    "        login = obj['owner']['login']\n",
    "        location = get_user_location(login)\n",
    "        if location is None:\n",
    "            location = \"NOLOCATION\"\n",
    "        fork_details[owner_id] = [login, f_date, location] \n",
    "        \n",
    "    \n",
    "\n",
    "    ## get fields related to rate limit from response header\n",
    "    rate_limit_remain = int(response.headers['X-RateLimit-Remaining'])\n",
    "    rate_limit_reset = int(response.headers['X-RateLimit-Reset'])\n",
    "\n",
    "    ## if <=100 sleep for 100 seconds\n",
    "    if rate_limit_remain <= 100:\n",
    "        diff = rate_limit_reset - int(time.time())\n",
    "        if diff < 0: \n",
    "            diff = 100\n",
    "        time.sleep(diff)\n",
    "    \n",
    "print len(fork_details)\n",
    "print get_dict_subset(fork_details)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parse and bin the count according to forked date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{u'2017-06-28': 50, u'2017-06-29': 42, u'2017-06-22': 53, u'2017-06-23': 29, u'2017-06-20': 32, u'2017-06-21': 32, u'2017-07-04': 47, u'2017-07-03': 32, u'2017-07-02': 23, u'2017-07-01': 24}\n"
     ]
    }
   ],
   "source": [
    "fork_counts = {}\n",
    "\n",
    "for k,v in fork_details.iteritems():\n",
    "    d = v[1]\n",
    "    d = d.split('T')[0]\n",
    "    if fork_counts.has_key(d):\n",
    "        fork_counts[d] += 1\n",
    "    else:\n",
    "        fork_counts[d] = 1\n",
    "        \n",
    "print get_dict_subset(fork_counts)\n",
    "#fork_counts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parse and bin the count according to user's city (location from which it was forked)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{u'New Delhi': 1, u'Chennai': 1, u'San Francisco Bay area': 1, u'Sacramento': 1, u'\\u5317\\u4eac': 2, u'Amman ': 1, u'Guatemala': 1, u'Kent': 1, u'Argentina': 1, u'Shenzhen': 2}\n"
     ]
    }
   ],
   "source": [
    "fork_city_counts = {}\n",
    "nolocation = 0\n",
    "fork_state_counts = {}\n",
    "fork_country_counts = {}\n",
    "\n",
    "for k, v in fork_details.iteritems():\n",
    "    city = v[2]\n",
    "    city = city.split(',')[0]\n",
    "    \"\"\" \n",
    "    state = v[2]\n",
    "    state = state.split(',')[1]\n",
    "    country = v[2]\n",
    "    country = country.split(',')[2]\n",
    "    \"\"\"\n",
    "\n",
    "    if fork_city_counts.has_key(city):\n",
    "        fork_city_counts[city] += 1\n",
    "    else:\n",
    "        fork_city_counts[city] = 1\n",
    "    \n",
    "    \"\"\"  \n",
    "    if fork_state_counts.has_key(state):\n",
    "        fork_state_counts[state] += 1\n",
    "    else:\n",
    "        fork_state_counts[state] = 1\n",
    "    \n",
    "    if fork_country_counts.has_key(country):\n",
    "        fork_country_counts[country] += 1\n",
    "    else:\n",
    "        fork_country_counts[country] = 1 \n",
    "    \"\"\"\n",
    "        \n",
    "print get_dict_subset(fork_city_counts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### Get the latitude, longitude of the cities using geopy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(37.7884969, -122.3558473), (31.9515694, 35.9239625), (51.2474823, 0.7105077), (-34.9964963, -64.9672817), (28.6138967, 77.2159562), (22.5442673, 114.0545327), (13.0801721, 80.2838331), (38.5815719, -121.4943996), (39.9059631, 116.391248), (15.6356088, -89.8988087)]\n"
     ]
    }
   ],
   "source": [
    "from geopy.geocoders import Nominatim\n",
    "geolocator = Nominatim()\n",
    "\n",
    "fork_city_counts.pop('NOLOCATION')\n",
    "\n",
    "locations = []\n",
    "for c in fork_city_counts.keys():\n",
    "    location = geolocator.geocode(c)\n",
    "    if location:\n",
    "        locations.append((location.latitude, location.longitude))\n",
    "    \n",
    "print locations[0:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize the fork locations on a map (using folium)\n",
    "\n",
    "<img src=\"forks_map.png\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import display, HTML\n",
    "import folium\n",
    "\n",
    "# Create an empty map with focus on SF\n",
    "SF_COORDINATES = (37.76, -122.45)\n",
    "m = folium.Map(location=SF_COORDINATES, zoom_start=5)\n",
    "\n",
    "marker_cluster = folium.MarkerCluster().add_to(m)\n",
    "\n",
    "for loc in locations:\n",
    "    folium.Marker(\n",
    "        location=loc,\n",
    "        icon=folium.Icon(color='green', icon='ok-sign'),\n",
    "    ).add_to(marker_cluster)\n",
    "\n",
    "\n",
    "m.save('forks_map.html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe width=\"800\" height=\"350\" frameborder=\"0\" scrolling=\"no\" marginheight=\"10\" marginwidth=\"10\" src=\"forks_map.html\"></iframe>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "HTML('<iframe width=\"800\" height=\"350\" frameborder=\"0\" scrolling=\"no\" marginheight=\"10\" marginwidth=\"10\" src=\"forks_map.html\"></iframe>')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}