Meetup Scraper + API by Topic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pycurl\n",
    "from StringIO import StringIO\n",
    "import json\n",
    "import csv\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from pandas.io.json import json_normalize\n",
    "import requests\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Input a topic name from https://www.meetup.com/topics/ . This example will conitnue with musical theatre\n",
    "topic = \"musical-theatre\"\n",
    "# Put your password / api key using https://secure.meetup.com/meetup_api/key/ . This will not work without it\n",
    "password = "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a link based on your topic of interest\n",
    "url = \"https://www.meetup.com/topics/\" + topic + \"/all/\"\n",
    "# Finding all of the meetups associated with this topic\n",
    "response = requests.get(url, params={\"search_api_views_fulltext\": \"\"})\n",
    "soup = BeautifulSoup(response.text)\n",
    "links_messy = soup.find_all('li', attrs={'class':'gridList-item'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a clean list of links\n",
    "group_name_list = []\n",
    "for i in range(0,len(links_messy)):\n",
    "    selector = links_messy[i].select('a')[0]\n",
    "    link = selector['href']\n",
    "    group_name = link.split(\"/\")[3]\n",
    "    group_name_list.append(group_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create an empty list to dump list of jsons in\n",
    "json_list = []\n",
    "# Iterate over each group name\n",
    "for name in tqdm(group_name_list):\n",
    "    # Create link for group\n",
    "    api_link = \"https://api.meetup.com/\" + name + \"?&sign=true&photo-host=public\"\n",
    "    # Pycurl call\n",
    "    buffer = StringIO()\n",
    "    c = pycurl.Curl()  \n",
    "    c.setopt(c.URL, api_link)\n",
    "    c.setopt(c.USERPWD, password)\n",
    "    c.setopt(c.WRITEDATA, buffer)\n",
    "    c.perform()\n",
    "    c.close()\n",
    "    # Convert output into a json, then append it to a list of jsons\n",
    "    body = buffer.getvalue()\n",
    "    group_json = json.loads(body)\n",
    "    json_list.append(group_json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a pandas dataframe with all of the data from  the json_list\n",
    "meetup_dataset = json_normalize(json_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Converting pandas dataframe into a csv2 file for further use \n",
    "filename =  topic + '-Meetup-Dataset.csv'\n",
    "meetup_dataset.to_csv(filename, encoding='utf-8')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}