Job_Salary_Prediction_Data_Scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import bs4\n",
    "from urllib.request import urlopen as uReq\n",
    "from bs4 import BeautifulSoup as soup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 601,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "my_url= \"https://www.indeed.com/salaries/Senior-Software-Engineer-Salaries,-Atlanta-GA?start=50\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 602,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "uClient= uReq(my_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 603,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Page_html = uClient.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 604,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "uClient.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 605,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "page_soup= soup(Page_html, \"html.parser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 606,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "salary_aggregate = page_soup.findAll(\"tr\", {\"class\": \"cmp-salary-aggregate-table-entry cmp-sal-separate-row\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 607,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 607,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filename= (\"BA_Atlanta5.csv\")\n",
    "f = open(filename, \"w\")\n",
    "headers = \"Title, AvgSal, minSal, maxSal\\n\"\n",
    "f.write(headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 608,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Title: Senior Software Engineer\n",
      "AvgSal:$102,918 per year\n",
      "MinSal:$36,000\n",
      "MaxSal:$170,000\n",
      "Title: Software Engineer\n",
      "AvgSal:$84,202 per year\n",
      "MinSal:$36,000\n",
      "MaxSal:$170,000\n"
     ]
    }
   ],
   "source": [
    "for container in salary_aggregate:\n",
    "    Title_container= container.findAll(\"div\",{\"class\": \"cmp-sal-title\"})\n",
    "    Title = Title_container[0].text\n",
    "    AvgSal_container= container.findAll(\"div\", {\"class\": \"cmp-sal-summary\"})\n",
    "    AvgSal = AvgSal_container[0].text\n",
    "    minSal_container= container.findAll(\"div\",{\"class\": \"cmp-sal-min cmp-sal-caption cmp-float-left\"})\n",
    "    minSal= minSal_container[0].text\n",
    "    maxSal_container= container.findAll(\"div\",{\"class\": \"cmp-sal-max cmp-sal-caption cmp-float-right\"})\n",
    "    maxSal= maxSal_container[0].text\n",
    "    print(\"Company Name: \"+ Title)\n",
    "    print(\"Average Salary:\" + AvgSal)\n",
    "    print(\"Minimum:\" + minSal)\n",
    "    print(\"Maximum:\" + maxSal)\n",
    "    \n",
    "    f.write(Title.replace(\",\", \" \") + \",\" + AvgSal.replace(\",\", \"\")+ \",\" + minSal.replace(\",\", \"\") + \",\" + maxSal.replace(\",\", \"\")+ \"\\n\")\n",
    "f.close()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}