urad_Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "from os import listdir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save the raw data to csv and get its important features (date, spend, ctr, and cpc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ----------------------------------------------Preprocessing----------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def json_to_csv(path = 'fb ads/'): #path is the folder of the json file lists\n",
    "    listfile = listdir(path)\n",
    "\n",
    "    data_list = []\n",
    "    #open all the data, and save it to fb_ads_raw for training\n",
    "    for ctr, i in enumerate(listfile):\n",
    "        with open(path + i, 'r', encoding='utf8') as data_file:        \n",
    "            for line in data_file:\n",
    "                data = json.loads(line)\n",
    "                #pick the important features\n",
    "                data_list.append([data[\"date_start\"], data[\"spend\"], data[\"ctr\"], data[\"cpc\"]])\n",
    "    print (\"Files from the folder-%s that contains-%d files are succesfully loaded...\" % (path, ctr) )\n",
    "    data_list = pd.DataFrame(data_list)\n",
    "    data_list.columns = [\"date\",\"spend\", \"ctr\", \"cpc\"]\n",
    "    data_list.to_csv(\"fb_ads_raw.csv\", header = True, encoding=\"utf-8\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prepare_data is used to:\n",
    "- get the data\n",
    "- add season and day columns to the data\n",
    "- group the data by date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def prepare_data():\n",
    "    csv_raw = pd.read_csv(\"fb_ads_raw.csv\")\n",
    "    csv_raw.date = pd.to_datetime(csv_raw['date'])\n",
    "\n",
    "    #get detailed season and day_of_week from the date function\n",
    "    month = csv_raw.date.dt.month\n",
    "    day = csv_raw.date.dt.dayofweek\n",
    "\n",
    "    #change months into season format\n",
    "    conditions = [ (month >= 9), (month >= 6), (month >= 3)]\n",
    "    choices = [3, 2, 1]\n",
    "    season = np.select(conditions, choices, default=0)\n",
    "\n",
    "    #insert the day and the season columns\n",
    "    csv_raw.insert(1,'day', day)\n",
    "    csv_raw.insert(0,'season', season)\n",
    "    grouped = csv_raw.groupby('date').mean()\n",
    "    return grouped"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get the important features (i.e.):\n",
    "- spend\n",
    "- ctr\n",
    "- season\n",
    "- day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def important_features(csv_raw):\n",
    "    important_features = csv_raw.spend.values.reshape(-1,1)\n",
    "    important_features = np.append(important_features, csv_raw.ctr.values.reshape(-1,1), axis = 1)\n",
    "    important_features = np.append(important_features, pd.get_dummies(csv_raw.season), axis = 1)\n",
    "    important_features = np.append(important_features, pd.get_dummies(csv_raw.day), axis = 1)\n",
    "    important_features = np.append(important_features, csv_raw.cpc.values.reshape(-1,1), axis = 1)\n",
    "    print (important_features.shape)\n",
    "    return important_features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get the features and the label data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def preprocess_data():\n",
    "    csv_raw = prepare_data()   \n",
    "    \n",
    "    #normalize the ctr and spend (make it into range 0 - 1)\n",
    "    csv_raw.spend = (csv_raw.spend-csv_raw.spend.min())/(csv_raw.spend.max()-csv_raw.spend.min())\n",
    "    csv_raw.ctr = (csv_raw.ctr-csv_raw.ctr.min())/(csv_raw.ctr.max()-csv_raw.ctr.min())\n",
    "    \n",
    "    #get  the features, and return them (spend, ctr, season, and day)\n",
    "    features = pd.DataFrame( important_features(csv_raw) )\n",
    "    \n",
    "    #save data to csv\n",
    "    features.to_csv(\"fb_ads_preprocess.csv\",index=False, header=False, encoding='utf-8')\n",
    "    print (\"Preprocessing is successful!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ---------------------------------------------Main Function------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files from the folder-fb ads/ that contains-22 files are succesfully loaded...\n",
      "(510, 5)\n",
      "(510, 14)\n",
      "Preprocessing is successful!\n"
     ]
    }
   ],
   "source": [
    "def get_data_preprocessing(path = 'fb ads/'):\n",
    "    json_to_csv(path)\n",
    "    preprocess_data()\n",
    "#get_data_preprocessing()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}