demo-transactions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import glob\n",
    "import pyarrow as pa\n",
    "import pyarrow.parquet as pq\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "import os\n",
    "import shutil\n",
    "import seaborn as sns\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "from functools import reduce\n",
    "import string\n",
    "%matplotlib inline  \n",
    "\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_colwidth', -1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset Completion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "generate = True\n",
    "number_of_people = 636262\n",
    "number_of_transactions = 6362620\n",
    "street_points_percentage = 0.15\n",
    "square_points_percentage = 0.2\n",
    "training_path = \"/Users/joaoneves/Documents/demo-iot-transactions/data/transactions_data/transactions_with_gps.csv\"\n",
    "test_path = \"/Users/joaoneves/Documents/demo-iot-transactions/data/transactions\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "london_coordinates = (51.5073509, -0.12775829999998223)\n",
    "\n",
    "street_points_def = [\n",
    "    [(51.507306, -0.140064),(51.509734, -0.132921), \"Jermyn Street\"],\n",
    "    [(51.513534, -0.157751),(51.516433, -0.130584), \"Oxford Street\"], \n",
    "    [(51.509980, -0.137049), (51.516591, -0.142516), \"Regent Street\"], \n",
    "    [(51.513695, -0.139419), (51.512300, -0.137901), \"Carnaby Street\"]\n",
    "]\n",
    "\n",
    "square_points_def = [\n",
    "    [(51.507311, -0.221633), 0.0001, \"Westfield London\"],\n",
    "    [london_coordinates, 0.1, \"Other\"]\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def generate_street_points(p1, p2, name, n=1000):\n",
    "    m = np.subtract(p2, p1)\n",
    "    m = m[1] / m[0]\n",
    "    b = p2[1] - m*p2[0]\n",
    "    points = []\n",
    "    for i in range(n):\n",
    "        x = random.uniform(p1[0], p2[0])\n",
    "        points.append([x, x * m + b, name])\n",
    "    return points\n",
    "\n",
    "radius_dists = (random.uniform, random.uniform)\n",
    "\n",
    "def generate_square_points(center, name, radius=0.1, n=1000):\n",
    "    result = []\n",
    "    for i in range(n):\n",
    "        point = np.add(center, np.array([dist(-radius,radius) for dist in radius_dists]))\n",
    "        result.append(list(point) + [name])\n",
    "    return result\n",
    "\n",
    "def convert_to_color(string):\n",
    "    hash_ = str(hash(string))\n",
    "    if len(hash_) >= 9:\n",
    "        r,g,b= int(hash_[-9:-6]) % 255, int(hash_[-6:-3]) % 255, int(hash_[-3:]) % 255\n",
    "        return \"rgb({}, {}, {})\".format(r,b,g)\n",
    "    else:\n",
    "        return \"rgb({}, {}, {})\".format(random.randint(0,255), random.randint(0,255), random.randint(0,255))\n",
    "    \n",
    "def generate_random_string(l):\n",
    "    return ''.join(random.choice(string.ascii_letters + string.digits) for i in range(l))\n",
    "\n",
    "def generate_people(n,p):\n",
    "    return pd.Series([str(random.randrange(p)) for i in range(n)])\n",
    "\n",
    "def convert_to_category(df, columns):\n",
    "    for column in columns:\n",
    "        df[column] = df[column].astype(\"category\")\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2545048"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random.seed(0)\n",
    "square_points_list = [generate_square_points(p[0], p[2], p[1],int(number_of_transactions*square_points_percentage)) \\\n",
    "                      for p in square_points_def]\n",
    "square_points = reduce(lambda a,b: list(a)+list(b), square_points_list)\n",
    "len(square_points)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3817572"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random.seed(0)\n",
    "street_points = [generate_street_points(p[0], p[1], p[2], int(number_of_transactions*street_points_percentage)) \\\n",
    "                 for p in street_points_def]\n",
    "street_points = reduce(lambda a,b: list(a)+list(b), street_points)\n",
    "len(street_points)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random.seed(0)\n",
    "gps_points = street_points + square_points\n",
    "random.shuffle(gps_points)\n",
    "gps_points = np.array(gps_points)\n",
    "len(gps_points) == number_of_transactions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def complete_df(df, seed=0):\n",
    "    np.random.seed(seed)\n",
    "    df[\"gps_latitude\"] = pd.Series(gps_points[:,0])\n",
    "    df[\"gps_longitude\"] = pd.Series(gps_points[:,1])\n",
    "    df[\"location\"] = pd.Series(gps_points[:,2])\n",
    "    df[\"id\"] = transactions_df.index\n",
    "    df[\"entity_id\"] = generate_people(len(df), number_of_people)\n",
    "    convert_to_category(df, [\"id\", \"isFraud\", \"isFlaggedFraud\"])\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "if os.path.isfile(training_path):\n",
    "    # load dataset\n",
    "    transactions_df = pd.read_csv(training_path)\n",
    "    transactions_df[\"id\"] = transactions_df.index\n",
    "    convert_to_category(transactions_df, [\"id\", \"entity_id\", \"isFraud\", \"isFlaggedFraud\"])\n",
    "else:\n",
    "    # generate dataset and save\n",
    "    data_dir = \"/Users/joaoneves/Documents/demo-iot-transactions/data/transactions_data/PS_20174392719_1491204439457_log.csv\"\n",
    "    transactions_df = pd.read_csv(data_dir)\n",
    "    complete_df(transactions_df)\n",
    "    transactions_df.to_csv(training_path, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>step</th>\n",
       "      <th>type</th>\n",
       "      <th>amount</th>\n",
       "      <th>nameOrig</th>\n",
       "      <th>oldbalanceOrg</th>\n",
       "      <th>newbalanceOrig</th>\n",
       "      <th>nameDest</th>\n",
       "      <th>oldbalanceDest</th>\n",
       "      <th>newbalanceDest</th>\n",
       "      <th>isFraud</th>\n",
       "      <th>isFlaggedFraud</th>\n",
       "      <th>gps_latitude</th>\n",
       "      <th>gps_longitude</th>\n",
       "      <th>location</th>\n",
       "      <th>id</th>\n",
       "      <th>entity_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>PAYMENT</td>\n",
       "      <td>9839.64</td>\n",
       "      <td>C1231006815</td>\n",
       "      <td>170136.0</td>\n",
       "      <td>160296.36</td>\n",
       "      <td>M1979787155</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>51.5132</td>\n",
       "      <td>-0.138924</td>\n",
       "      <td>Carnaby Street</td>\n",
       "      <td>0</td>\n",
       "      <td>242771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>PAYMENT</td>\n",
       "      <td>1864.28</td>\n",
       "      <td>C1666544295</td>\n",
       "      <td>21249.0</td>\n",
       "      <td>19384.72</td>\n",
       "      <td>M2044282225</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>51.5074</td>\n",
       "      <td>-0.221652</td>\n",
       "      <td>Westfield London</td>\n",
       "      <td>1</td>\n",
       "      <td>543211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>TRANSFER</td>\n",
       "      <td>181.00</td>\n",
       "      <td>C1305486145</td>\n",
       "      <td>181.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>C553264065</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>51.5078</td>\n",
       "      <td>-0.138742</td>\n",
       "      <td>Jermyn Street</td>\n",
       "      <td>2</td>\n",
       "      <td>605382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>CASH_OUT</td>\n",
       "      <td>181.00</td>\n",
       "      <td>C840083671</td>\n",
       "      <td>181.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>C38997010</td>\n",
       "      <td>21182.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>51.5123</td>\n",
       "      <td>-0.137944</td>\n",
       "      <td>Carnaby Street</td>\n",
       "      <td>3</td>\n",
       "      <td>574098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>PAYMENT</td>\n",
       "      <td>11668.14</td>\n",
       "      <td>C2048537720</td>\n",
       "      <td>41554.0</td>\n",
       "      <td>29885.86</td>\n",
       "      <td>M1230701703</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>51.5136</td>\n",
       "      <td>-0.139334</td>\n",
       "      <td>Carnaby Street</td>\n",
       "      <td>4</td>\n",
       "      <td>454080</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   step      type    amount     nameOrig  oldbalanceOrg  newbalanceOrig  \\\n",
       "0  1     PAYMENT   9839.64   C1231006815  170136.0       160296.36        \n",
       "1  1     PAYMENT   1864.28   C1666544295  21249.0        19384.72         \n",
       "2  1     TRANSFER  181.00    C1305486145  181.0          0.00             \n",
       "3  1     CASH_OUT  181.00    C840083671   181.0          0.00             \n",
       "4  1     PAYMENT   11668.14  C2048537720  41554.0        29885.86         \n",
       "\n",
       "      nameDest  oldbalanceDest  newbalanceDest isFraud isFlaggedFraud  \\\n",
       "0  M1979787155  0.0             0.0             0       0               \n",
       "1  M2044282225  0.0             0.0             0       0               \n",
       "2  C553264065   0.0             0.0             1       0               \n",
       "3  C38997010    21182.0         0.0             1       0               \n",
       "4  M1230701703  0.0             0.0             0       0               \n",
       "\n",
       "  gps_latitude gps_longitude          location id entity_id  \n",
       "0  51.5132     -0.138924      Carnaby Street    0  242771    \n",
       "1  51.5074     -0.221652      Westfield London  1  543211    \n",
       "2  51.5078     -0.138742      Jermyn Street     2  605382    \n",
       "3  51.5123     -0.137944      Carnaby Street    3  574098    \n",
       "4  51.5136     -0.139334      Carnaby Street    4  454080    "
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transactions_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Streaming Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['242771', '543211', '605382', ..., '408170', '511895', '471177'], dtype=object)"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transactions_df[\"entity_id\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_files_path = os.path.join(test_path, \"transactions*.csv\")\n",
    "test_files = glob.glob(test_files_path)\n",
    "\n",
    "for file in test_files:\n",
    "    os.remove(file)\n",
    "\n",
    "for i, entity in enumerate(transactions_df[\"entity_id\"].unique()[:100]):\n",
    "    transactions_df[transactions_df[\"entity_id\"]==entity]\\\n",
    "    .to_csv(os.path.join(test_path, \"transactions_{}.csv\".format(i)), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.ml.classification import LogisticRegression, LogisticRegressionModel\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql import Row\n",
    "from pyspark.ml.feature import VectorAssembler \n",
    "\n",
    "spark = SparkSession.builder.getOrCreate()\n",
    "\n",
    "sc = spark.sparkContext\n",
    "\n",
    "training_dir = \"/Users/joaoneves/Documents/demo-iot-transactions/data/models\"\n",
    "model_path = os.path.join(training_dir, \"pythonLogisticRegression\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['step', 'type', 'amount', 'nameOrig', 'oldbalanceOrg', 'newbalanceOrig', 'nameDest', 'oldbalanceDest', 'newbalanceDest', 'isFraud', 'isFlaggedFraud', 'gps_latitude', 'gps_longitude', 'id']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[Row(step='1', type='PAYMENT', amount='9839.64', nameOrig='C1231006815', oldbalanceOrg='170136.0', newbalanceOrig='160296.36', nameDest='M1979787155', oldbalanceDest='0.0', newbalanceDest='0.0', isFraud='0', isFlaggedFraud='0', gps_latitude='51.50256769719365', gps_longitude='-0.12312710200156707', id='0'),\n",
       " Row(step='1', type='PAYMENT', amount='1864.28', nameOrig='C1666544295', oldbalanceOrg='21249.0', newbalanceOrig='19384.72', nameDest='M2044282225', oldbalanceDest='0.0', newbalanceDest='0.0', isFraud='0', isFlaggedFraud='0', gps_latitude='51.457455017283436', gps_longitude='-0.10674969663788776', id='1'),\n",
       " Row(step='1', type='TRANSFER', amount='181.0', nameOrig='C1305486145', oldbalanceOrg='181.0', newbalanceOrig='0.0', nameDest='C553264065', oldbalanceDest='0.0', newbalanceDest='0.0', isFraud='1', isFlaggedFraud='0', gps_latitude='51.467931861733966', gps_longitude='-0.11230149709138582', id='2'),\n",
       " Row(step='1', type='CASH_OUT', amount='181.0', nameOrig='C840083671', oldbalanceOrg='181.0', newbalanceOrig='0.0', nameDest='C38997010', oldbalanceDest='21182.0', newbalanceDest='0.0', isFraud='1', isFlaggedFraud='0', gps_latitude='51.44128652311481', gps_longitude='-0.19586448153175046', id='3')]"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = spark.read.csv(training_path, header=True)\n",
    "\n",
    "# data.printSchema()\n",
    "columns = data.columns\n",
    "float_columns = ['step', 'amount', 'oldbalanceOrg', 'newbalanceOrig', 'oldbalanceDest', 'newbalanceDest', 'gps_latitude', 'gps_longitude']\n",
    "target_column = 'isFraud'\n",
    "print(columns)\n",
    "data.take(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(amount=9839.64, gps_latitude=51.50256769719365, gps_longitude=-0.12312710200156707, id='0', isFlaggedFraud='0', label=0, nameDest='M1979787155', nameOrig='C1231006815', newbalanceDest=0.0, newbalanceOrig=160296.36, oldbalanceDest=0.0, oldbalanceOrg=170136.0, step=1.0, type='PAYMENT')"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def parse_row(row):\n",
    "    new_row = {}\n",
    "    for column in columns:\n",
    "        if column in float_columns:\n",
    "            new_row[column] = float(row[column])\n",
    "        elif column == target_column:\n",
    "            new_row[\"label\"] = int(round(float(row[column])))\n",
    "        else:\n",
    "            new_row[column] = row[column]\n",
    "    return Row(**new_row)\n",
    "\n",
    "data_parsed = data.rdd.map(parse_row).toDF()\n",
    "\n",
    "data_parsed.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(amount=9839.64, gps_latitude=51.50256769719365, gps_longitude=-0.12312710200156707, id='0', isFlaggedFraud='0', label=0, nameDest='M1979787155', nameOrig='C1231006815', newbalanceDest=0.0, newbalanceOrig=160296.36, oldbalanceDest=0.0, oldbalanceOrg=170136.0, step=1.0, type='PAYMENT', features=DenseVector([1.0, 9839.64, 170136.0, 160296.36, 0.0, 0.0, 51.5026, -0.1231]))"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vecAssembler = VectorAssembler(inputCols=['step','amount', \\\n",
    "                                          'oldbalanceOrg', 'newbalanceOrig', \\\n",
    "                                          'oldbalanceDest', 'newbalanceDest', \\\n",
    "                                          'gps_latitude', 'gps_longitude'], \\\n",
    "                               outputCol=\"features\")\n",
    "\n",
    "transformed_data = vecAssembler.transform(data_parsed)\n",
    "\n",
    "transformed_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr = LogisticRegression(maxIter=1000, threshold=0.01, tol=1e-5)\n",
    "\n",
    "model = lr.fit(transformed_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------+--------------------+\n",
      "|            recall|           precision|\n",
      "+------------------+--------------------+\n",
      "|               0.0|                 1.0|\n",
      "| 0.774869109947644| 0.09813415574402468|\n",
      "| 0.816997443078047|0.052232938667164865|\n",
      "| 0.843784244490442| 0.03607946854370145|\n",
      "|0.8671618166321685|0.027854150941182453|\n",
      "|0.8818945574089857|0.022683854156879693|\n",
      "|0.8892000487032753|0.019072117832939425|\n",
      "|0.8929745525386582|0.016424501934929052|\n",
      "|0.8988189455740898|0.014470509133749298|\n",
      "|0.9025934494094728|0.012920171362639737|\n",
      "|0.9075855351272373|0.011694988915334494|\n",
      "|0.9109947643979057|0.010673628348882993|\n",
      "|0.9137952027273835|0.009967050961445243|\n",
      "|0.9178132229392426| 0.00923102545450983|\n",
      "| 0.920370144892244|0.008587752653637155|\n",
      "|0.9237793741629124| 0.00803860905681168|\n",
      "|0.9262145379276756|0.007550874350949393|\n",
      "|0.9290149762571533|0.007123877032359021|\n",
      "|0.9325459637160599|0.006750045608416273|\n",
      "|0.9347376111043467| 0.00640672568498638|\n",
      "+------------------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n",
      "0.962573037988077\n"
     ]
    }
   ],
   "source": [
    "model.evaluate(transformed_data).pr.show()\n",
    "print(model.evaluate(transformed_data).areaUnderROC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+\n",
      "|           threshold|            recall|\n",
      "+--------------------+------------------+\n",
      "|                 1.0| 0.774869109947644|\n",
      "|0.006561491282644194| 0.816997443078047|\n",
      "|0.003949357813573477| 0.843784244490442|\n",
      "|0.002835492888867661|0.8671618166321685|\n",
      "|0.002107414662537...|0.8818945574089857|\n",
      "|0.001733748409879...|0.8892000487032753|\n",
      "|0.001544655807152...|0.8929745525386582|\n",
      "|0.001444811147117...|0.8988189455740898|\n",
      "|0.001352713724098...|0.9025934494094728|\n",
      "|0.001277444178936144|0.9075855351272373|\n",
      "|0.001201056567940...|0.9109947643979057|\n",
      "|0.001135738789277...|0.9117253135273348|\n",
      "|0.001115631623257...|0.9163521246803847|\n",
      "|0.001053026804150195|0.9192743211981006|\n",
      "|9.946928734003158E-4|0.9218312431511019|\n",
      "| 9.42524646181952E-4|0.9245099232923414|\n",
      "|8.910491138542175E-4|0.9277973943747717|\n",
      "|8.474195713039216E-4|0.9312066236454402|\n",
      "|8.056726289204571E-4|0.9331547546572507|\n",
      "|7.664274069040154E-4|0.9359551929867284|\n",
      "+--------------------+------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model.evaluate(transformed_data).recallByThreshold.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Save and load model\n",
    "model.write().overwrite().save(model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sameModel = LogisticRegressionModel.load(model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred = sameModel.transform(transformed_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "anomalies = pred.rdd.filter(lambda row: row[\"prediction\"]==1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "64"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tests[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "transactions_ids = anomalies.map(lambda row: (get_file_with_id(int(row[\"id\"])), row[\"id\"])).groupByKey().mapValues(lambda l: list(l)).cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(11112, ['146', '165']),\n",
       " (12224, ['1362']),\n",
       " (22224, ['1898', '1904']),\n",
       " (31112, ['2417']),\n",
       " (51112, ['3569']),\n",
       " (4224, ['8706']),\n",
       " (5224, ['9284']),\n",
       " (7224, ['10395', '10396']),\n",
       " (11336, ['12787']),\n",
       " (12336, ['13327', '13333', '13336'])]"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transactions_ids.take(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use Cases Study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>step</th>\n",
       "      <th>amount</th>\n",
       "      <th>oldbalanceOrg</th>\n",
       "      <th>newbalanceOrig</th>\n",
       "      <th>oldbalanceDest</th>\n",
       "      <th>newbalanceDest</th>\n",
       "      <th>gps_latitude</th>\n",
       "      <th>gps_longitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>6.362620e+06</td>\n",
       "      <td>6.362620e+06</td>\n",
       "      <td>6.362620e+06</td>\n",
       "      <td>6.362620e+06</td>\n",
       "      <td>6.362620e+06</td>\n",
       "      <td>6.362620e+06</td>\n",
       "      <td>6.362620e+06</td>\n",
       "      <td>6.362620e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.433972e+02</td>\n",
       "      <td>1.798619e+05</td>\n",
       "      <td>8.338831e+05</td>\n",
       "      <td>8.551137e+05</td>\n",
       "      <td>1.100702e+06</td>\n",
       "      <td>1.224996e+06</td>\n",
       "      <td>5.150732e+01</td>\n",
       "      <td>-1.277554e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.423320e+02</td>\n",
       "      <td>6.038582e+05</td>\n",
       "      <td>2.888243e+06</td>\n",
       "      <td>2.924049e+06</td>\n",
       "      <td>3.399180e+06</td>\n",
       "      <td>3.674129e+06</td>\n",
       "      <td>5.772837e-02</td>\n",
       "      <td>5.777090e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>5.140735e+01</td>\n",
       "      <td>-2.277581e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.560000e+02</td>\n",
       "      <td>1.338957e+04</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>5.145732e+01</td>\n",
       "      <td>-1.777979e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.390000e+02</td>\n",
       "      <td>7.487194e+04</td>\n",
       "      <td>1.420800e+04</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.327057e+05</td>\n",
       "      <td>2.146614e+05</td>\n",
       "      <td>5.150732e+01</td>\n",
       "      <td>-1.277567e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>3.350000e+02</td>\n",
       "      <td>2.087215e+05</td>\n",
       "      <td>1.073152e+05</td>\n",
       "      <td>1.442584e+05</td>\n",
       "      <td>9.430367e+05</td>\n",
       "      <td>1.111909e+06</td>\n",
       "      <td>5.155729e+01</td>\n",
       "      <td>-7.772319e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>7.430000e+02</td>\n",
       "      <td>9.244552e+07</td>\n",
       "      <td>5.958504e+07</td>\n",
       "      <td>4.958504e+07</td>\n",
       "      <td>3.560159e+08</td>\n",
       "      <td>3.561793e+08</td>\n",
       "      <td>5.160735e+01</td>\n",
       "      <td>-2.775831e-02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               step        amount  oldbalanceOrg  newbalanceOrig  \\\n",
       "count  6.362620e+06  6.362620e+06  6.362620e+06   6.362620e+06     \n",
       "mean   2.433972e+02  1.798619e+05  8.338831e+05   8.551137e+05     \n",
       "std    1.423320e+02  6.038582e+05  2.888243e+06   2.924049e+06     \n",
       "min    1.000000e+00  0.000000e+00  0.000000e+00   0.000000e+00     \n",
       "25%    1.560000e+02  1.338957e+04  0.000000e+00   0.000000e+00     \n",
       "50%    2.390000e+02  7.487194e+04  1.420800e+04   0.000000e+00     \n",
       "75%    3.350000e+02  2.087215e+05  1.073152e+05   1.442584e+05     \n",
       "max    7.430000e+02  9.244552e+07  5.958504e+07   4.958504e+07     \n",
       "\n",
       "       oldbalanceDest  newbalanceDest  gps_latitude  gps_longitude  \n",
       "count  6.362620e+06    6.362620e+06    6.362620e+06  6.362620e+06   \n",
       "mean   1.100702e+06    1.224996e+06    5.150732e+01 -1.277554e-01   \n",
       "std    3.399180e+06    3.674129e+06    5.772837e-02  5.777090e-02   \n",
       "min    0.000000e+00    0.000000e+00    5.140735e+01 -2.277581e-01   \n",
       "25%    0.000000e+00    0.000000e+00    5.145732e+01 -1.777979e-01   \n",
       "50%    1.327057e+05    2.146614e+05    5.150732e+01 -1.277567e-01   \n",
       "75%    9.430367e+05    1.111909e+06    5.155729e+01 -7.772319e-02   \n",
       "max    3.560159e+08    3.561793e+08    5.160735e+01 -2.775831e-02   "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transactions_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "220.0"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "6362620 / 28921"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "transactions_orig = transactions_df[\"nameOrig\"].value_counts()\n",
    "transactions_dest = transactions_df[\"nameDest\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6353307"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transactions_grouped_by_person = transactions_df.groupby(\"nameOrig\").agg(sum)\n",
    "len(transactions_grouped_by_person)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "transactions_grouped_by_destination = transactions_df.groupby(\"nameDest\").agg(sum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = np.transpose(np.array([('Jermyn Street', 11446.18),('Carnaby Street', 9839.64)])).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['11446.18', '9839.64']"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}