Spark demo #2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ[\"PYSPARK_SUBMIT_ARGS\"]='--packages com.databricks:spark-csv_2.10:1.1.0 pyspark-shell'\n",
    "os.environ[\"SPARK_HOME\"]='/home/cheshire/spark/'\n",
    "\n",
    "import sys\n",
    "\n",
    "sys.path.append(os.environ.get('SPARK_HOME', None)+\"/python\")\n",
    "sys.path.append(os.environ.get('SPARK_HOME', None)+\"/python/lib/py4j-0.8.2.1-src.zip\")\n",
    "\n",
    "import py4j\n",
    "from pyspark import SparkContext,SparkConf,SQLContext\n",
    "\n",
    "conf = (SparkConf().setMaster(\"local[2]\")\n",
    "        .setAppName(\"ML demo\")\n",
    "        .set(\"spark.executor.memory\", \"2g\")\n",
    "        .set(\"spark.cores.max\", \"2\"))\n",
    "\n",
    "sc = SparkContext(conf=conf)\n",
    "\n",
    "sqlCtx = SQLContext(sc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'1.5.0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc.version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = sqlCtx.read.format('com.databricks.spark.csv').options(header='true').load(\"/home/cheshire/Documents/spark_demo/titanic_train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[PassengerId: string, Survived: string, Pclass: string, Name: string, Sex: string, Age: string, SibSp: string, Parch: string, Ticket: string, Fare: string, Cabin: string, Embarked: string]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+--------+------+--------------------+------+---+-----+-----+----------------+-------+-----+--------+\n",
      "|PassengerId|Survived|Pclass|                Name|   Sex|Age|SibSp|Parch|          Ticket|   Fare|Cabin|Embarked|\n",
      "+-----------+--------+------+--------------------+------+---+-----+-----+----------------+-------+-----+--------+\n",
      "|          1|       0|     3|Braund, Mr. Owen ...|  male| 22|    1|    0|       A/5 21171|   7.25|     |       S|\n",
      "|          2|       1|     1|Cumings, Mrs. Joh...|female| 38|    1|    0|        PC 17599|71.2833|  C85|       C|\n",
      "|          3|       1|     3|Heikkinen, Miss. ...|female| 26|    0|    0|STON/O2. 3101282|  7.925|     |       S|\n",
      "|          4|       1|     1|Futrelle, Mrs. Ja...|female| 35|    1|    0|          113803|   53.1| C123|       S|\n",
      "|          5|       0|     3|Allen, Mr. Willia...|  male| 35|    0|    0|          373450|   8.05|     |       S|\n",
      "|          6|       0|     3|    Moran, Mr. James|  male|   |    0|    0|          330877| 8.4583|     |       Q|\n",
      "|          7|       0|     1|McCarthy, Mr. Tim...|  male| 54|    0|    0|           17463|51.8625|  E46|       S|\n",
      "|          8|       0|     3|Palsson, Master. ...|  male|  2|    3|    1|          349909| 21.075|     |       S|\n",
      "|          9|       1|     3|Johnson, Mrs. Osc...|female| 27|    0|    2|          347742|11.1333|     |       S|\n",
      "|         10|       1|     2|Nasser, Mrs. Nich...|female| 14|    1|    0|          237736|30.0708|     |       C|\n",
      "|         11|       1|     3|Sandstrom, Miss. ...|female|  4|    1|    1|         PP 9549|   16.7|   G6|       S|\n",
      "|         12|       1|     1|Bonnell, Miss. El...|female| 58|    0|    0|          113783|  26.55| C103|       S|\n",
      "|         13|       0|     3|Saundercock, Mr. ...|  male| 20|    0|    0|       A/5. 2151|   8.05|     |       S|\n",
      "|         14|       0|     3|Andersson, Mr. An...|  male| 39|    1|    5|          347082| 31.275|     |       S|\n",
      "|         15|       0|     3|Vestrom, Miss. Hu...|female| 14|    0|    0|          350406| 7.8542|     |       S|\n",
      "|         16|       1|     2|Hewlett, Mrs. (Ma...|female| 55|    0|    0|          248706|     16|     |       S|\n",
      "|         17|       0|     3|Rice, Master. Eugene|  male|  2|    4|    1|          382652| 29.125|     |       Q|\n",
      "|         18|       1|     2|Williams, Mr. Cha...|  male|   |    0|    0|          244373|     13|     |       S|\n",
      "|         19|       0|     3|Vander Planke, Mr...|female| 31|    1|    0|          345763|     18|     |       S|\n",
      "|         20|       1|     3|Masselmani, Mrs. ...|female|   |    0|    0|            2649|  7.225|     |       C|\n",
      "+-----------+--------+------+--------------------+------+---+-----+-----+----------------+-------+-----+--------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(Age=u'70.5'),\n",
       " Row(Age=u'36.5'),\n",
       " Row(Age=u'50'),\n",
       " Row(Age=u'51'),\n",
       " Row(Age=u'0.75'),\n",
       " Row(Age=u'52'),\n",
       " Row(Age=u'53'),\n",
       " Row(Age=u'54'),\n",
       " Row(Age=u'55.5'),\n",
       " Row(Age=u'55'),\n",
       " Row(Age=u'56'),\n",
       " Row(Age=u'57'),\n",
       " Row(Age=u'58'),\n",
       " Row(Age=u'40.5'),\n",
       " Row(Age=u'59'),\n",
       " Row(Age=u'45.5'),\n",
       " Row(Age=u'30.5'),\n",
       " Row(Age=u'20.5'),\n",
       " Row(Age=u'1'),\n",
       " Row(Age=u'2'),\n",
       " Row(Age=u'3'),\n",
       " Row(Age=u'0.83'),\n",
       " Row(Age=u'4'),\n",
       " Row(Age=u'60'),\n",
       " Row(Age=u'5'),\n",
       " Row(Age=u'61'),\n",
       " Row(Age=u'62'),\n",
       " Row(Age=u'6'),\n",
       " Row(Age=u'7'),\n",
       " Row(Age=u'63'),\n",
       " Row(Age=u'8'),\n",
       " Row(Age=u'64'),\n",
       " Row(Age=u'65'),\n",
       " Row(Age=u'9'),\n",
       " Row(Age=u'66'),\n",
       " Row(Age=u'0.92'),\n",
       " Row(Age=u'34.5'),\n",
       " Row(Age=u'70'),\n",
       " Row(Age=u'71'),\n",
       " Row(Age=u'74'),\n",
       " Row(Age=u'24.5'),\n",
       " Row(Age=u'10'),\n",
       " Row(Age=u'11'),\n",
       " Row(Age=u'12'),\n",
       " Row(Age=u'14.5'),\n",
       " Row(Age=u'13'),\n",
       " Row(Age=u'14'),\n",
       " Row(Age=u'15'),\n",
       " Row(Age=u'16'),\n",
       " Row(Age=u'17'),\n",
       " Row(Age=u'18'),\n",
       " Row(Age=u'19'),\n",
       " Row(Age=u'80'),\n",
       " Row(Age=u'0.42'),\n",
       " Row(Age=u'20'),\n",
       " Row(Age=u'21'),\n",
       " Row(Age=u'22'),\n",
       " Row(Age=u'23.5'),\n",
       " Row(Age=u'23'),\n",
       " Row(Age=u'24'),\n",
       " Row(Age=u'25'),\n",
       " Row(Age=u'26'),\n",
       " Row(Age=u'27'),\n",
       " Row(Age=u'28.5'),\n",
       " Row(Age=u'28'),\n",
       " Row(Age=u'29'),\n",
       " Row(Age=u'30'),\n",
       " Row(Age=u'31'),\n",
       " Row(Age=u'32.5'),\n",
       " Row(Age=u'32'),\n",
       " Row(Age=u'33'),\n",
       " Row(Age=u'34'),\n",
       " Row(Age=u'35'),\n",
       " Row(Age=u'36'),\n",
       " Row(Age=u'37'),\n",
       " Row(Age=u'38'),\n",
       " Row(Age=u'39'),\n",
       " Row(Age=u''),\n",
       " Row(Age=u'40'),\n",
       " Row(Age=u'41'),\n",
       " Row(Age=u'42'),\n",
       " Row(Age=u'43'),\n",
       " Row(Age=u'0.67'),\n",
       " Row(Age=u'44'),\n",
       " Row(Age=u'45'),\n",
       " Row(Age=u'46'),\n",
       " Row(Age=u'47'),\n",
       " Row(Age=u'48'),\n",
       " Row(Age=u'49')]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.select('Age').distinct().collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(Embarked=u'C'), Row(Embarked=u'Q'), Row(Embarked=u'S'), Row(Embarked=u'')]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.select('Embarked').distinct().collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+-----+\n",
      "|Survived|count|\n",
      "+--------+-----+\n",
      "|       0|  549|\n",
      "|       1|  342|\n",
      "+--------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.groupBy(\"Survived\").count().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+-----+\n",
      "|Survived|count|\n",
      "+--------+-----+\n",
      "|       0|  549|\n",
      "|       1|  342|\n",
      "+--------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.registerTempTable(\"titanic\")\n",
    "sqlCtx.sql('SELECT Survived, count(Survived) as count FROM titanic GROUP BY Survived').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+--------+------+--------------------+------+---+-----+-----+----------------+-------+-----+--------+-------------+-------------+\n",
      "|PassengerId|Survived|Pclass|                Name|   Sex|Age|SibSp|Parch|          Ticket|   Fare|Cabin|Embarked|EmbarkedIndex|  EmbarkedVec|\n",
      "+-----------+--------+------+--------------------+------+---+-----+-----+----------------+-------+-----+--------+-------------+-------------+\n",
      "|          1|       0|     3|Braund, Mr. Owen ...|  male| 22|    1|    0|       A/5 21171|   7.25|     |       S|          0.0|(3,[0],[1.0])|\n",
      "|          2|       1|     1|Cumings, Mrs. Joh...|female| 38|    1|    0|        PC 17599|71.2833|  C85|       C|          1.0|(3,[1],[1.0])|\n",
      "|          3|       1|     3|Heikkinen, Miss. ...|female| 26|    0|    0|STON/O2. 3101282|  7.925|     |       S|          0.0|(3,[0],[1.0])|\n",
      "|          4|       1|     1|Futrelle, Mrs. Ja...|female| 35|    1|    0|          113803|   53.1| C123|       S|          0.0|(3,[0],[1.0])|\n",
      "|          5|       0|     3|Allen, Mr. Willia...|  male| 35|    0|    0|          373450|   8.05|     |       S|          0.0|(3,[0],[1.0])|\n",
      "|          6|       0|     3|    Moran, Mr. James|  male|   |    0|    0|          330877| 8.4583|     |       Q|          2.0|(3,[2],[1.0])|\n",
      "|          7|       0|     1|McCarthy, Mr. Tim...|  male| 54|    0|    0|           17463|51.8625|  E46|       S|          0.0|(3,[0],[1.0])|\n",
      "|          8|       0|     3|Palsson, Master. ...|  male|  2|    3|    1|          349909| 21.075|     |       S|          0.0|(3,[0],[1.0])|\n",
      "|          9|       1|     3|Johnson, Mrs. Osc...|female| 27|    0|    2|          347742|11.1333|     |       S|          0.0|(3,[0],[1.0])|\n",
      "|         10|       1|     2|Nasser, Mrs. Nich...|female| 14|    1|    0|          237736|30.0708|     |       C|          1.0|(3,[1],[1.0])|\n",
      "|         11|       1|     3|Sandstrom, Miss. ...|female|  4|    1|    1|         PP 9549|   16.7|   G6|       S|          0.0|(3,[0],[1.0])|\n",
      "|         12|       1|     1|Bonnell, Miss. El...|female| 58|    0|    0|          113783|  26.55| C103|       S|          0.0|(3,[0],[1.0])|\n",
      "|         13|       0|     3|Saundercock, Mr. ...|  male| 20|    0|    0|       A/5. 2151|   8.05|     |       S|          0.0|(3,[0],[1.0])|\n",
      "|         14|       0|     3|Andersson, Mr. An...|  male| 39|    1|    5|          347082| 31.275|     |       S|          0.0|(3,[0],[1.0])|\n",
      "|         15|       0|     3|Vestrom, Miss. Hu...|female| 14|    0|    0|          350406| 7.8542|     |       S|          0.0|(3,[0],[1.0])|\n",
      "|         16|       1|     2|Hewlett, Mrs. (Ma...|female| 55|    0|    0|          248706|     16|     |       S|          0.0|(3,[0],[1.0])|\n",
      "|         17|       0|     3|Rice, Master. Eugene|  male|  2|    4|    1|          382652| 29.125|     |       Q|          2.0|(3,[2],[1.0])|\n",
      "|         18|       1|     2|Williams, Mr. Cha...|  male|   |    0|    0|          244373|     13|     |       S|          0.0|(3,[0],[1.0])|\n",
      "|         19|       0|     3|Vander Planke, Mr...|female| 31|    1|    0|          345763|     18|     |       S|          0.0|(3,[0],[1.0])|\n",
      "|         20|       1|     3|Masselmani, Mrs. ...|female|   |    0|    0|            2649|  7.225|     |       C|          1.0|(3,[1],[1.0])|\n",
      "+-----------+--------+------+--------------------+------+---+-----+-----+----------------+-------+-----+--------+-------------+-------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.feature import OneHotEncoder, StringIndexer\n",
    "\n",
    "stringIndexer = StringIndexer(inputCol=\"Embarked\", outputCol=\"EmbarkedIndex\")\n",
    "model = stringIndexer.fit(df)\n",
    "indexed = model.transform(df)\n",
    "encoder = OneHotEncoder(inputCol=\"EmbarkedIndex\", outputCol=\"EmbarkedVec\")\n",
    "df_t = encoder.transform(indexed)\n",
    "df_t.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.mllib.regression import LabeledPoint\n",
    "from numpy import array\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def parse_age(str_age):\n",
    "    try:\n",
    "        return float(str_age)\n",
    "    except:\n",
    "        return -1.0\n",
    "    \n",
    "def transform_row(r):\n",
    "    return LabeledPoint(\n",
    "        int(r.Survived),\n",
    "        [\n",
    "            int(r.Pclass),\n",
    "            r.Sex == 'male',\n",
    "            parse_age(r.Age),\n",
    "            int(r.SibSp),\n",
    "            int(r.Parch),\n",
    "            float(r.Fare),\n",
    "        ] + list(r.EmbarkedVec.toArray())\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = df_t.map(transform_row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[LabeledPoint(0.0, [3.0,1.0,22.0,1.0,0.0,7.25,1.0,0.0,0.0]),\n",
       " LabeledPoint(1.0, [1.0,0.0,38.0,1.0,0.0,71.2833,0.0,1.0,0.0]),\n",
       " LabeledPoint(1.0, [3.0,0.0,26.0,0.0,0.0,7.925,1.0,0.0,0.0]),\n",
       " LabeledPoint(1.0, [1.0,0.0,35.0,1.0,0.0,53.1,1.0,0.0,0.0]),\n",
       " LabeledPoint(0.0, [3.0,1.0,35.0,0.0,0.0,8.05,1.0,0.0,0.0]),\n",
       " LabeledPoint(0.0, [3.0,1.0,-1.0,0.0,0.0,8.4583,0.0,0.0,1.0]),\n",
       " LabeledPoint(0.0, [1.0,1.0,54.0,0.0,0.0,51.8625,1.0,0.0,0.0]),\n",
       " LabeledPoint(0.0, [3.0,1.0,2.0,3.0,1.0,21.075,1.0,0.0,0.0]),\n",
       " LabeledPoint(1.0, [3.0,0.0,27.0,0.0,2.0,11.1333,1.0,0.0,0.0]),\n",
       " LabeledPoint(1.0, [2.0,0.0,14.0,1.0,0.0,30.0708,0.0,1.0,0.0])]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.take(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PythonRDD[75] at RDD at PythonRDD.scala:43"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(train, test) = data.randomSplit([0.7, 0.3])\n",
    "train.cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "602 289\n"
     ]
    }
   ],
   "source": [
    "print train.count(), test.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate_model(model, test_data):\n",
    "    example_values = test_data.map(lambda x: x.features)\n",
    "    example_target = test_data.map(lambda x: x.label)\n",
    "    \n",
    "    example_predictions = model.predict(example_values)\n",
    "    \n",
    "    classifier_eval = example_target.zip(example_predictions)\n",
    "    errors = classifier_eval.map(lambda x: abs(x[0]-x[1]))\n",
    "    \n",
    "    return 1.0-errors.sum()/errors.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from pyspark.mllib.classification import \\\n",
    "  LogisticRegressionWithLBFGS, LogisticRegressionWithSGD, SVMWithSGD, NaiveBayes\n",
    "from pyspark.mllib.tree import RandomForest, RandomForestModel\n",
    "from pyspark.mllib.tree import GradientBoostedTrees, GradientBoostedTreesModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7785467128027681"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_bf = LogisticRegressionWithLBFGS.train(train)\n",
    "evaluate_model(model_bf, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6262975778546713"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_svm = SVMWithSGD.train(train)\n",
    "evaluate_model(model_svm, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6885813148788927"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_lrsgd = LogisticRegressionWithSGD.train(train)\n",
    "evaluate_model(model_lrsgd, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.685121107266436"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_rf = RandomForest.trainClassifier(train, numClasses=2, categoricalFeaturesInfo={},\n",
    "                                     numTrees=3, featureSubsetStrategy=\"auto\",\n",
    "                                     impurity='gini', maxDepth=4, maxBins=32)\n",
    "evaluate_model(model_rf, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7958477508650519"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_rf2 = RandomForest.trainClassifier(train, numClasses=2, categoricalFeaturesInfo={},\n",
    "                                     numTrees=30, featureSubsetStrategy=\"auto\",\n",
    "                                     impurity='gini', maxDepth=4, maxBins=32)\n",
    "evaluate_model(model_rf2, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7889273356401384"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_gbt = GradientBoostedTrees.trainClassifier(train,\n",
    "    categoricalFeaturesInfo={}, numIterations=3)\n",
    "evaluate_model(model_gbt, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7854671280276817"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_gbt2 = GradientBoostedTrees.trainClassifier(train,\n",
    "    categoricalFeaturesInfo={}, numIterations=10, maxDepth=7)\n",
    "evaluate_model(model_gbt2, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}