Add training and test functions to integrate the native XGBoost library

.gitignore

@@ -15,3 +15,5 @@ scalastyle-output.xml
 scalastyle.txt
 derby.log
 spark/bin/zinc-*
+*.dylib
+*.so

.travis.yml

@@ -29,12 +29,16 @@ branches:
     - master
     - develop
 
+before_install:
+  - mvn validate -Pxgboost
+
 notifications:
   email: false
 
 script:
-  - mvn test -Pspark-1.6
-  - mvn test -Pspark-2.0
+  - mvn -q test -Pspark-2.0
+  # test the spark-1.6 module only in this second run
+  - mvn -q test -Pspark-1.6 -Dtest=org.apache.spark.*
 
 after_success:
   - mvn clean cobertura:cobertura coveralls:report

bin/build_xgboost.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# Hivemall: Hive scalable Machine Learning Library
+#
+# Copyright (C) 2015 Makoto YUI
+# Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+set -o pipefail
+
+# Target commit hash value
+XGBOOST_HASHVAL='85443403310e90bd8a90a1f817841520838b4ac7'
+
+# Move to a top directory
+if [ "$HIVEMALL_HOME" == "" ]; then
+  if [ -e ../bin/${0##*/} ]; then
+    HIVEMALL_HOME=".."
+  elif [ -e ./bin/${0##*/} ]; then
+    HIVEMALL_HOME="."
+  else
+    echo "env HIVEMALL_HOME not defined"
+    exit 1
+  fi
+fi
+
+cd $HIVEMALL_HOME
+
+# Final output dir for a custom-compiled xgboost binary
+HIVEMALL_LIB_DIR="$HIVEMALL_HOME/xgboost/src/main/resources/lib/"
+rm -rf $HIVEMALL_LIB_DIR >> /dev/null
+mkdir -p $HIVEMALL_LIB_DIR
+
+# Move to an output directory
+XGBOOST_OUT="$HIVEMALL_HOME/target/xgboost-$XGBOOST_HASHVAL"
+rm -rf $XGBOOST_OUT >> /dev/null
+mkdir -p $XGBOOST_OUT
+cd $XGBOOST_OUT
+
+# Fetch xgboost sources
+git clone --progress https://github.com/maropu/xgboost.git
+cd xgboost
+git checkout $XGBOOST_HASHVAL
+
+# Resolve dependent sources
+git submodule init
+git submodule update
+
+# Copy a built binary to the output
+cd jvm-packages
+ENABLE_STATIC_LINKS=1 ./create_jni.sh
+cp ./lib/libxgboost4j.* "$HIVEMALL_LIB_DIR"
+

pom.xml

@@ -43,6 +43,7 @@
 	<modules>
 		<module>core</module>
 		<module>nlp</module>
+		<module>xgboost</module>
 		<module>mixserv</module>
 	</modules>
 
@@ -52,6 +53,7 @@
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<protobuf.version>2.5.0</protobuf.version>
 		<protoc.path>${env.PROTOC_PATH}</protoc.path>
+		<scala.version>2.11.8</scala.version>
 	</properties>
 
 	<repositories>
@@ -70,25 +72,48 @@
 
 	<profiles>
 		<profile>
-			<id>spark-1.6</id>
+			<id>spark-2.0</id>
 			<modules>
-				<module>spark/spark-1.6</module>
+				<module>spark/spark-2.0</module>
 				<module>spark/spark-common</module>
 			</modules>
 			<properties>
-				<spark.version>1.6.1</spark.version>
+				<spark.version>2.0.0</spark.version>
 			</properties>
 		</profile>
 		<profile>
-			<id>spark-2.0</id>
+			<id>spark-1.6</id>
 			<modules>
-				<module>spark/spark-2.0</module>
+				<module>spark/spark-1.6</module>
 				<module>spark/spark-common</module>
 			</modules>
 			<properties>
-				<spark.version>2.0.0</spark.version>
+				<spark.version>1.6.1</spark.version>
 			</properties>
 		</profile>
+		<profile>
+			<id>compile-xgboost</id>
+			<build>
+				<plugins>
+					<plugin>
+						<artifactId>exec-maven-plugin</artifactId>
+						<groupId>org.codehaus.mojo</groupId>
+						<executions>
+							<execution>
+								<id>native</id>
+								<phase>generate-sources</phase>
+								<goals>
+									<goal>exec</goal>
+								</goals>
+								<configuration>
+									<executable>./bin/build_xgboost.sh</executable>
+								</configuration>
+							</execution>
+						</executions>
+					</plugin>
+				</plugins>
+			</build>
+		</profile>
 		<profile>
 			<id>doclint-java8-disable</id>
 			<activation>

resources/ddl/define-all-as-permanent.hive

@@ -575,3 +575,21 @@ CREATE FUNCTION rf_ensemble as 'hivemall.smile.tools.RandomForestEnsembleUDAF' U
 DROP FUNCTION IF EXISTS guess_attribute_types;
 CREATE FUNCTION guess_attribute_types as 'hivemall.smile.tools.GuessAttributesUDF' USING JAR '${hivemall_jar}';
 
+------------------------------
+-- XGBoost related features --
+------------------------------
+
+DROP FUNCTION train_xgboost_regr;
+CREATE FUNCTION train_xgboost_regr AS 'hivemall.xgboost.regression.XGBoostRegressionUDTF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION train_xgboost_classifier;
+CREATE FUNCTION train_xgboost_classifier AS 'hivemall.xgboost.classification.XGBoostBinaryClassifierUDTF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION train_multiclass_xgboost_classifier;
+CREATE FUNCTION train_multiclass_xgboost_classifier AS 'hivemall.xgboost.classification.XGBoostMulticlassClassifierUDTF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION xgboost_predict;
+CREATE FUNCTION xgboost_predict AS 'hivemall.xgboost.tools.XGBoostPredictUDTF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION xgboost_multiclass_predict;
+CREATE FUNCTION xgboost_multiclass_predict AS 'hivemall.xgboost.tools.XGBoostMulticlassPredictUDTF' USING JAR '${hivemall_jar}';

resources/ddl/define-all.hive

@@ -571,6 +571,25 @@ create temporary function rf_ensemble as 'hivemall.smile.tools.RandomForestEnsem
 drop temporary function guess_attribute_types;
 create temporary function guess_attribute_types as 'hivemall.smile.tools.GuessAttributesUDF';
 
+------------------------------
+-- XGBoost related features --
+------------------------------
+
+drop temporary function train_xgboost_regr;
+create temporary function train_xgboost_regr as 'hivemall.xgboost.regression.XGBoostRegressionUDTF';
+
+drop temporary function train_xgboost_classifier;
+create temporary function train_xgboost_classifier as 'hivemall.xgboost.classification.XGBoostBinaryClassifierUDTF';
+
+drop temporary function train_multiclass_xgboost_classifier;
+create temporary function train_multiclass_xgboost_classifier as 'hivemall.xgboost.classification.XGBoostMulticlassClassifierUDTF';
+
+drop temporary function xgboost_predict;
+create temporary function xgboost_predict as 'hivemall.xgboost.tools.XGBoostPredictUDTF';
+
+drop temporary function xgboost_multiclass_predict;
+create temporary function xgboost_multiclass_predict as 'hivemall.xgboost.tools.XGBoostMulticlassPredictUDTF';
+
 --------------------------------------------------------------------------------------------------
 -- macros available from hive 0.12.0
 -- see https://issues.apache.org/jira/browse/HIVE-2655

resources/ddl/import-packages.spark

@@ -9,8 +9,7 @@ import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.hive.HivemallOps._
 import org.apache.spark.sql.hive.HivemallUtils
+import org.apache.spark.sql.hive.XGBoostOptions
 // Needed for implicit conversions
 import org.apache.spark.sql.hive.HivemallUtils._
-import sqlContext.implicits._
 
-val ft2vec = HivemallUtils.funcVectorizer()

spark/spark-1.6/pom.xml

@@ -9,16 +9,10 @@
 		<relativePath>../../pom.xml</relativePath>
 	</parent>
 
-	<artifactId>hivemall-spark</artifactId>
-	<name>Hivemall on Spark</name>
+	<artifactId>hivemall-spark-${spark.version}_${scala.version}</artifactId>
+	<name>Hivemall on Spark 1.6</name>
 	<packaging>jar</packaging>
 
-	<properties>
-		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-		<spark.version>1.6.1</spark.version>
-		<scala.version>2.11.8</scala.version>
-	</properties>
-
 	<dependencies>
 		<!-- hivemall dependencies -->
 		<dependency>
@@ -27,18 +21,13 @@
 			<version>${project.version}</version>
 			<scope>compile</scope>
 		</dependency>
-		<dependency>
-			<groupId>io.github.myui</groupId>
-			<artifactId>hivemall-mixserv</artifactId>
-			<version>${project.version}</version>
-			<scope>compile</scope>
-		</dependency>
 		<dependency>
 			<groupId>io.github.myui</groupId>
 			<artifactId>hivemall-spark-common</artifactId>
 			<version>${project.version}</version>
 			<scope>compile</scope>
 		</dependency>
+
 		<!-- other third-party dependencies -->
 		<dependency>
 			<groupId>org.scala-lang</groupId>
@@ -64,6 +53,12 @@
 			<version>${spark.version}</version>
 			<scope>compile</scope>
 		</dependency>
+		<dependency>
+			<groupId>org.apache.spark</groupId>
+			<artifactId>spark-streaming_2.11</artifactId>
+			<version>${spark.version}</version>
+			<scope>compile</scope>
+		</dependency>
 		<dependency>
 			<groupId>org.apache.spark</groupId>
 			<artifactId>spark-mllib_2.11</artifactId>
@@ -76,7 +71,14 @@
 			<version>1.8</version>
 			<scope>compile</scope>
 		</dependency>
+
 		<!-- test dependencies -->
+		<dependency>
+			<groupId>io.github.myui</groupId>
+			<artifactId>hivemall-mixserv</artifactId>
+			<version>${project.version}</version>
+			<scope>test</scope>
+		</dependency>
 		<dependency>
 			<groupId>org.xerial</groupId>
 			<artifactId>xerial-core</artifactId>
@@ -126,17 +128,17 @@
 					</jvmArgs>
 				</configuration>
 			</plugin>
-			<!-- hivemall-spark-xx.jar -->
+			<!-- hivemall-spark_2.11-xx.jar -->
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-jar-plugin</artifactId>
 				<version>2.5</version>
 				<configuration>
-					<finalName>${project.artifactId}-v1.6-${project.version}</finalName>
+					<finalName>${project.artifactId}-${project.version}</finalName>
 					<outputDirectory>${project.parent.build.directory}</outputDirectory>
 				</configuration>
 			</plugin>
-			<!-- hivemall-spark-xx-with-dependencies.jar including minimum dependencies -->
+			<!-- hivemall-spark_2.11-xx-with-dependencies.jar including minimum dependencies -->
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-shade-plugin</artifactId>
@@ -149,7 +151,7 @@
 							<goal>shade</goal>
 						</goals>
 						<configuration>
-							<finalName>${project.artifactId}-v1.6-${project.version}-with-dependencies</finalName>
+							<finalName>${project.artifactId}-${project.version}-with-dependencies</finalName>
 							<outputDirectory>${project.parent.build.directory}</outputDirectory>
 							<minimizeJar>false</minimizeJar>
 							<createDependencyReducedPom>false</createDependencyReducedPom>