Add training and test functions to integrate the native XGBoost library

.gitignore

@@ -15,3 +15,5 @@ scalastyle-output.xml
 scalastyle.txt
 derby.log
 spark/bin/zinc-*
+*.dylib
+*.so

.travis.yml

@@ -29,12 +29,16 @@ branches:
     - master
     - develop
 
+before_install:
+  - mvn validate -Pxgboost
+
 notifications:
   email: false
 
 script:
-  - mvn test -Pspark-1.6
-  - mvn test -Pspark-2.0
+  - mvn -q test -Pspark-2.0
+  # test the spark-1.6 module only in this second run
+  - mvn -q test -Pspark-1.6 -Dtest=org.apache.spark.*
 
 after_success:
   - mvn clean cobertura:cobertura coveralls:report

bin/build_xgboost.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# Hivemall: Hive scalable Machine Learning Library
+#
+# Copyright (C) 2015 Makoto YUI
+# Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+set -o pipefail
+
+# Target commit hash value
+XGBOOST_HASHVAL='e6f89a09074b3b25d460a99ecd195fd16b903511'
+
+# Move to a working directory
+WORKING_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Final output dir for a custom-compiled xgboost binary
+HIVEMALL_LIB_DIR="$WORKING_DIR/../xgboost/src/main/resources/lib/"
+rm -rf $HIVEMALL_LIB_DIR >> /dev/null
+mkdir -p $HIVEMALL_LIB_DIR
+
+# Move to an output directory
+XGBOOST_OUT="../target/xgboost-$XGBOOST_HASHVAL"
+rm -rf $XGBOOST_OUT >> /dev/null
+mkdir -p $XGBOOST_OUT
+cd $XGBOOST_OUT
+
+# Fetch xgboost sources
+git clone --progress https://github.com/maropu/xgboost.git
+cd xgboost
+git checkout $XGBOOST_HASHVAL
+
+# Resolve dependent sources
+git submodule init
+git submodule update
+
+# Copy a built binary to the output
+cd jvm-packages
+ENABLE_STATIC_LINKS=1 ./create_jni.sh
+cp ./lib/libxgboost4j.* "$HIVEMALL_LIB_DIR"
+

pom.xml

@@ -69,6 +69,12 @@
 	</distributionManagement>
 
 	<profiles>
+		<profile>
+			<id>xgboost</id>
+			<modules>
+				<module>xgboost</module>
+			</modules>
+		</profile>
 		<profile>
 			<id>spark-1.6</id>
 			<modules>
@@ -84,6 +90,7 @@
 			<modules>
 				<module>spark/spark-2.0</module>
 				<module>spark/spark-common</module>
+				<module>xgboost</module>
 			</modules>
 			<properties>
 				<spark.version>2.0.0</spark.version>

resources/ddl/define-all-as-permanent.hive

@@ -575,3 +575,21 @@ CREATE FUNCTION rf_ensemble as 'hivemall.smile.tools.RandomForestEnsembleUDAF' U
 DROP FUNCTION IF EXISTS guess_attribute_types;
 CREATE FUNCTION guess_attribute_types as 'hivemall.smile.tools.GuessAttributesUDF' USING JAR '${hivemall_jar}';
 
+------------------------------
+-- XGBoost related features --
+------------------------------
+
+DROP FUNCTION train_xgboost_regr;
+CREATE FUNCTION train_xgboost_regr AS 'hivemall.xgboost.regression.XGBoostRegressionUDTF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION train_xgboost_classifier;
+CREATE FUNCTION train_xgboost_classifier AS 'hivemall.xgboost.classification.XGBoostBinaryClassifierUDTF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION train_multiclass_xgboost_classifier;
+CREATE FUNCTION train_multiclass_xgboost_classifier AS 'hivemall.xgboost.classification.XGBoostMulticlassClassifierUDTF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION xgboost_predict;
+CREATE FUNCTION xgboost_predict AS 'hivemall.xgboost.tools.XGBoostPredictUDTF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION xgboost_multiclass_predict;
+CREATE FUNCTION xgboost_multiclass_predict AS 'hivemall.xgboost.tools.XGBoostMulticlassPredictUDTF' USING JAR '${hivemall_jar}';

resources/ddl/define-all.hive

@@ -571,6 +571,25 @@ create temporary function rf_ensemble as 'hivemall.smile.tools.RandomForestEnsem
 drop temporary function guess_attribute_types;
 create temporary function guess_attribute_types as 'hivemall.smile.tools.GuessAttributesUDF';
 
+------------------------------
+-- XGBoost related features --
+------------------------------
+
+drop temporary function train_xgboost_regr;
+create temporary function train_xgboost_regr as 'hivemall.xgboost.regression.XGBoostRegressionUDTF';
+
+drop temporary function train_xgboost_classifier;
+create temporary function train_xgboost_classifier as 'hivemall.xgboost.classification.XGBoostBinaryClassifierUDTF';
+
+drop temporary function train_multiclass_xgboost_classifier;
+create temporary function train_multiclass_xgboost_classifier as 'hivemall.xgboost.classification.XGBoostMulticlassClassifierUDTF';
+
+drop temporary function xgboost_predict;
+create temporary function xgboost_predict as 'hivemall.xgboost.tools.XGBoostPredictUDTF';
+
+drop temporary function xgboost_multiclass_predict;
+create temporary function xgboost_multiclass_predict as 'hivemall.xgboost.tools.XGBoostMulticlassPredictUDTF';
+
 --------------------------------------------------------------------------------------------------
 -- macros available from hive 0.12.0
 -- see https://issues.apache.org/jira/browse/HIVE-2655

resources/ddl/import-packages.spark

@@ -9,8 +9,7 @@ import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.hive.HivemallOps._
 import org.apache.spark.sql.hive.HivemallUtils
+import org.apache.spark.sql.hive.XGBoostOptions
 // Needed for implicit conversions
 import org.apache.spark.sql.hive.HivemallUtils._
-import sqlContext.implicits._
 
-val ft2vec = HivemallUtils.funcVectorizer()

spark/spark-1.6/pom.xml

@@ -9,13 +9,12 @@
 		<relativePath>../../pom.xml</relativePath>
 	</parent>
 
-	<artifactId>hivemall-spark</artifactId>
-	<name>Hivemall on Spark</name>
+	<artifactId>hivemall-spark_2.11</artifactId>
+	<name>Hivemall on Spark 1.6</name>
 	<packaging>jar</packaging>
 
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-		<spark.version>1.6.1</spark.version>
 		<scala.version>2.11.8</scala.version>
 	</properties>
 
@@ -27,18 +26,13 @@
 			<version>${project.version}</version>
 			<scope>compile</scope>
 		</dependency>
-		<dependency>
-			<groupId>io.github.myui</groupId>
-			<artifactId>hivemall-mixserv</artifactId>
-			<version>${project.version}</version>
-			<scope>compile</scope>
-		</dependency>
 		<dependency>
 			<groupId>io.github.myui</groupId>
 			<artifactId>hivemall-spark-common</artifactId>
 			<version>${project.version}</version>
 			<scope>compile</scope>
 		</dependency>
+
 		<!-- other third-party dependencies -->
 		<dependency>
 			<groupId>org.scala-lang</groupId>
@@ -64,6 +58,12 @@
 			<version>${spark.version}</version>
 			<scope>compile</scope>
 		</dependency>
+		<dependency>
+			<groupId>org.apache.spark</groupId>
+			<artifactId>spark-streaming_2.11</artifactId>
+			<version>${spark.version}</version>
+			<scope>compile</scope>
+		</dependency>
 		<dependency>
 			<groupId>org.apache.spark</groupId>
 			<artifactId>spark-mllib_2.11</artifactId>
@@ -76,7 +76,14 @@
 			<version>1.8</version>
 			<scope>compile</scope>
 		</dependency>
+
 		<!-- test dependencies -->
+		<dependency>
+			<groupId>io.github.myui</groupId>
+			<artifactId>hivemall-mixserv</artifactId>
+			<version>${project.version}</version>
+			<scope>test</scope>
+		</dependency>
 		<dependency>
 			<groupId>org.xerial</groupId>
 			<artifactId>xerial-core</artifactId>
@@ -126,7 +133,7 @@
 					</jvmArgs>
 				</configuration>
 			</plugin>
-			<!-- hivemall-spark-xx.jar -->
+			<!-- hivemall-spark_2.11-xx.jar -->
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-jar-plugin</artifactId>
@@ -136,7 +143,7 @@
 					<outputDirectory>${project.parent.build.directory}</outputDirectory>
 				</configuration>
 			</plugin>
-			<!-- hivemall-spark-xx-with-dependencies.jar including minimum dependencies -->
+			<!-- hivemall-spark_2.11-xx-with-dependencies.jar including minimum dependencies -->
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-shade-plugin</artifactId>

spark/spark-2.0/pom.xml

@@ -9,14 +9,13 @@
 		<relativePath>../../pom.xml</relativePath>
 	</parent>
 
-	<artifactId>hivemall-spark</artifactId>
-	<name>Hivemall on Spark</name>
+	<artifactId>hivemall-spark_2.11</artifactId>
+	<name>Hivemall on Spark 2.0</name>
 	<packaging>jar</packaging>
 
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<scala.version>2.11.8</scala.version>
-		<spark.version>2.0.0</spark.version>
 	</properties>
 
 	<dependencies>
@@ -29,7 +28,7 @@
 		</dependency>
 		<dependency>
 			<groupId>io.github.myui</groupId>
-			<artifactId>hivemall-mixserv</artifactId>
+			<artifactId>hivemall-xgboost</artifactId>
 			<version>${project.version}</version>
 			<scope>compile</scope>
 		</dependency>
@@ -39,6 +38,7 @@
 			<version>${project.version}</version>
 			<scope>compile</scope>
 		</dependency>
+
 		<!-- other third-party dependencies -->
 		<dependency>
 			<groupId>org.scala-lang</groupId>
@@ -64,6 +64,12 @@
 			<version>${spark.version}</version>
 			<scope>compile</scope>
 		</dependency>
+		<dependency>
+			<groupId>org.apache.spark</groupId>
+			<artifactId>spark-streaming_2.11</artifactId>
+			<version>${spark.version}</version>
+			<scope>compile</scope>
+		</dependency>
 		<dependency>
 			<groupId>org.apache.spark</groupId>
 			<artifactId>spark-mllib_2.11</artifactId>
@@ -76,7 +82,14 @@
 			<version>1.8</version>
 			<scope>compile</scope>
 		</dependency>
+
 		<!-- test dependencies -->
+		<dependency>
+			<groupId>io.github.myui</groupId>
+			<artifactId>hivemall-mixserv</artifactId>
+			<version>${project.version}</version>
+			<scope>test</scope>
+		</dependency>
 		<dependency>
 			<groupId>org.xerial</groupId>
 			<artifactId>xerial-core</artifactId>
@@ -126,7 +139,7 @@
 					</jvmArgs>
 				</configuration>
 			</plugin>
-			<!-- hivemall-spark-xx.jar -->
+			<!-- hivemall-spark_2.11-xx.jar -->
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-jar-plugin</artifactId>
@@ -136,7 +149,7 @@
 					<outputDirectory>${project.parent.build.directory}</outputDirectory>
 				</configuration>
 			</plugin>
-			<!-- hivemall-spark-xx-with-dependencies.jar including minimum dependencies -->
+			<!-- hivemall-spark_2.11-xx-with-dependencies.jar including minimum dependencies -->
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-shade-plugin</artifactId>
@@ -156,10 +169,13 @@
 							<artifactSet>
 								<includes>
 									<include>io.github.myui:hivemall-core</include>
+									<include>io.github.myui:hivemall-xgboost</include>
 									<include>io.github.myui:hivemall-spark-common</include>
 									<include>com.github.haifengl:smile-core</include>
 									<include>com.github.haifengl:smile-math</include>
 									<include>com.github.haifengl:smile-data</include>
+									<include>ml.dmlc:xgboost4j</include>
+									<include>com.esotericsoftware.kryo:kryo</include>
 								</includes>
 							</artifactSet>
 						</configuration>

spark/spark-2.0/src/main/java/hivemall/xgboost/classification/XGBoostBinaryClassifierUDTFWrapper.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package hivemall.xgboost.classification;
+
+import java.util.UUID;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+
+/** An alternative implementation of [[hivemall.xgboost.classification.XGBoostBinaryClassifierUDTF]]. */
+@Description(
+    name = "train_xgboost_classifier",
+    value = "_FUNC_(string[] features, double target [, string options]) - Returns a relation consisting of <string model_id, array<byte> pred_model>"
+)
+public class XGBoostBinaryClassifierUDTFWrapper extends XGBoostBinaryClassifierUDTF {
+    private long sequence;
+    private long taskId;
+
+    public XGBoostBinaryClassifierUDTFWrapper() {
+        this.sequence = 0L;
+        this.taskId = Thread.currentThread().getId();
+    }
+
+    @Override
+    protected String generateUniqueModelId() {
+        sequence++;
+        /**
+         * TODO: Check if it is unique over all tasks in executors of Spark.
+         */
+        return "xgbmodel-" + taskId + "-" + UUID.randomUUID() + "-" + sequence;
+    }
+}