[WIP] Automated test with Yarn cluster for Spark integration.

.travis.yml

@@ -22,35 +22,34 @@ before_install:
   - "sh -e /etc/init.d/xvfb start"
 
 install:
-  - mvn package -DskipTests -Phadoop-2.3 -Ppyspark -B
+  - mvn package -DskipTests -Phadoop-2.4 -Ppyspark -B
 
 before_script:
-  -
+  - ./testing/startYarnCluster.sh 2.5.2
 
 script:
 # spark 1.4
-  - mvn package -Pbuild-distr -Phadoop-2.3 -Ppyspark -B
-  - ./testing/startSparkCluster.sh 1.4.0 2.3
-  - SPARK_HOME=`pwd`/spark-1.4.0-bin-hadoop2.3 mvn verify -Pusing-packaged-distr -Phadoop-2.3 -Ppyspark -B
-  - ./testing/stopSparkCluster.sh 1.4.0 2.3
+  - mvn package -Pbuild-distr -Pyarn -Phadoop-2.4 -Ppyspark -B
+  - ./testing/startSparkCluster.sh 1.4.1 2.4
+  - SPARK_HOME=`pwd`/spark-1.4.1-bin-hadoop2.5 mvn verify -Pusing-packaged-distr -Pyarn -Phadoop-2.4 -B
+  - ./testing/stopSparkCluster.sh 1.4.1 2.4
+  - SPARK_HOME=`pwd`/spark-1.4.1-bin-hadoop2.5 HADOOP_CONF_DIR=./hadoop-2.5.2/etc/hadoop YARN=true mvn verify -Pusing-packaged-distr -Pyarn -Phadoop-2.4 -B  
+
  # spark 1.3
-  - mvn clean package -DskipTests -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark'
-  - mvn package -Pbuild-distr -Pspark-1.3 -Phadoop-2.3 -B
-  - ./testing/startSparkCluster.sh 1.3.1 2.3
-  - SPARK_HOME=./spark-1.3.1-bin-hadoop2.3 mvn verify -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark'
-  - ./testing/stopSparkCluster.sh 1.3.1 2.3
+  - mvn clean package -DskipTests -Pspark-1.3 -Phadoop-2.4 -B -pl 'zeppelin-interpreter,spark'
+  - ./testing/startSparkCluster.sh 1.3.1 2.4
+  - SPARK_HOME=`pwd`/spark-1.3.1-bin-hadoop2.4 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server'
+  - ./testing/stopSparkCluster.sh 1.3.1 2.4
 # spark 1.2
-  - mvn clean package -DskipTests -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark'
-  - mvn package -Pbuild-distr -Pspark-1.2 -Phadoop-2.3 -B
-  - ./testing/startSparkCluster.sh 1.2.1 2.3
-  - SPARK_HOME=./spark-1.2.1-bin-hadoop2.3 mvn verify -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark'
-  - ./testing/stopSparkCluster.sh 1.2.1 2.3
+  - mvn clean package -DskipTests -Pspark-1.2 -Phadoop-2.4 -B -pl 'zeppelin-interpreter,spark'
+  - ./testing/startSparkCluster.sh 1.2.1 2.4
+  - SPARK_HOME=`pwd`/spark-1.2.1-bin-hadoop2.4 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server'
+  - ./testing/stopSparkCluster.sh 1.2.1 2.4
 # spark 1.1
-  - mvn clean package -DskipTests -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark'
-  - mvn package -Pbuild-distr -Pspark-1.1 -Phadoop-2.3 -B
-  - ./testing/startSparkCluster.sh 1.1.1 2.3
-  - SPARK_HOME=./spark-1.1.1-bin-hadoop2.3 mvn verify -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark'
-  - ./testing/stopSparkCluster.sh 1.1.1 2.3
+  - mvn clean package -DskipTests -Pspark-1.1 -Phadoop-2.4 -B -pl 'zeppelin-interpreter,spark'
+  - ./testing/startSparkCluster.sh 1.1.1 2.4
+  - SPARK_HOME=`pwd`/spark-1.1.1-bin-hadoop2.4 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server'
+  - ./testing/stopSparkCluster.sh 1.1.1 2.4
 
 after_failure:
   - cat target/rat.txt
@@ -59,7 +58,7 @@ after_failure:
   - cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out
 
 after_script:
-  -
+  - ./testing/stopYarnCluster.sh 2.5.2
 
 notifications:
   slack:

pom.xml

@@ -431,6 +431,7 @@
               <exclude>Roadmap.md</exclude>
               <exclude>conf/interpreter.json</exclude>
               <exclude>spark-*-bin*/**</exclude>
+              <exclude>hadoop-*/**</exclude>
             </excludes>
           </configuration>
 

testing/startYarnCluster.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+if [ $# -ne 1 ]; then
+    echo "usage) $0 [hadoop version]"
+    echo "   eg) $0 2.6.0"
+    exit 1
+fi
+
+
+
+HADOOP_VERSION="${1}"
+
+FWDIR=$(dirname "${BASH_SOURCE-$0}")
+ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
+export HADOOP_HOME=${ZEPPELIN_HOME}/hadoop-${HADOOP_VERSION}
+
+if [ ! -d "${HADOOP_HOME}" ]; then    
+    wget -q http://www.us.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz
+    tar zxf hadoop-${HADOOP_VERSION}.tar.gz
+fi
+
+DEFAULT_LIBEXEC_DIR=${HADOOP_HOME}/libexec
+HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
+. $HADOOP_LIBEXEC_DIR/yarn-config.sh
+
+# start resourceManager
+$HADOOP_HOME/sbin/yarn-daemon.sh --config $YARN_CONF_DIR start resourcemanager
+
+# start nodeManager
+$HADOOP_HOME/sbin/yarn-daemon.sh --config $YARN_CONF_DIR start nodemanager

testing/stopYarnCluster.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ $# -ne 1 ]; then
+    echo "usage) $0 [hadoop version]"
+    echo "   eg) $0 2.6.0"
+    exit 1
+fi
+
+HADOOP_VERSION="${1}"
+
+FWDIR=$(dirname "${BASH_SOURCE-$0}")
+ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
+export HADOOP_HOME=${ZEPPELIN_HOME}/hadoop-${HADOOP_VERSION}
+
+DEFAULT_LIBEXEC_DIR=${HADOOP_HOME}/libexec
+HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
+. $HADOOP_LIBEXEC_DIR/yarn-config.sh
+
+# stop nodeManager
+$HADOOP_HOME/sbin/yarn-daemon.sh --config $YARN_CONF_DIR stop nodemanager
+
+# stop resourceManager
+$HADOOP_HOME/sbin/yarn-daemon.sh --config $YARN_CONF_DIR stop resourcemanager

zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java

@@ -112,7 +112,11 @@ protected static void startUp() throws Exception {
         InterpreterSetting sparkIntpSetting = ZeppelinServer.notebook.getInterpreterFactory().get().get(0);
 
         // set spark master
-        sparkIntpSetting.getProperties().setProperty("master", "spark://" + getHostname() + ":7071");
+        if ("true".equals(System.getenv("YARN"))) {
+          sparkIntpSetting.getProperties().setProperty("master", "yarn-client");
+        } else {
+          sparkIntpSetting.getProperties().setProperty("master", "spark://" + getHostname() + ":7071");
+        }
 
         // set spark home for pyspark
         sparkIntpSetting.getProperties().setProperty("spark.home", getSparkHome());
@@ -123,6 +127,13 @@ protected static void startUp() throws Exception {
         // assume first one is spark
         InterpreterSetting sparkIntpSetting = ZeppelinServer.notebook.getInterpreterFactory().get().get(0);
 
+        // set spark master
+        if ("true".equals(System.getenv("YARN"))) {
+          sparkIntpSetting.getProperties().setProperty("master", "yarn-client");
+        } else {
+          sparkIntpSetting.getProperties().setProperty("master", "spark://" + getHostname() + ":7071");
+        }
+
         String sparkHome = getSparkHome();
         if (sparkHome != null) {
           // set spark home for pyspark
@@ -145,9 +156,13 @@ private static String getHostname() {
   }
 
   private static String getSparkHome() {
-    String sparkHome = getSparkHomeRecursively(new File(System.getProperty("user.dir")));
-    System.out.println("SPARK HOME detected " + sparkHome);
-    return sparkHome;
+    if (System.getenv("SPARK_HOME") != null) {
+      return new File(System.getenv("SPARK_HOME")).getAbsolutePath();
+    } else { // autodetect spark.home that ./testing/startSparkCluster.sh creates 
+      String sparkHome = getSparkHomeRecursively(new File(System.getProperty("user.dir")));
+      System.out.println("SPARK HOME detected " + sparkHome);
+      return sparkHome;
+    }
   }
 
   boolean isPyspark() {