diff --git a/.travis.yml b/.travis.yml
index 08e8429b24e..34e998022c0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -42,9 +42,9 @@ matrix:
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check -Dcheckstyle.skip=true" TEST_PROJECTS=""
- # Test all modules with spark 2.0.0 and scala 2.11
+ # Test all modules with spark 2.0.1 and scala 2.11
- jdk: "oraclejdk7"
- env: SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat -Dcheckstyle.skip=true" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat -Dcheckstyle.skip=true" TEST_PROJECTS=""
+ env: SCALA_VER="2.11" SPARK_VER="2.0.1" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat -Dcheckstyle.skip=true" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat -Dcheckstyle.skip=true" TEST_PROJECTS=""
# Test all modules with scala 2.10
- jdk: "oraclejdk7"
@@ -79,9 +79,8 @@ matrix:
env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Phadoop-2.3 -Ppyspark -Pscala-2.11" BUILD_FLAG="package -pl spark,python -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python -Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
before_install:
- - mkdir -p .spark-dist
- - echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit'" >> ~/.mavenrc
- - ./testing/install_external_dependencies.sh
+ - echo "MAVEN_OPTS='-Dorg.slf4j.simpleLogger.defaultLogLevel=warn -Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit'" >> ~/.mavenrc
+ - ./testing/install_external_dependencies.sh > /dev/null 2>&1
- ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin || true
- ls .node_modules && cp -r .node_modules zeppelin-web/node_modules || echo "node_modules are not cached"
- "/sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1600x1024x16"
@@ -89,7 +88,7 @@ before_install:
- source ~/.environ
install:
- - mvn -X -Dorg.slf4j.simpleLogger.defaultLogLevel=warn $BUILD_FLAG $PROFILE -B
+ - mvn -q $BUILD_FLAG $PROFILE -B
before_script:
- travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER
@@ -97,7 +96,7 @@ before_script:
- tail conf/zeppelin-env.sh
script:
- - mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=warn $TEST_FLAG $PROFILE -B $TEST_PROJECTS
+ - mvn -q $TEST_FLAG $PROFILE -B $TEST_PROJECTS
after_success:
- echo "Travis exited with ${TRAVIS_TEST_RESULT}"
@@ -110,3 +109,4 @@ after_failure:
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out
- cat zeppelin-web/npm-debug.log
- cat spark-*/logs/*
+ - ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin || true
diff --git a/spark-dependencies/pom.xml b/spark-dependencies/pom.xml
index 2fd0714889a..f93794f861b 100644
--- a/spark-dependencies/pom.xml
+++ b/spark-dependencies/pom.xml
@@ -57,12 +57,12 @@
spark-${spark.version}
- http://archive.apache.org/dist/spark/${spark.archive}/${spark.archive}.tgz
+ http://d3kbcqa49mib13.cloudfront.net/${spark.archive}.tgz
- http://archive.apache.org/dist/spark/spark-${spark.version}/spark-${spark.version}-bin-without-hadoop.tgz
+ http://d3kbcqa49mib13.cloudfront.net/spark-${spark.version}-bin-without-hadoop.tgz
- ${project.build.directory}/../../.spark-dist
+ ${project.basedir}/../.spark-dist
0.8.2.1
@@ -823,7 +823,7 @@
com.googlecode.maven-download-plugin
download-maven-plugin
- 1.2.1
+ 1.3.0
download-pyspark-files
@@ -832,6 +832,7 @@
wget
+ 60000
5
${spark.download.url}
${spark.dist.cache}
@@ -903,7 +904,7 @@
com.googlecode.maven-download-plugin
download-maven-plugin
- 1.2.1
+ 1.3.0
download-sparkr-files
@@ -912,6 +913,8 @@
wget
+ 60000
+ 5
${spark.bin.download.url}
true
${project.build.directory}/spark-bin-dist
diff --git a/testing/downloadSpark.sh b/testing/downloadSpark.sh
index 21320bccec6..b22aeb7b257 100755
--- a/testing/downloadSpark.sh
+++ b/testing/downloadSpark.sh
@@ -16,8 +16,8 @@
# limitations under the License.
#
-if [[ "$#" -ne 2 ]]; then
- echo "usage) $0 [spark version] [hadoop version]"
+if [[ "$#" -ne 2 && "$#" -ne 1 ]]; then
+ echo "usage) $0 (spark version) [hadoop version]"
echo " eg) $0 1.3.1 2.6"
exit 0
fi
@@ -43,14 +43,31 @@ ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
#######################################
download_with_retry() {
local url="$1"
- wget --retry-connrefused --waitretry=1 --read-timeout=20 --timeout=15 -t 3 "${url}"
+ local start_time=`date +%s`
+
+ wget -v -d --retry-connrefused --waitretry=1 --read-timeout=20 --timeout=15 -t 3 "${url}"
if [[ "$?" -ne 0 ]]; then
echo "3 download attempts for ${url} failed"
fi
+ local end_time=`date +%s`
+ local download_time="$((end_time-start_time))"
}
+#Special 'mode' for downloading only Spark sourcese, required by pySpark.
+#A workaroun for failing download on CI: populate maven-download-plugin cache \w wget
+#https://github.com/travis-ci/travis-ci/issues/6950
+if [[ -z "$HADOOP_VERSION" ]]; then
+ spark_archive_file="spark-${SPARK_VERSION}.tgz"
+ mvn_plugin_cache_dir="${HOME}/.m2/repository/.cache/maven-download-plugin"
+ download_with_retry "http://d3kbcqa49mib13.cloudfront.net/${spark_archive_file}"
+ mkdir -p "${mvn_plugin_cache_dir}"
+ mv "${spark_archive_file}" "${mvn_plugin_cache_dir}"
+ exit 0
+fi
+
SPARK_CACHE=".spark-dist"
SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}"
+
export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}"
echo "SPARK_HOME is ${SPARK_HOME}"
@@ -64,11 +81,8 @@ if [[ ! -d "${SPARK_HOME}" ]]; then
# download spark from archive if not cached
echo "${SPARK_VERSION} being downloaded from archives"
- STARTTIME=`date +%s`
#timeout -s KILL "${MAX_DOWNLOAD_TIME_SEC}" wget "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz"
- download_with_retry "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz"
- ENDTIME=`date +%s`
- DOWNLOADTIME="$((ENDTIME-STARTTIME))"
+ download_with_retry "http://d3kbcqa49mib13.cloudfront.net/${SPARK_ARCHIVE}.tgz"
fi
# extract archive in un-cached root, clean-up on failure