diff --git a/.travis.yml b/.travis.yml index 08e8429b24e..34e998022c0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,9 +42,9 @@ matrix: - jdk: "oraclejdk7" env: SCALA_VER="2.11" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check -Dcheckstyle.skip=true" TEST_PROJECTS="" - # Test all modules with spark 2.0.0 and scala 2.11 + # Test all modules with spark 2.0.1 and scala 2.11 - jdk: "oraclejdk7" - env: SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat -Dcheckstyle.skip=true" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat -Dcheckstyle.skip=true" TEST_PROJECTS="" + env: SCALA_VER="2.11" SPARK_VER="2.0.1" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat -Dcheckstyle.skip=true" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat -Dcheckstyle.skip=true" TEST_PROJECTS="" # Test all modules with scala 2.10 - jdk: "oraclejdk7" @@ -79,9 +79,8 @@ matrix: env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Phadoop-2.3 -Ppyspark -Pscala-2.11" BUILD_FLAG="package -pl spark,python -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python -Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" before_install: - - mkdir -p .spark-dist - - echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit'" >> ~/.mavenrc - - ./testing/install_external_dependencies.sh + - echo "MAVEN_OPTS='-Dorg.slf4j.simpleLogger.defaultLogLevel=warn -Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit'" >> ~/.mavenrc + - ./testing/install_external_dependencies.sh > /dev/null 2>&1 - ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin || true - ls .node_modules && cp -r .node_modules zeppelin-web/node_modules || echo "node_modules are not cached" - "/sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1600x1024x16" @@ -89,7 +88,7 @@ before_install: - source ~/.environ install: - - mvn -X -Dorg.slf4j.simpleLogger.defaultLogLevel=warn $BUILD_FLAG $PROFILE -B + - mvn -q $BUILD_FLAG $PROFILE -B before_script: - travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER @@ -97,7 +96,7 @@ before_script: - tail conf/zeppelin-env.sh script: - - mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=warn $TEST_FLAG $PROFILE -B $TEST_PROJECTS + - mvn -q $TEST_FLAG $PROFILE -B $TEST_PROJECTS after_success: - echo "Travis exited with ${TRAVIS_TEST_RESULT}" @@ -110,3 +109,4 @@ after_failure: - cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out - cat zeppelin-web/npm-debug.log - cat spark-*/logs/* + - ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin || true diff --git a/spark-dependencies/pom.xml b/spark-dependencies/pom.xml index 2fd0714889a..f93794f861b 100644 --- a/spark-dependencies/pom.xml +++ b/spark-dependencies/pom.xml @@ -57,12 +57,12 @@ spark-${spark.version} - http://archive.apache.org/dist/spark/${spark.archive}/${spark.archive}.tgz + http://d3kbcqa49mib13.cloudfront.net/${spark.archive}.tgz - http://archive.apache.org/dist/spark/spark-${spark.version}/spark-${spark.version}-bin-without-hadoop.tgz + http://d3kbcqa49mib13.cloudfront.net/spark-${spark.version}-bin-without-hadoop.tgz - ${project.build.directory}/../../.spark-dist + ${project.basedir}/../.spark-dist 0.8.2.1 @@ -823,7 +823,7 @@ com.googlecode.maven-download-plugin download-maven-plugin - 1.2.1 + 1.3.0 download-pyspark-files @@ -832,6 +832,7 @@ wget + 60000 5 ${spark.download.url} ${spark.dist.cache} @@ -903,7 +904,7 @@ com.googlecode.maven-download-plugin download-maven-plugin - 1.2.1 + 1.3.0 download-sparkr-files @@ -912,6 +913,8 @@ wget + 60000 + 5 ${spark.bin.download.url} true ${project.build.directory}/spark-bin-dist diff --git a/testing/downloadSpark.sh b/testing/downloadSpark.sh index 21320bccec6..b22aeb7b257 100755 --- a/testing/downloadSpark.sh +++ b/testing/downloadSpark.sh @@ -16,8 +16,8 @@ # limitations under the License. # -if [[ "$#" -ne 2 ]]; then - echo "usage) $0 [spark version] [hadoop version]" +if [[ "$#" -ne 2 && "$#" -ne 1 ]]; then + echo "usage) $0 (spark version) [hadoop version]" echo " eg) $0 1.3.1 2.6" exit 0 fi @@ -43,14 +43,31 @@ ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)" ####################################### download_with_retry() { local url="$1" - wget --retry-connrefused --waitretry=1 --read-timeout=20 --timeout=15 -t 3 "${url}" + local start_time=`date +%s` + + wget -v -d --retry-connrefused --waitretry=1 --read-timeout=20 --timeout=15 -t 3 "${url}" if [[ "$?" -ne 0 ]]; then echo "3 download attempts for ${url} failed" fi + local end_time=`date +%s` + local download_time="$((end_time-start_time))" } +#Special 'mode' for downloading only Spark sourcese, required by pySpark. +#A workaroun for failing download on CI: populate maven-download-plugin cache \w wget +#https://github.com/travis-ci/travis-ci/issues/6950 +if [[ -z "$HADOOP_VERSION" ]]; then + spark_archive_file="spark-${SPARK_VERSION}.tgz" + mvn_plugin_cache_dir="${HOME}/.m2/repository/.cache/maven-download-plugin" + download_with_retry "http://d3kbcqa49mib13.cloudfront.net/${spark_archive_file}" + mkdir -p "${mvn_plugin_cache_dir}" + mv "${spark_archive_file}" "${mvn_plugin_cache_dir}" + exit 0 +fi + SPARK_CACHE=".spark-dist" SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" + export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}" echo "SPARK_HOME is ${SPARK_HOME}" @@ -64,11 +81,8 @@ if [[ ! -d "${SPARK_HOME}" ]]; then # download spark from archive if not cached echo "${SPARK_VERSION} being downloaded from archives" - STARTTIME=`date +%s` #timeout -s KILL "${MAX_DOWNLOAD_TIME_SEC}" wget "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz" - download_with_retry "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz" - ENDTIME=`date +%s` - DOWNLOADTIME="$((ENDTIME-STARTTIME))" + download_with_retry "http://d3kbcqa49mib13.cloudfront.net/${SPARK_ARCHIVE}.tgz" fi # extract archive in un-cached root, clean-up on failure