Direct Spark download url for CI as INFRA-12996

.travis.yml

@@ -42,9 +42,9 @@ matrix:
     - jdk: "oraclejdk7"
       env: SCALA_VER="2.11" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check -Dcheckstyle.skip=true" TEST_PROJECTS=""
 
-    # Test all modules with spark 2.0.0 and scala 2.11
+    # Test all modules with spark 2.0.1 and scala 2.11
     - jdk: "oraclejdk7"
-      env: SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat -Dcheckstyle.skip=true" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat -Dcheckstyle.skip=true" TEST_PROJECTS=""
+      env: SCALA_VER="2.11" SPARK_VER="2.0.1" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat -Dcheckstyle.skip=true" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat -Dcheckstyle.skip=true" TEST_PROJECTS=""
 
     # Test all modules with scala 2.10
     - jdk: "oraclejdk7"
@@ -79,25 +79,24 @@ matrix:
       env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Phadoop-2.3 -Ppyspark -Pscala-2.11" BUILD_FLAG="package -pl spark,python -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python -Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
 
 before_install:
-  - mkdir -p .spark-dist
-  - echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit'" >> ~/.mavenrc
-  - ./testing/install_external_dependencies.sh
+  - echo "MAVEN_OPTS='-Dorg.slf4j.simpleLogger.defaultLogLevel=warn -Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit'" >> ~/.mavenrc
+  - ./testing/install_external_dependencies.sh > /dev/null 2>&1
   - ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin || true
   - ls .node_modules && cp -r .node_modules zeppelin-web/node_modules || echo "node_modules are not cached"
   - "/sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1600x1024x16"
   - ./dev/change_scala_version.sh $SCALA_VER
   - source ~/.environ
 
 install:
-  - mvn -X -Dorg.slf4j.simpleLogger.defaultLogLevel=warn $BUILD_FLAG $PROFILE -B
+  - mvn -q $BUILD_FLAG $PROFILE -B
 
 before_script:
   - travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER
   - echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh
   - tail conf/zeppelin-env.sh
 
 script:
-  - mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=warn $TEST_FLAG $PROFILE -B $TEST_PROJECTS
+  - mvn -q $TEST_FLAG $PROFILE -B $TEST_PROJECTS
 
 after_success:
   - echo "Travis exited with ${TRAVIS_TEST_RESULT}"
@@ -110,3 +109,4 @@ after_failure:
   - cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out
   - cat zeppelin-web/npm-debug.log
   - cat spark-*/logs/*
+  - ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin || true

spark-dependencies/pom.xml

@@ -57,12 +57,12 @@
 
     <spark.archive>spark-${spark.version}</spark.archive>
     <spark.download.url>
-      http://archive.apache.org/dist/spark/${spark.archive}/${spark.archive}.tgz
+      http://d3kbcqa49mib13.cloudfront.net/${spark.archive}.tgz
     </spark.download.url>
     <spark.bin.download.url>
-      http://archive.apache.org/dist/spark/spark-${spark.version}/spark-${spark.version}-bin-without-hadoop.tgz
+      http://d3kbcqa49mib13.cloudfront.net/spark-${spark.version}-bin-without-hadoop.tgz
     </spark.bin.download.url>
-    <spark.dist.cache>${project.build.directory}/../../.spark-dist</spark.dist.cache>
+    <spark.dist.cache>${project.basedir}/../.spark-dist</spark.dist.cache>
     <py4j.version>0.8.2.1</py4j.version>
   </properties>
 
@@ -823,7 +823,7 @@
           <plugin>
             <groupId>com.googlecode.maven-download-plugin</groupId>
             <artifactId>download-maven-plugin</artifactId>
-            <version>1.2.1</version>
+            <version>1.3.0</version>
             <executions>
               <execution>
                 <id>download-pyspark-files</id>
@@ -832,6 +832,7 @@
                   <goal>wget</goal>
                 </goals>
                 <configuration>
+                  <readTimeOut>60000</readTimeOut>
                   <retries>5</retries>
                   <url>${spark.download.url}</url>
                   <outputDirectory>${spark.dist.cache}</outputDirectory>
@@ -903,7 +904,7 @@
           <plugin>
             <groupId>com.googlecode.maven-download-plugin</groupId>
             <artifactId>download-maven-plugin</artifactId>
-            <version>1.2.1</version>
+            <version>1.3.0</version>
             <executions>
               <execution>
                 <id>download-sparkr-files</id>
@@ -912,6 +913,8 @@
                   <goal>wget</goal>
                 </goals>
                 <configuration>
+                  <readTimeOut>60000</readTimeOut>
+                  <retries>5</retries>
                   <url>${spark.bin.download.url}</url>
                   <unpack>true</unpack>
                   <outputDirectory>${project.build.directory}/spark-bin-dist</outputDirectory>

testing/downloadSpark.sh

@@ -16,8 +16,8 @@
 # limitations under the License.
 #
 
-if [[ "$#" -ne 2 ]]; then
-    echo "usage) $0 [spark version] [hadoop version]"
+if [[ "$#" -ne 2 && "$#" -ne 1 ]]; then
+    echo "usage) $0 (spark version) [hadoop version]"
     echo "   eg) $0 1.3.1 2.6"
     exit 0
 fi
@@ -43,14 +43,31 @@ ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
 #######################################
 download_with_retry() {
     local url="$1"
-    wget --retry-connrefused --waitretry=1 --read-timeout=20 --timeout=15 -t 3 "${url}"
+    local start_time=`date +%s`
+
+    wget -v -d --retry-connrefused --waitretry=1 --read-timeout=20 --timeout=15 -t 3 "${url}"
     if [[ "$?" -ne 0 ]]; then
         echo "3 download attempts for ${url} failed"
     fi
+    local end_time=`date +%s`
+    local download_time="$((end_time-start_time))"
 }
 
+#Special 'mode' for downloading only Spark sourcese, required by pySpark.
+#A workaroun for failing download on CI: populate maven-download-plugin cache \w wget
+#https://github.com/travis-ci/travis-ci/issues/6950
+if [[ -z "$HADOOP_VERSION" ]]; then
+   spark_archive_file="spark-${SPARK_VERSION}.tgz"
+   mvn_plugin_cache_dir="${HOME}/.m2/repository/.cache/maven-download-plugin"
+   download_with_retry "http://d3kbcqa49mib13.cloudfront.net/${spark_archive_file}"
+   mkdir -p "${mvn_plugin_cache_dir}"
+   mv "${spark_archive_file}" "${mvn_plugin_cache_dir}"
+   exit 0
+fi
+
 SPARK_CACHE=".spark-dist"
 SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}"
+
 export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}"
 echo "SPARK_HOME is ${SPARK_HOME}"
 
@@ -64,11 +81,8 @@ if [[ ! -d "${SPARK_HOME}" ]]; then
 
         # download spark from archive if not cached
         echo "${SPARK_VERSION} being downloaded from archives"
-        STARTTIME=`date +%s`
         #timeout -s KILL "${MAX_DOWNLOAD_TIME_SEC}" wget "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz"
-        download_with_retry "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz"
-        ENDTIME=`date +%s`
-        DOWNLOADTIME="$((ENDTIME-STARTTIME))"
+        download_with_retry "http://d3kbcqa49mib13.cloudfront.net/${SPARK_ARCHIVE}.tgz"
     fi
 
     # extract archive in un-cached root, clean-up on failure