Notebook: HDFS as a backend storage (Read & Write Mode)

.appveyor.yml

@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+version: '1.0.0-dev.{build}'
+
+shallow_clone: true
+
+build: off
+
+os:
+  - Visual Studio 2015
+
+install:
+  - echo "Install"
+
+build_script:
+  - echo "Build"

.github/PULL_REQUEST_TEMPLATE

@@ -1,6 +1,6 @@
 ### What is this PR for?
 A few sentences describing the overall goals of the pull request's commits.
-First time? Check out the contributing guide - https://github.com/apache/zeppelin/blob/master/CONTRIBUTING.md
+First time? Check out the contributing guide - https://zeppelin.apache.org/contribution/contributions.html
 
 
 ### What type of PR is it?

.gitignore

@@ -1,56 +1,68 @@
 *.class
+*.pyc
 
 # Package Files #
 *.jar
 *.war
 *.ear
 
 # interpreter
-/interpreter/
+/interpreter/*
+!/interpreter/lib
 
 # interpreter temp files
-spark/derby.log
+derby.log
 spark/metastore_db
 spark-1.*-bin-hadoop*
 .spark-dist
-zeppelin-server/derby.log
 
 lens/lens-cli-hist.log
 
+
 # conf file
 conf/zeppelin-env.sh
 conf/zeppelin-env.cmd
 conf/zeppelin-site.xml
+conf/shiro.ini
 conf/keystore
 conf/truststore
 conf/interpreter.json
 conf/notebook-authorization.json
 conf/shiro.ini
 conf/credentials.json
+conf/helium.json
 
 # other generated files
 spark/dependency-reduced-pom.xml
 reports
 
-#webapp
+# webapp
 zeppelin-web/node_modules
 zeppelin-web/dist
 zeppelin-web/.tmp
 zeppelin-web/src/fonts/Roboto*
 zeppelin-web/src/fonts/Source-Code-Pro*
 zeppelin-web/src/fonts/Patua-One*
 zeppelin-web/.sass-cache
+zeppelin-web/npm-debug.log
+zeppelin-web/yarn-error.log
 zeppelin-web/bower_components
+zeppelin-web/yarn.lock
 **nbproject/
 **node/
 
-#R
+# R
 /r/lib/
+.Rhistory
+/R/
+
+# scio
+.bigquery/
 
 # project level
 /logs/
 /run/
-/metastore_db/
+**/metastore_db/
 /*.log
 /jobs/
 /zan-repo/
@@ -83,14 +95,17 @@ Thumbs.db
 .project
 .settings/
 
-#intelliJ IDEA project files
+# intelliJ IDEA project files
 .idea/
 *.iml
 
 # maven target files
 target/
 **/target/
 
+# maven flattened pom files
+**/.flattened-pom.xml
+
 # Generated by Jekyll 
 docs/_site/
 
@@ -104,3 +119,9 @@ tramp
 .\#*
 *.swp
 **/dependency-reduced-pom.xml
+
+# Generated by zeppelin-examples
+/helium
+
+# tmp files
+/tmp/

.travis.yml

@@ -18,90 +18,125 @@ language: java
 sudo: false
 
 cache:
+  apt: true
   directories:
     - .spark-dist
-    - ${HOME}/.m2/repository/.cache/maven-download-plugin
+    - ${HOME}/.m2
+    - ${HOME}/R
+    -  zeppelin-web/node
+    -  zeppelin-web/node_modules
+    -  zeppelin-web/bower_components
 
 addons:
   apt:
     sources:
     - r-packages-precise
     packages:
     - r-base-dev
-    - r-cran-evaluate
-    - r-cran-base64enc
+
+env:
+  global:
+    # Interpreters does not required by zeppelin-server integration tests
+    - INTERPRETERS='!hbase,!pig,!jdbc,!file,!flink,!ignite,!kylin,!python,!lens,!cassandra,!elasticsearch,!bigquery,!alluxio,!scio,!livy,!groovy'
 
 matrix:
   include:
-    # Test all modules with spark 2.0.0 and scala 2.11
+    # Test License compliance using RAT tool
     - jdk: "oraclejdk7"
-      env: SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
-
-    # Test all modules with scala 2.10
+      env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
+
+    # Test core modules
+    #
+    # Several tests were excluded from this configuration due to the following issues:
+    # HeliumBundleFactoryTest      - https://issues.apache.org/jira/browse/ZEPPELIN-2469
+    # HeliumApplicationFactoryTest - https://issues.apache.org/jira/browse/ZEPPELIN-2470
+    # NotebookTest                 - https://issues.apache.org/jira/browse/ZEPPELIN-2471
+    # ZeppelinRestApiTest          - https://issues.apache.org/jira/browse/ZEPPELIN-2473
+    # After issues are fixed these tests need to be included back by removing them from the "-Dtests.to.exclude" property
     - jdk: "oraclejdk7"
-      env: SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pscala-2.10" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
+      env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/ZeppelinSparkClusterTest.java,**/org.apache.zeppelin.spark.*,**/HeliumBundleFactoryTest.java,**/HeliumApplicationFactoryTest.java,**/NotebookTest.java,**/ZeppelinRestApiTest.java -DfailIfNoTests=false"
 
-    # Test all modules with scala 2.11
+    # Test selenium with spark module for 1.6.3
     - jdk: "oraclejdk7"
-      env: SCALA_VER="2.11" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
+      env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
 
-    # Test spark module for 1.5.2
+    # Test interpreter modules
     - jdk: "oraclejdk7"
-      env: SCALA_VER="2.10" SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
+      env: SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS=""
 
-    # Test spark module for 1.4.1
+    # Test spark module for 2.1.0 with scala 2.11, livy
     - jdk: "oraclejdk7"
-      env: SCALA_VER="2.10" SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
+      env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.1 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
 
-    # Test spark module for 1.3.1
+    # Test spark module for 2.0.2 with scala 2.11
     - jdk: "oraclejdk7"
-      env: SCALA_VER="2.10" SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
+      env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
 
-    # Test spark module for 1.2.2
+    # Test spark module for 1.6.3 with scala 2.10
     - jdk: "oraclejdk7"
-      env: SCALA_VER="2.10" SPARK_VER="1.2.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
+      env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
 
-    # Test spark module for 1.1.1
+    # Test spark module for 1.6.3 with scala 2.11
     - jdk: "oraclejdk7"
-      env: SCALA_VER="2.10" SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
+      env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
 
-    # Test selenium with spark module for 1.6.1
-    - jdk: "oraclejdk7"
-      env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
+    # Test python/pyspark with python 2, livy 0.2
+    - sudo: required
+      jdk: "oraclejdk7"
+      env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" LIVY_VER="0.2.0" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Plivy-0.2" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
+
+    # Test python/pyspark with python 3, livy 0.3
+    - sudo: required
+      jdk: "oraclejdk7"
+      env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" LIVY_VER="0.3.0" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Pscala-2.11 -Plivy-0.3" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
 
 before_install:
-  - "ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin"
-  - mkdir -p ~/R
-  - echo 'R_LIBS=~/R' > ~/.Renviron
-  - R -e "install.packages('knitr', repos = 'http://cran.us.r-project.org', lib='~/R')"
-  - export R_LIBS='~/R'
+  # check files included in commit range, clear bower_components if a bower.json file has changed.
+  # bower cache clearing can also be forced by putting "bower clear" or "clear bower" in a commit message
+  - changedfiles=$(git diff --name-only $TRAVIS_COMMIT_RANGE 2>/dev/null) || changedfiles=""
+  - echo $changedfiles
+  - hasbowerchanged=$(echo $changedfiles | grep -c "bower.json" || true);
+  - gitlog=$(git log $TRAVIS_COMMIT_RANGE 2>/dev/null) || gitlog="" 
+  - clearcache=$(echo $gitlog | grep -c -E "clear bower|bower clear" || true)
+  - if [ "$hasbowerchanged" -gt 0 ] || [ "$clearcache" -gt 0 ]; then echo "Clearing bower_components cache"; rm -r zeppelin-web/bower_components; npm cache clear; else echo "Using cached bower_components."; fi
+  - echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=warn'" >> ~/.mavenrc
+  - ./testing/install_external_dependencies.sh
+  - ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin || true
+  - ls .node_modules && cp -r .node_modules zeppelin-web/node_modules || echo "node_modules are not cached"
   - "/sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1600x1024x16"
   - ./dev/change_scala_version.sh $SCALA_VER
+  - source ~/.environ
 
 install:
-  - mvn $BUILD_FLAG $PROFILE -B
+  - mvn $BUILD_FLAG $MODULES $PROFILE -B
 
 before_script:
-  - travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER
-  - ./testing/startSparkCluster.sh $SPARK_VER $HADOOP_VER
+  - if [[ -n $SPARK_VER ]]; then travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER; fi
+  - if [[ -n $LIVY_VER ]]; then ./testing/downloadLivy.sh $LIVY_VER; fi
+  - if [[ -n $LIVY_VER ]]; then export LIVY_HOME=`pwd`/livy-server-$LIVY_VER; fi
+  - if [[ -n $LIVY_VER ]]; then export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER; fi
   - echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh
+  - echo "export ZEPPELIN_HELIUM_REGISTRY=helium" >> conf/zeppelin-env.sh
   - tail conf/zeppelin-env.sh
 
 script:
-  - mvn $TEST_FLAG $PROFILE -B $TEST_PROJECTS
-
+  - mvn $TEST_FLAG $MODULES $PROFILE -B $TEST_PROJECTS
 
 after_success:
   - echo "Travis exited with ${TRAVIS_TEST_RESULT}"
 
 after_failure:
   - echo "Travis exited with ${TRAVIS_TEST_RESULT}"
-  - cat target/rat.txt
-  - cat zeppelin-server/target/rat.txt
+  - find . -name rat.txt | xargs cat
   - cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.log
   - cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out
   - cat zeppelin-web/npm-debug.log
   - cat spark-*/logs/*
-
-after_script:
-  - ./testing/stopSparkCluster.sh $SPARK_VER $HADOOP_VER
+  - cat livy/target/tmp/*/output.log
+  - ls -R livy/target/tmp/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*
+  - cat livy/target/tmp/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*/*/*/stdout
+  - cat livy/target/tmp/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*/*/*/stderr
+  - cat livy/target/tmp/livy-int-test/*/output.log
+  - ls -R livy/target/tmp/livy-int-test/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*
+  - cat livy/target/tmp/livy-int-test/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*/*/*/stdout
+  - cat livy/target/tmp/livy-int-test/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*/*/*/stderr