diff --git a/README.md b/README.md index 93df3121745..46e188ca09b 100644 --- a/README.md +++ b/README.md @@ -85,10 +85,8 @@ If you set `SPARK_HOME`, you should deploy spark binary on the same location to Yarn # ./conf/zeppelin-env.sh - export SPARK_YARN_JAR=/path/to/spark-assembly-*.jar export HADOOP_CONF_DIR=/path/to/hadoop_conf_dir - -`SPARK_YARN_JAR` is deployed for running executor, this could be a local path or HDFS. HDFS allows YARN to cache it on nodes so that it doesn't need to be distributed each time an application runs. To point to a jar on HDFS, for example, set this configuration to "hdfs:///some/path". + `HADOOP_CONF_DIR` should contains yarn-site.xml and core-site.xml. ### Run diff --git a/spark/pom.xml b/spark/pom.xml index 49b4d082a01..fcecc1bc3e9 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -1,5 +1,6 @@ - + 4.0.0 @@ -9,7 +10,6 @@ 0.5.0-SNAPSHOT - com.nflabs.zeppelin zeppelin-spark jar 0.5.0-SNAPSHOT @@ -67,7 +67,7 @@ org.scala-lang scala-library - + org.scala-lang scala-compiler @@ -77,7 +77,7 @@ org.scala-lang scala-reflect - + org.eclipse.jetty.aggregate jetty-all-server @@ -94,27 +94,27 @@ ${spark.version} - + org.apache.spark spark-sql_2.10 ${spark.version} - + org.apache.spark spark-hive_2.10 ${spark.version} - + org.apache.spark spark-streaming_2.10 ${spark.version} - + org.apache.spark spark-streaming-twitter_2.10 ${spark.version} @@ -125,7 +125,7 @@ spark-catalyst_${scala.binary.version} ${spark.version} - + org.apache.maven @@ -139,12 +139,12 @@ org.sonatype.sisu sisu-inject-plexus - + org.apache.maven maven-model - - + + org.sonatype.aether @@ -161,7 +161,7 @@ aether-impl 1.12 - + org.apache.maven maven-aether-provider @@ -170,7 +170,7 @@ org.sonatype.aether aether-api - + org.sonatype.aether aether-spi @@ -178,24 +178,24 @@ org.sonatype.aether aether-util - + org.sonatype.aether aether-impl - + org.codehaus.plexus plexus-utils - - + + - + org.sonatype.aether aether-connector-file 1.12 - + org.sonatype.aether aether-connector-wagon @@ -205,9 +205,9 @@ org.apache.maven.wagon wagon-provider-api - + - + org.apache.maven.wagon wagon-provider-api @@ -216,10 +216,10 @@ org.codehaus.plexus plexus-utils - - + + - + org.apache.maven.wagon wagon-http-lightweight @@ -230,7 +230,7 @@ wagon-http-shared - + org.apache.maven.wagon @@ -259,7 +259,7 @@ test - + yarn @@ -272,7 +272,7 @@ - + @@ -286,11 +286,11 @@ maven-enforcer-plugin - 1.3.1 - - - enforce - none + 1.3.1 + + + enforce + none @@ -307,25 +307,42 @@ - maven-dependency-plugin - 2.8 + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + reference.conf + + + - copy-dependencies package - copy-dependencies + shade - - ${project.build.directory}/../../interpreter/spark - false - false - true - runtime - + + + + + maven-dependency-plugin + 2.8 + - copy-artifact package copy @@ -335,7 +352,6 @@ false false true - runtime ${project.groupId} @@ -343,11 +359,11 @@ ${project.version} ${project.packaging} - + - +