From 0a56299c55d7161d6d1c8385a50b6e4bf366d404 Mon Sep 17 00:00:00 2001 From: Nelson Costa Date: Sun, 24 Sep 2017 21:58:29 +0100 Subject: [PATCH 1/4] [ZEPPELIN-2949] Allow custom Spark UI URL --- .../java/org/apache/zeppelin/spark/SparkInterpreter.java | 7 ++++++- .../org/apache/zeppelin/spark/SparkInterpreterTest.java | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 18da0344f05..cc290a33947 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -206,7 +206,7 @@ public synchronized void onJobStart(SparkListenerJobStart jobStart) { private String getJobUrl(int jobId) { String jobUrl = null; if (sparkUrl != null) { - jobUrl = sparkUrl + "/jobs/job?id=" + jobId; + jobUrl = sparkUrl + "/jobs/job/?id=" + jobId; } return jobUrl; } @@ -936,6 +936,11 @@ public String getSparkUIUrl() { return sparkUrl; } + String sparkUrlProp = property.getProperty("zeppelin.spark.uiWebUrl", ""); + if (!sparkUrlProp.isEmpty()) { + return sparkUrlProp; + } + if (sparkVersion.newerThanEquals(SparkVersion.SPARK_2_0_0)) { Option uiWebUrlOption = (Option) Utils.invokeMethod(sc, "uiWebUrl"); if (uiWebUrlOption.isDefined()) { diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java index ece292b8290..ec682baa1a0 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java @@ -347,7 +347,7 @@ public void testParagraphUrls() { } String sparkUIUrl = repl.getSparkUIUrl(); assertNotNull(jobUrl); - assertTrue(jobUrl.startsWith(sparkUIUrl + "/jobs/job?id=")); + assertTrue(jobUrl.startsWith(sparkUIUrl + "/jobs/job/?id=")); } } From 826bad4a0d9f61d68bd27641dfbbf42ef2430148 Mon Sep 17 00:00:00 2001 From: Nelson Costa Date: Mon, 25 Sep 2017 12:51:38 +0100 Subject: [PATCH 2/4] [ZEPPELIN-2949] Updated documentation --- docs/interpreter/spark.md | 5 +++++ spark/src/main/resources/interpreter-setting.json | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/docs/interpreter/spark.md b/docs/interpreter/spark.md index be5b3e52863..6b73144c518 100644 --- a/docs/interpreter/spark.md +++ b/docs/interpreter/spark.md @@ -145,6 +145,11 @@ You can also set other Spark properties which are not listed in the table. For a true Do not change - developer only setting, not for production use + + zeppelin.spark.uiWebUrl + + Override Spark UI default URL + Without any configuration, Spark interpreter works out of box in local mode. But if you want to connect to your Spark cluster, you'll need to follow below two simple steps. diff --git a/spark/src/main/resources/interpreter-setting.json b/spark/src/main/resources/interpreter-setting.json index 7c13c49a16c..893142f265a 100644 --- a/spark/src/main/resources/interpreter-setting.json +++ b/spark/src/main/resources/interpreter-setting.json @@ -67,6 +67,13 @@ "defaultValue": true, "description": "Do not change - developer only setting, not for production use", "type": "checkbox" + }, + "zeppelin.spark.uiWebUrl": { + "envName": null, + "propertyName": "zeppelin.spark.uiWebUrl", + "defaultValue": "", + "description": "Override Spark UI default URL", + "type": "string" } }, "editor": { From 118339fc9c282011e838c63d479842d76e5c77b0 Mon Sep 17 00:00:00 2001 From: Nelson Costa Date: Wed, 11 Oct 2017 13:51:36 +0100 Subject: [PATCH 3/4] [ZEPPELIN-2949] Updated documentation --- docs/interpreter/spark.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/interpreter/spark.md b/docs/interpreter/spark.md index 6b73144c518..5bd0ee06ea2 100644 --- a/docs/interpreter/spark.md +++ b/docs/interpreter/spark.md @@ -148,7 +148,7 @@ You can also set other Spark properties which are not listed in the table. For a zeppelin.spark.uiWebUrl - Override Spark UI default URL + Overrides Spark UI default URL (ex: http://{hostName}/{uniquePath) @@ -188,7 +188,7 @@ For example, * **yarn-client** in Yarn client mode * **mesos://host:5050** in Mesos cluster -That's it. Zeppelin will work with any version of Spark and any deployment type without rebuilding Zeppelin in this way. +That's it. Zeppelin will work with any version of Spark and any deployment type without rebuilding Zeppelin in this way. For the further information about Spark & Zeppelin version compatibility, please refer to "Available Interpreters" section in [Zeppelin download page](https://zeppelin.apache.org/download.html). > Note that without exporting `SPARK_HOME`, it's running in local mode with included version of Spark. The included version may vary depending on the build profile. @@ -215,7 +215,7 @@ There are two ways to load external libraries in Spark interpreter. First is usi Please see [Dependency Management](../usage/interpreter/dependency_management.html) for the details. ### 2. Loading Spark Properties -Once `SPARK_HOME` is set in `conf/zeppelin-env.sh`, Zeppelin uses `spark-submit` as spark interpreter runner. `spark-submit` supports two ways to load configurations. +Once `SPARK_HOME` is set in `conf/zeppelin-env.sh`, Zeppelin uses `spark-submit` as spark interpreter runner. `spark-submit` supports two ways to load configurations. The first is command line options such as --master and Zeppelin can pass these options to `spark-submit` by exporting `SPARK_SUBMIT_OPTIONS` in `conf/zeppelin-env.sh`. Second is reading configuration options from `SPARK_HOME/conf/spark-defaults.conf`. Spark properties that user can set to distribute libraries are: @@ -248,7 +248,7 @@ Here are few examples: ```bash export SPARK_SUBMIT_OPTIONS="--packages com.databricks:spark-csv_2.10:1.2.0 --jars /path/mylib1.jar,/path/mylib2.jar --files /path/mylib1.py,/path/mylib2.zip,/path/mylib3.egg" ``` - + * `SPARK_HOME/conf/spark-defaults.conf` ``` @@ -413,17 +413,17 @@ To learn more about dynamic form, checkout [Dynamic Form](../usage/dynamic_form/ ## Matplotlib Integration (pyspark) -Both the `python` and `pyspark` interpreters have built-in support for inline visualization using `matplotlib`, -a popular plotting library for python. More details can be found in the [python interpreter documentation](../interpreter/python.html), -since matplotlib support is identical. More advanced interactive plotting can be done with pyspark through +Both the `python` and `pyspark` interpreters have built-in support for inline visualization using `matplotlib`, +a popular plotting library for python. More details can be found in the [python interpreter documentation](../interpreter/python.html), +since matplotlib support is identical. More advanced interactive plotting can be done with pyspark through utilizing Zeppelin's built-in [Angular Display System](../usage/display_system/angular_backend.html), as shown below: ## Interpreter setting option -You can choose one of `shared`, `scoped` and `isolated` options wheh you configure Spark interpreter. -Spark interpreter creates separated Scala compiler per each notebook but share a single SparkContext in `scoped` mode (experimental). +You can choose one of `shared`, `scoped` and `isolated` options wheh you configure Spark interpreter. +Spark interpreter creates separated Scala compiler per each notebook but share a single SparkContext in `scoped` mode (experimental). It creates separated SparkContext per each notebook in `isolated` mode. ## IPython support From 2ae8f2d064988e0e243cf6952101c5344ade0d03 Mon Sep 17 00:00:00 2001 From: Nelson Costa Date: Thu, 19 Oct 2017 10:21:54 +0100 Subject: [PATCH 4/4] [ZEPPELIN-2949] Last amendments --- docs/interpreter/spark.md | 2 +- .../java/org/apache/zeppelin/spark/SparkInterpreter.java | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/interpreter/spark.md b/docs/interpreter/spark.md index 5bd0ee06ea2..b641ef6a569 100644 --- a/docs/interpreter/spark.md +++ b/docs/interpreter/spark.md @@ -148,7 +148,7 @@ You can also set other Spark properties which are not listed in the table. For a - +
zeppelin.spark.uiWebUrl Overrides Spark UI default URL (ex: http://{hostName}/{uniquePath)Overrides Spark UI default URL. Value should be a full URL (ex: http://{hostName}/{uniquePath}
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index cc290a33947..bb38841909a 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -26,11 +26,9 @@ import java.lang.reflect.Method; import java.net.URL; import java.net.URLClassLoader; -import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.NoSuchElementException; import java.util.Properties; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; @@ -51,7 +49,6 @@ import org.apache.spark.ui.SparkUI; import org.apache.spark.ui.jobs.JobProgressListener; import org.apache.zeppelin.interpreter.BaseZeppelinContext; -import org.apache.zeppelin.interpreter.DefaultInterpreterProperty; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; @@ -72,7 +69,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Joiner; import scala.Console; import scala.Enumeration.Value; import scala.None; @@ -937,7 +933,7 @@ public String getSparkUIUrl() { } String sparkUrlProp = property.getProperty("zeppelin.spark.uiWebUrl", ""); - if (!sparkUrlProp.isEmpty()) { + if (!StringUtils.isBlank(sparkUrlProp)) { return sparkUrlProp; }