diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template
index 8ef4edce9dc..6d33433f965 100755
--- a/conf/zeppelin-site.xml.template
+++ b/conf/zeppelin-site.xml.template
@@ -718,4 +718,10 @@
Disable it can save lots of memory
+
+ zeppelin.spark.only_yarn_cluster
+ false
+ Whether only allow yarn cluster mode
+
+
diff --git a/docs/interpreter/spark.md b/docs/interpreter/spark.md
index 277efb3144e..3c07e0156a5 100644
--- a/docs/interpreter/spark.md
+++ b/docs/interpreter/spark.md
@@ -259,6 +259,8 @@ For the further information about Spark & Zeppelin version compatibility, please
> Note that without exporting `SPARK_HOME`, it's running in local mode with included version of Spark. The included version may vary depending on the build profile.
+> Yarn client mode and local mode will run driver in the same machine with zeppelin server, this would be dangerous for production. Because it may run out of memory when there's many spark interpreters running at the same time. So we suggest you only allow yarn-cluster mode via setting `zeppelin.spark.only_yarn_cluster` in `zeppelin-site.xml`.
+
## SparkContext, SQLContext, SparkSession, ZeppelinContext
SparkContext, SQLContext, SparkSession (for spark 2.x) and ZeppelinContext are automatically created and exposed as variable names `sc`, `sqlContext`, `spark` and `z`, respectively, in Scala, Kotlin, Python and R environments.
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index dfcf840bfcd..cbd894eaa3e 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -739,6 +739,10 @@ public String getZeppelinSearchIndexPath() {
return getRelativeDir(ConfVars.ZEPPELIN_SEARCH_INDEX_PATH);
}
+ public Boolean isOnlyYarnCluster() {
+ return getBoolean(ConfVars.ZEPPELIN_SPARK_ONLY_YARN_CLUSTER);
+ }
+
public String getClusterAddress() {
return getString(ConfVars.ZEPPELIN_CLUSTER_ADDR);
}
@@ -998,7 +1002,8 @@ public enum ConfVars {
ZEPPELIN_SEARCH_INDEX_REBUILD("zeppelin.search.index.rebuild", false),
ZEPPELIN_SEARCH_USE_DISK("zeppelin.search.use.disk", true),
ZEPPELIN_SEARCH_INDEX_PATH("zeppelin.search.index.path", "/tmp/zeppelin-index"),
- ZEPPELIN_JOBMANAGER_ENABLE("zeppelin.jobmanager.enable", true);
+ ZEPPELIN_JOBMANAGER_ENABLE("zeppelin.jobmanager.enable", false),
+ ZEPPELIN_SPARK_ONLY_YARN_CLUSTER("zeppelin.spark.only_yarn_cluster", false);
private String varName;
@SuppressWarnings("rawtypes")
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
index 5252eeb9ea7..9255c989ce4 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
@@ -76,6 +76,10 @@ public Map buildEnvFromProperties(InterpreterLaunchContext conte
if (isYarnMode() && getDeployMode().equals("cluster")) {
env.put("ZEPPELIN_SPARK_YARN_CLUSTER", "true");
sparkProperties.setProperty("spark.yarn.submit.waitAppCompletion", "false");
+ } else if (zConf.isOnlyYarnCluster()){
+ throw new IOException("Only yarn-cluster mode is allowed, please set " +
+ ZeppelinConfiguration.ConfVars.ZEPPELIN_SPARK_ONLY_YARN_CLUSTER.getVarName() +
+ " to false if you want to use other modes.");
}
StringBuilder sparkConfBuilder = new StringBuilder();