[ZEPPELIN-935] Adding more configurations to livy interpreter

docs/interpreter/livy.md

@@ -23,14 +23,19 @@ Additional requirements for the Livy interpreter are:
  * Livy server.
 
 ### Configuration
+We added some common configurations for spark, and you can set any configuration you want.
+This link contains all spark configurations: http://spark.apache.org/docs/latest/configuration.html#available-properties.
+And instead of starting property with `spark.` it should be replaced with `livy.spark.`.
+Example: `spark.master` to `livy.spark.master`
+
 <table class="table-configuration">
   <tr>
     <th>Property</th>
     <th>Default</th>
     <th>Description</th>
   </tr>
   <tr>
-      <td>zeppelin.livy.master</td>
+      <td>livy.spark.master</td>
       <td>local[*]</td>
       <td>Spark master uri. ex) spark://masterhost:7077</td>
     </tr>
@@ -44,6 +49,56 @@ Additional requirements for the Livy interpreter are:
     <td>1000</td>
     <td>Max number of SparkSQL result to display.</td>
   </tr>
+    <tr>
+    <td>livy.spark.driver.cores</td>
+    <td></td>
+    <td>Driver cores. ex) 1, 2.</td>
+  </tr>
+    <tr>
+    <td>livy.spark.driver.memory</td>
+    <td></td>
+    <td>Driver memory. ex) 512m, 32g.</td>
+  </tr>
+    <tr>
+    <td>livy.spark.executor.instances</td>
+    <td></td>
+    <td>Executor instances. ex) 1, 4.</td>
+  </tr>
+    <tr>
+    <td>livy.spark.executor.cores</td>
+    <td></td>
+    <td>Num cores per executor. ex) 1, 4.</td>
+  </tr>
+  <tr>
+    <td>livy.spark.executor.memory</td>
+    <td></td>
+    <td>Executor memory per worker instance. ex) 512m, 32g.</td>
+  </tr>
+  <tr>
+    <td>livy.spark.dynamicAllocation.enabled</td>
+    <td></td>
+    <td>Use dynamic resource allocation. ex) True, False.</td>
+  </tr>
+  <tr>
+    <td>livy.spark.dynamicAllocation.cachedExecutorIdleTimeout</td>
+    <td></td>
+    <td>Remove an executor which has cached data blocks.</td>
+  </tr>
+  <tr>
+    <td>livy.spark.dynamicAllocation.minExecutors</td>
+    <td></td>
+    <td>Lower bound for the number of executors.</td>
+  </tr>
+  <tr>
+    <td>livy.spark.dynamicAllocation.initialExecutors</td>
+    <td></td>
+    <td>Initial number of executors to run.</td>
+  </tr>
+  <tr>
+    <td>livy.spark.dynamicAllocation.maxExecutors</td>
+    <td></td>
+    <td>Upper bound for the number of executors.</td>
+  </tr>
 </table>
 
 
@@ -105,3 +160,6 @@ The session would have timed out, you may need to restart the interpreter.
 > Blacklisted configuration values in session config: spark.master
 
 edit `conf/spark-blacklist.conf` file in livy server and comment out `#spark.master` line.
+
+if you choose to work on livy in `apps/spark/java` directory in https://github.com/cloudera/hue ,
+copy `spark-user-configurable-options.template` to `spark-user-configurable-options.conf` file in livy server and comment out `#spark.master` 

livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java

@@ -20,6 +20,7 @@
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import com.google.gson.reflect.TypeToken;
+
 import org.apache.commons.lang3.StringUtils;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
@@ -40,7 +41,9 @@
 import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Properties;
 
 
@@ -60,27 +63,28 @@ public class LivyHelper {
 
   public Integer createSession(InterpreterContext context, String kind) throws Exception {
     try {
+      Map<String, String> conf = new HashMap<String, String>();
+
+      Iterator<Entry<Object, Object>> it = property.entrySet().iterator();
+      while (it.hasNext()) {
+        Entry<Object, Object> pair = it.next();
+        if (pair.getKey().toString().startsWith("livy.spark.") && 
+            !pair.getValue().toString().isEmpty())
+          conf.put(pair.getKey().toString().substring(5), pair.getValue().toString());
+      }
+
+      String confData = gson.toJson(conf);
+
       String json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions",
-          "POST",
+          "POST", 
           "{" +
               "\"kind\": \"" + kind + "\", " +
-              "\"master\": \"" + property.getProperty("zeppelin.livy.master") + "\", " +
-              "\"proxyUser\": \"" + context.getAuthenticationInfo().getUser() + "\"" +
+              "\"conf\": " + confData + ", " + 
+              "\"proxyUser\": " + context.getAuthenticationInfo().getUser() + 
               "}",
           context.getParagraphId()
       );
-      if (json.contains("CreateInteractiveRequest[\\\"master\\\"]")) {
-        json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions",
-            "POST",
-            "{" +
-                "\"kind\": \"" + kind + "\", " +
-                "\"conf\":{\"spark.master\": \""
-                + property.getProperty("zeppelin.livy.master") + "\"}," +
-                "\"proxyUser\": \"" + context.getAuthenticationInfo().getUser() + "\"" +
-                "}",
-            context.getParagraphId()
-        );
-      }
+
       Map jsonMap = (Map<Object, Object>) gson.fromJson(json,
           new TypeToken<Map<Object, Object>>() {
           }.getType());

livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java

@@ -45,7 +45,22 @@ public class LivySparkInterpreter extends Interpreter {
         LivySparkInterpreter.class.getName(),
         new InterpreterPropertyBuilder()
             .add("zeppelin.livy.url", DEFAULT_URL, "The URL for Livy Server.")
-            .add("zeppelin.livy.master", LOCAL, "Spark master uri. ex) spark://masterhost:7077")
+            .add("livy.spark.master", LOCAL, "Spark master uri. ex) spark://masterhost:7077")
+            .add("livy.spark.driver.cores", "", "Driver cores. ex) 1, 2")
+            .add("livy.spark.driver.memory", "", "Driver memory. ex) 512m, 32g")
+            .add("livy.spark.executor.instances", "", "Executor instances. ex) 1, 4")
+            .add("livy.spark.executor.cores", "", "Num cores per executor. ex) 1, 4")
+            .add("livy.spark.executor.memory", "", 
+                 "Executor memory per worker instance. ex) 512m, 32g")
+            .add("livy.spark.dynamicAllocation.enabled", "", "Use dynamic resource allocation")
+            .add("livy.spark.dynamicAllocation.cachedExecutorIdleTimeout", "", 
+                 "Remove an executor which has cached data blocks")
+            .add("livy.spark.dynamicAllocation.minExecutors", "", 
+                 "Lower bound for the number of executors if dynamic allocation is enabled. ")
+            .add("livy.spark.dynamicAllocation.initialExecutors", "", 
+                 "Initial number of executors to run if dynamic allocation is enabled. ")
+            .add("livy.spark.dynamicAllocation.maxExecutors", "", 
+                 "Upper bound for the number of executors if dynamic allocation is enabled. ")
             .build()
     );
   }