ZEPPELIN-804 Refactoring registration mechanism on Interpreters

docs/development/writingzeppelininterpreter.md

@@ -36,14 +36,48 @@ In 'Separate Interpreter for each note' mode, new Interpreter instance will be c
 ### Make your own Interpreter
 
 Creating a new interpreter is quite simple. Just extend [org.apache.zeppelin.interpreter](https://github.com/apache/incubator-zeppelin/blob/master/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java) abstract class and implement some methods.
-You can include `org.apache.zeppelin:zeppelin-interpreter:[VERSION]` artifact in your build system.
-Your interpreter name is derived from the static register method.
-
+You can include `org.apache.zeppelin:zeppelin-interpreter:[VERSION]` artifact in your build system. And you should your jars under your interpreter directory with specific directory name. Zeppelin server reads interpreter directories recursively and initializes interpreters including your own interpreter.
+
+There are three locations where you can store your interpreter group, name and other information. Zeppelin server tries to find the location below. Next, Zeppelin tries to find `interpareter-setting.json` in your interpreter jar. 
+```
+{ZEPPELIN_INTERPRETER_DIR}/{YOUR_OWN_INTERPRETER_DIR}/interpreter-setting.json
+```
+
+Here is an example of `interpareter-setting.json` on your own interpreter.
+```json
+[
+  {
+    "interpreterGroup": "your-group",
+    "interpreterName": "your-name",
+    "interpreterClassName": "your.own.interpreter.class",
+    "properties": {
+      "propertiies1": {
+        "envName": null,
+        "propertyName": "property.1.name",
+        "defaultValue": "propertyDefaultValue",
+        "description": "Property description"
+      },
+      "properties2": {
+        "envName": PROPERTIES_2,
+        "propertyName": null,
+        "defaultValue": "property2DefaultValue",
+        "description": "Property 2 description"
+      }, ...
+    }
+  },
+  {
+    ...
+  } 
+]
+```
+
+Finally, Zeppelin uses static initialization with the following:
 ```
 static {
     Interpreter.register("MyInterpreterName", MyClassName.class.getName());
   }
 ```
+**Static initialization is deprecated and will be supported until 0.6.0.**
 
 The name will appear later in the interpreter name option box during the interpreter configuration process.
 The name of the interpreter is what you later write to identify a paragraph which should be interpreted using this interpreter.

spark/pom.xml

@@ -313,6 +313,7 @@
             <exclude>**/metastore_db/</exclude>
             <exclude>**/README.md</exclude>
             <exclude>**/dependency-reduced-pom.xml</exclude>
+            <exclude>**/interpreter-setting.json</exclude>
           </excludes>
         </configuration>
       </plugin>

spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java

@@ -62,23 +62,6 @@
  *
  */
 public class DepInterpreter extends Interpreter {
-
-  static {
-    Interpreter.register(
-        "dep",
-        "spark",
-        DepInterpreter.class.getName(),
-        new InterpreterPropertyBuilder()
-            .add("zeppelin.dep.localrepo",
-                getSystemDefault("ZEPPELIN_DEP_LOCALREPO", null, "local-repo"),
-                "local repository for dependency loader")
-            .add("zeppelin.dep.additionalRemoteRepository",
-                "spark-packages,http://dl.bintray.com/spark-packages/maven,false;",
-                "A list of 'id,remote-repository-URL,is-snapshot;' for each remote repository.")
-            .build());
-
-  }
-
   private SparkIMain intp;
   private ByteArrayOutputStream out;
   private SparkDependencyContext depc;

spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java

@@ -77,17 +77,6 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
   private String scriptPath;
   boolean pythonscriptRunning = false;
 
-  static {
-    Interpreter.register(
-        "pyspark",
-        "spark",
-        PySparkInterpreter.class.getName(),
-        new InterpreterPropertyBuilder()
-          .add("zeppelin.pyspark.python",
-               SparkInterpreter.getSystemDefault("PYSPARK_PYTHON", null, "python"),
-               "Python command to run pyspark with").build());
-  }
-
   public PySparkInterpreter(Properties property) {
     super(property);
 

spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java

@@ -80,38 +80,6 @@
 public class SparkInterpreter extends Interpreter {
   public static Logger logger = LoggerFactory.getLogger(SparkInterpreter.class);
 
-  static {
-    Interpreter.register(
-      "spark",
-      "spark",
-      SparkInterpreter.class.getName(),
-      new InterpreterPropertyBuilder()
-        .add("spark.app.name",
-          getSystemDefault("SPARK_APP_NAME", "spark.app.name", "Zeppelin"),
-          "The name of spark application.")
-        .add("master",
-          getSystemDefault("MASTER", "spark.master", "local[*]"),
-          "Spark master uri. ex) spark://masterhost:7077")
-        .add("spark.executor.memory",
-          getSystemDefault(null, "spark.executor.memory", ""),
-          "Executor memory per worker instance. ex) 512m, 32g")
-        .add("spark.cores.max",
-          getSystemDefault(null, "spark.cores.max", ""),
-          "Total number of cores to use. Empty value uses all available core.")
-        .add("zeppelin.spark.useHiveContext",
-          getSystemDefault("ZEPPELIN_SPARK_USEHIVECONTEXT",
-            "zeppelin.spark.useHiveContext", "true"),
-          "Use HiveContext instead of SQLContext if it is true.")
-        .add("zeppelin.spark.maxResult",
-          getSystemDefault("ZEPPELIN_SPARK_MAXRESULT", "zeppelin.spark.maxResult", "1000"),
-          "Max number of SparkSQL result to display.")
-        .add("args", "", "spark commandline args")
-        .add("zeppelin.spark.printREPLOutput", "true",
-          "Print REPL output")
-        .build()
-    );
-  }
-
   private ZeppelinContext z;
   private SparkILoop interpreter;
   private SparkIMain intp;

spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java

@@ -42,32 +42,6 @@ public class SparkRInterpreter extends Interpreter {
   private static String renderOptions;
   private ZeppelinR zeppelinR;
 
-  static {
-    Interpreter.register(
-      "r",
-      "spark",
-      SparkRInterpreter.class.getName(),
-      new InterpreterPropertyBuilder()
-          .add("zeppelin.R.cmd",
-              SparkInterpreter.getSystemDefault("ZEPPELIN_R_CMD", "zeppelin.R.cmd", "R"),
-              "R repl path")
-          .add("zeppelin.R.knitr",
-              SparkInterpreter.getSystemDefault("ZEPPELIN_R_KNITR", "zeppelin.R.knitr", "true"),
-              "whether use knitr or not")
-          .add("zeppelin.R.image.width",
-              SparkInterpreter.getSystemDefault("ZEPPELIN_R_IMAGE_WIDTH",
-                  "zeppelin.R.image.width", "100%"),
-              "")
-          .add("zeppelin.R.render.options",
-              SparkInterpreter.getSystemDefault("ZEPPELIN_R_RENDER_OPTIONS",
-                  "zeppelin.R.render.options",
-                  "out.format = 'html', comment = NA, "
-                      + "echo = FALSE, results = 'asis', message = F, warning = F"),
-              "")
-          .build());
-  }
-
-
   public SparkRInterpreter(Properties property) {
     super(property);
   }

spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java

@@ -46,27 +46,6 @@ public class SparkSqlInterpreter extends Interpreter {
   Logger logger = LoggerFactory.getLogger(SparkSqlInterpreter.class);
   AtomicInteger num = new AtomicInteger(0);
 
-  static {
-    Interpreter.register(
-        "sql",
-        "spark",
-        SparkSqlInterpreter.class.getName(),
-        new InterpreterPropertyBuilder()
-            .add("zeppelin.spark.maxResult",
-                SparkInterpreter.getSystemDefault("ZEPPELIN_SPARK_MAXRESULT",
-                    "zeppelin.spark.maxResult", "1000"),
-                "Max number of SparkSQL result to display.")
-            .add("zeppelin.spark.concurrentSQL",
-                SparkInterpreter.getSystemDefault("ZEPPELIN_SPARK_CONCURRENTSQL",
-                    "zeppelin.spark.concurrentSQL", "false"),
-                "Execute multiple SQL concurrently if set true.")
-            .add("zeppelin.spark.sql.stacktrace",
-                SparkInterpreter.getSystemDefault("ZEPPELIN_SPARK_SQL_STACKTRACE",
-                    "zeppelin.spark.sql.stacktrace", "false"),
-                "Show full exception stacktrace for SQL queries if set to true.")
-            .build());
-  }
-
   private String getJobGroup(InterpreterContext context){
     return "zeppelin-" + context.getParagraphId();
   }

spark/src/main/resources/interpreter-setting.json

@@ -0,0 +1,146 @@
+[
+  {
+    "interpreterGroup": "spark",
+    "interpreterName": "spark",
+    "interpreterClassName": "org.apache.zeppelin.spark.SparkInterpreter",
+    "properties": {
+      "spark.executor.memory": {
+        "envName": null,
+        "propertyName": "spark.executor.memory",
+        "defaultValue": "",
+        "description": "Executor memory per worker instance. ex) 512m, 32g"
+      },
+      "args": {
+        "envName": null,
+        "propertyName": null,
+        "defaultValue": "",
+        "description": "spark commandline args"
+      },
+      "zeppelin.spark.useHiveContext": {
+        "envName": "ZEPPELIN_SPARK_USEHIVECONTEXT",
+        "propertyName": "zeppelin.spark.useHiveContext",
+        "defaultValue": "true",
+        "description": "Use HiveContext instead of SQLContext if it is true."
+      },
+      "spark.app.name": {
+        "envName": "SPARK_APP_NAME",
+
+        "propertyName": "spark.app.name",
+        "defaultValue": "Zeppelin",
+        "description": "The name of spark application."
+      },
+      "zeppelin.spark.printREPLOutput": {
+        "envName": null,
+        "propertyName": null,
+        "defaultValue": "true",
+        "description": "Print REPL output"
+      },
+      "spark.cores.max": {
+        "envName": null,
+        "propertyName": "spark.cores.max",
+        "defaultValue": "",
+        "description": "Total number of cores to use. Empty value uses all available core."
+      },
+      "zeppelin.spark.maxResult": {
+        "envName": "ZEPPELIN_SPARK_MAXRESULT",
+        "propertyName": "zeppelin.spark.maxResult",
+        "defaultValue": "1000",
+        "description": "Max number of SparkSQL result to display."
+      },
+      "master": {
+        "envName": "MASTER",
+        "propertyName": "spark.master",
+        "defaultValue": "local[*]",
+        "description": "Spark master uri. ex) spark://masterhost:7077"
+      }
+    }
+  },
+  {
+    "interpreterGroup": "spark",
+    "interpreterName": "sql",
+    "interpreterClassName": "org.apache.zeppelin.spark.SparkSqlInterpreter",
+    "properties": {
+      "zeppelin.spark.concurrentSQL": {
+        "envName": "ZEPPELIN_SPARK_CONCURRENTSQL",
+        "propertyName": "zeppelin.spark.concurrentSQL",
+        "defaultValue": "false",
+        "description": "Execute multiple SQL concurrently if set true."
+      },
+      "zeppelin.spark.sql.stacktrace": {
+        "envName": "ZEPPELIN_SPARK_SQL_STACKTRACE",
+        "propertyName": "zeppelin.spark.sql.stacktrace",
+        "defaultValue": "false",
+        "description": "Show full exception stacktrace for SQL queries if set to true."
+      },
+      "zeppelin.spark.maxResult": {
+        "envName": "ZEPPELIN_SPARK_MAXRESULT",
+        "propertyName": "zeppelin.spark.maxResult",
+        "defaultValue": "1000",
+        "description": "Max number of SparkSQL result to display."
+      }
+    }
+  },
+  {
+    "interpreterGroup": "spark",
+    "interpreterName": "dep",
+    "interpreterClassName": "org.apache.zeppelin.spark.DepInterpreter",
+    "properties": {
+      "zeppelin.dep.localrepo": {
+        "envName": "ZEPPELIN_DEP_LOCALREPO",
+        "propertyName": null,
+        "defaultValue": "local-repo",
+        "description": "local repository for dependency loader"
+      },
+      "zeppelin.dep.additionalRemoteRepository": {
+        "envName": null,
+        "propertyName": null,
+        "defaultValue": "spark-packages,http://dl.bintray.com/spark-packages/maven,false;",
+        "description": "A list of 'id,remote-repository-URL,is-snapshot;' for each remote repository."
+      }
+    }
+  },
+  {
+    "interpreterGroup": "spark",
+    "interpreterName": "pyspark",
+    "interpreterClassName": "org.apache.zeppelin.spark.PySparkInterpreter",
+    "properties": {
+      "zeppelin.pyspark.python": {
+        "envName": "PYSPARK_PYTHON",
+        "propertyName": null,
+        "defaultValue": "python",
+        "description": "Python command to run pyspark with"
+      }
+    }
+  },
+  {
+    "interpreterGroup": "spark",
+    "interpreterName": "r",
+    "interpreterClassName": "org.apache.zeppelin.spark.SparkRInterpreter",
+    "properties": {
+      "zeppelin.R.knitr": {
+        "envName": "ZEPPELIN_R_KNITR",
+        "propertyName": "zeppelin.R.knitr",
+        "defaultValue": "true",
+        "description": "whether use knitr or not"
+      },
+      "zeppelin.R.cmd": {
+        "envName": "ZEPPELIN_R_CMD",
+        "propertyName": "zeppelin.R.cmd",
+        "defaultValue": "R",
+        "description": "R repl path"
+      },
+      "zeppelin.R.image.width": {
+        "envName": "ZEPPELIN_R_IMAGE_WIDTH",
+        "propertyName": "zeppelin.R.image.width",
+        "defaultValue": "100%",
+        "description": ""
+      },
+      "zeppelin.R.render.options": {
+        "envName": "ZEPPELIN_R_RENDER_OPTIONS",
+        "propertyName": "zeppelin.R.render.options",
+        "defaultValue": "out.format = 'html', comment = NA, echo = FALSE, results = 'asis', message = F, warning = F",
+        "description": ""
+      }
+    }
+  }
+]

spark/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java

@@ -39,14 +39,21 @@ public class DepInterpreterTest {
   private File tmpDir;
   private SparkInterpreter repl;
 
+  private Properties getTestProperties() {
+    Properties p = new Properties();
+    p.setProperty("zeppelin.dep.localrepo", "local-repo");
+    p.setProperty("zeppelin.dep.additionalRemoteRepository", "spark-packages,http://dl.bintray.com/spark-packages/maven,false;");
+    return p;
+  }
+
   @Before
   public void setUp() throws Exception {
     tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis());
     System.setProperty("zeppelin.dep.localrepo", tmpDir.getAbsolutePath() + "/local-repo");
 
     tmpDir.mkdirs();
 
-    Properties p = new Properties();
+    Properties p = getTestProperties();
 
     dep = new DepInterpreter(p);
     dep.open();