[ZEPPELIN-732] Helium Application

.travis.yml

@@ -35,7 +35,7 @@ matrix:
   include:
     # Test all modules
     - jdk: "oraclejdk7"
-      env: SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
+      env: SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
 
     # Test spark module for 1.5.2
     - jdk: "oraclejdk7"
@@ -59,7 +59,7 @@ matrix:
 
     # Test selenium with spark module for 1.6.1
     - jdk: "oraclejdk7"
-      env: TEST_SELENIUM="true" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
+      env: TEST_SELENIUM="true" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pexamples" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
 
 before_install:
   - "ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin"

README.md

@@ -201,6 +201,13 @@ Available profiles are
 -Pmapr51
 ```
 
+#### -Pexamples (optional)
+
+Bulid examples under zeppelin-examples directory
+
+
+
+
 
 Here're some examples:
 
@@ -231,6 +238,7 @@ mvn clean package -Dignite.version=1.6.0 -DskipTests
 mvn clean package -Pscalding -DskipTests
 ```
 
+
 ### Configure
 If you wish to configure Zeppelin option (like port number), configure the following files:
 

docs/_includes/themes/zeppelin/_navigation.html

@@ -104,6 +104,7 @@
                 <li role="separator" class="divider"></li>
                 <li class="title"><span><b>Contibute</b><span></li>
                 <li><a href="{{BASE_PATH}}/development/writingzeppelininterpreter.html">Writing Zeppelin Interpreter</a></li>
+                <li><a href="{{BASE_PATH}}/development/writingzeppelinapplication.html">Writing Zeppelin Application</a></li>                
                 <li><a href="{{BASE_PATH}}/development/howtocontribute.html">How to contribute (code)</a></li>
                 <li><a href="{{BASE_PATH}}/development/howtocontributewebsite.html">How to contribute (website)</a></li>
               </ul>

docs/development/writingzeppelinapplication.md

@@ -0,0 +1,185 @@
+---
+layout: page
+title: "Writing Zeppelin Application"
+description: ""
+group: development
+---
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+{% include JB/setup %}
+
+# What is Zeppelin Application (Experimental)
+
+Apache Zeppelin Application is a package that runs on Interpreter process and displays it's output inside of the notebook. While application runs on Interpreter process, it's able to access resources provided by Interpreter through ResourcePool. Output is always rendered by AngularDisplaySystem. Therefore application provides all the possiblities of making interactive graphical application that uses data and processing power of any Interpreter.
+
+
+
+## Writing your own Application
+
+Writing Application means extending `org.apache.zeppelin.helium.Application`. You can use your favorite IDE and language while Java class files are packaged into jar. `Application` class looks like
+
+```java
+
+/**
+ * Constructor. Invoked when application is loaded
+ */
+public Application(ApplicationContext context);
+
+/**
+ * Invoked when there're (possible) updates in required resource set.
+ * i.e. invoked after application load and after paragraph finishes.
+ */
+public abstract void run(ResourceSet args);
+
+/**
+ * Invoked before application unload.
+ * Application is automatically unloaded with paragraph/notebook removal
+ */
+public abstract void unload();
+```
+
+
+You can check example applications under [./zeppelin-examples](https://github.com/apache/incubator-zeppelin/tree/master/zeppelin-examples) directory.
+
+
+## Development mode
+
+In the development mode, you can run your Application in your IDE as a normal java application and see the result inside of Zeppelin notebook.
+
+org.apache.zeppelin.interpreter.dev.ZeppelinApplicationDevServer can run Zeppelin Application in development mode.
+
+```java
+
+// entry point for development mode
+public static void main(String[] args) throws Exception {
+
+  // add resources for development mode
+  LocalResourcePool pool = new LocalResourcePool("dev");
+  pool.put("date", new Date());
+
+  // run application in devlopment mode with give resource
+  // in this case, Clock.class.getName() will be the application class name  
+  ZeppelinApplicationDevServer devServer = new ZeppelinApplicationDevServer(
+      Clock.class.getName(),
+      pool.getAll());
+
+  // start development mode
+  devServer.start();
+  devServer.join();
+}
+```
+
+
+In the Zeppelin notebook, run `%dev run` will connect to application running in development mode.
+
+
+
+
+## Package file
+
+Package file is a json file that provides information about the application.
+Json file contains following informations
+
+```
+{
+  name : "[organization].[name]",
+  description : "Description",
+  artifact : "groupId:artifactId:version",
+  className : "your.package.name.YourApplicationClass",
+  resources : [
+    ["resource.name", ":resource.class.name"],
+    ["alternative.resource.name", ":alternative.class.name"]
+  ],
+  icon : "<i class="icon"></i>"
+}
+
+```
+
+#### name
+
+Name is a string in '[group].[name]' format.
+[group] and [name] allows only [A-Za-z0-9_].
+Group is normally organization name who creates this application.
+
+#### description
+
+Short description. about application
+
+#### artifact
+
+Location of the jar artifact.
+"groupId:artifactId:version" will make load artifact from maven repository.
+If jar is in local filesystem, absolute/relative can be used.
+
+e.g.
+
+When artifact exists in Maven repository
+
+`artifact: "org.apache.zeppelin:zeppelin-examples:0.6.0"`
+
+When artifact exists in local filesystem
+
+`artifact: "zeppelin-example/target/zeppelin-example-0.6.0.jar"`
+
+
+#### className
+
+Entry point. Class that extends `org.apache.zeppelin.helium.Application`
+
+
+#### resources
+
+Two dimensional array that defines required resources by name or by className. Helium Application launcher will compare resources in the ResourcePool with informations in this field and suggest application only when all required resources are available in the ResourcePool.
+
+Resouce name is a string which will be compared with name of objects in the ResourcePool. className is a string with ":" prepended, which will be compared with className of the objects in the ResourcePool.
+
+Application may require two or more resources. Required resource can be listed inside of json array. For example, if application requires object "name1", "name2" and "className1" type of object to run, resources field can be
+
+```
+resources: [
+  [ "name1", "name2", ":className1", ...]
+]
+```
+
+If Application can handle alternative combination of required resource, alternative set can be listed as below.
+
+```
+resources: [
+  [ "name", ":className"],
+  [ "altName", ":altClassName1"],
+  ...
+]
+```
+
+Easier way of understanding this scheme is
+
+```
+resources: [
+   [ 'resource' AND 'resource' AND ... ] OR
+   [ 'resource' AND 'resource' AND ... ] OR
+   ...
+]
+```
+
+
+#### icon
+
+Icon to be used on the application button. String in this field will be rendered as a html.
+
+e.g.
+
+```
+icon: "<i class='fa fa-clock-o'></i>"
+```
+

pom.xml

@@ -667,6 +667,13 @@
       </modules>
     </profile>
 
+    <profile>
+      <id>examples</id>
+      <modules>
+        <module>zeppelin-examples</module>
+      </modules>
+    </profile>
+
     <profile>
       <id>build-distr</id>
       <activation>

spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java

@@ -50,6 +50,8 @@
 import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.apache.zeppelin.interpreter.InterpreterUtils;
 import org.apache.zeppelin.interpreter.WrappedInterpreter;
+import org.apache.zeppelin.resource.ResourcePool;
+import org.apache.zeppelin.resource.WellKnownResourceName;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
 import org.apache.zeppelin.scheduler.Scheduler;
 import org.apache.zeppelin.scheduler.SchedulerFactory;
@@ -533,7 +535,9 @@ public void open() {
       binder.put("sc", sc);
       binder.put("sqlc", sqlc);
       binder.put("z", z);
-
+      binder.put("intp", intp);
+      intp.interpret("@transient val intp = _binder.get(\"intp\").asInstanceOf[org.apache.spark" +
+          ".repl.SparkIMain]");
       intp.interpret("@transient val z = "
               + "_binder.get(\"z\").asInstanceOf[org.apache.zeppelin.spark.ZeppelinContext]");
       intp.interpret("@transient val sc = "
@@ -759,13 +763,10 @@ public InterpreterResult interpret(String[] lines, InterpreterContext context) {
   public InterpreterResult interpretInput(String[] lines, InterpreterContext context) {
     SparkEnv.set(env);
 
-    // add print("") to make sure not finishing with comment
-    // see https://github.com/NFLabs/zeppelin/issues/151
-    String[] linesToRun = new String[lines.length + 1];
+    String[] linesToRun = new String[lines.length];
     for (int i = 0; i < lines.length; i++) {
       linesToRun[i] = lines[i];
     }
-    linesToRun[lines.length] = "print(\"\")";
 
     Console.setOut(context.out);
     out.setInterpreterOutput(context.out);
@@ -828,17 +829,39 @@ public InterpreterResult interpretInput(String[] lines, InterpreterContext conte
       }
     }
 
+    // make sure code does not finish with comment
+    if (r == Code.INCOMPLETE) {
+      scala.tools.nsc.interpreter.Results.Result res;
+      res = intp.interpret(incomplete + "\nprint(\"\")");
+      r = getResultCode(res);
+    }
+
     if (r == Code.INCOMPLETE) {
       sc.clearJobGroup();
       out.setInterpreterOutput(null);
       return new InterpreterResult(r, "Incomplete expression");
     } else {
       sc.clearJobGroup();
+      putLatestVarInResourcePool(context);
       out.setInterpreterOutput(null);
       return new InterpreterResult(Code.SUCCESS);
     }
   }
 
+  private void putLatestVarInResourcePool(InterpreterContext context) {
+    String varName = intp.mostRecentVar();
+    if (varName == null || varName.isEmpty()) {
+      return;
+    }
+
+    Object lastObj = getValue(varName);
+    if (lastObj != null) {
+      ResourcePool resourcePool = context.getResourcePool();
+      resourcePool.put(context.getNoteId(), context.getParagraphId(),
+          WellKnownResourceName.ZeppelinReplResult.toString(), lastObj);
+    }
+  };
+
 
   @Override
   public void cancel(InterpreterContext context) {

spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java

@@ -29,6 +29,7 @@
 import org.apache.spark.SparkConf;
 import org.apache.spark.SparkContext;
 import org.apache.zeppelin.display.AngularObjectRegistry;
+import org.apache.zeppelin.resource.LocalResourcePool;
 import org.apache.zeppelin.user.AuthenticationInfo;
 import org.apache.zeppelin.display.GUI;
 import org.apache.zeppelin.interpreter.*;
@@ -95,7 +96,7 @@ public void setUp() throws Exception {
         new HashMap<String, Object>(),
         new GUI(),
         new AngularObjectRegistry(intpGroup.getId(), null),
-        null,
+        new LocalResourcePool("id"),
         new LinkedList<InterpreterContextRunner>(),
         new InterpreterOutput(new InterpreterOutputListener() {
           @Override

spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java

@@ -24,6 +24,7 @@
 import java.util.Properties;
 
 import org.apache.zeppelin.display.AngularObjectRegistry;
+import org.apache.zeppelin.resource.LocalResourcePool;
 import org.apache.zeppelin.user.AuthenticationInfo;
 import org.apache.zeppelin.display.GUI;
 import org.apache.zeppelin.interpreter.*;
@@ -77,7 +78,7 @@ public void setUp() throws Exception {
     context = new InterpreterContext("note", "id", "title", "text", new AuthenticationInfo(),
         new HashMap<String, Object>(), new GUI(),
         new AngularObjectRegistry(intpGroup.getId(), null),
-        null,
+        new LocalResourcePool("id"),
         new LinkedList<InterpreterContextRunner>(), new InterpreterOutput(new InterpreterOutputListener() {
       @Override
       public void onAppend(InterpreterOutput out, byte[] line) {