diff --git a/.travis.yml b/.travis.yml index 7fa8e156ab0..19a05cb1d55 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,7 +35,7 @@ matrix: include: # Test all modules - jdk: "oraclejdk7" - env: SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS="" + env: SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS="" # Test spark module for 1.5.2 - jdk: "oraclejdk7" @@ -59,7 +59,7 @@ matrix: # Test selenium with spark module for 1.6.1 - jdk: "oraclejdk7" - env: TEST_SELENIUM="true" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false" + env: TEST_SELENIUM="true" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pexamples" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false" before_install: - "ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin" diff --git a/README.md b/README.md index 990cc9aa4fc..eee427cee3a 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,13 @@ Available profiles are -Pmapr51 ``` +#### -Pexamples (optional) + +Bulid examples under zeppelin-examples directory + + + + Here're some examples: @@ -231,6 +238,7 @@ mvn clean package -Dignite.version=1.6.0 -DskipTests mvn clean package -Pscalding -DskipTests ``` + ### Configure If you wish to configure Zeppelin option (like port number), configure the following files: diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html index fce8cee4631..8a99c840cf6 100644 --- a/docs/_includes/themes/zeppelin/_navigation.html +++ b/docs/_includes/themes/zeppelin/_navigation.html @@ -104,6 +104,7 @@
  • Contibute
  • Writing Zeppelin Interpreter
    • +
  • Writing Zeppelin Application
  • How to contribute (code)
  • How to contribute (website)
    • diff --git a/docs/development/writingzeppelinapplication.md b/docs/development/writingzeppelinapplication.md new file mode 100644 index 00000000000..1256d1be4a0 --- /dev/null +++ b/docs/development/writingzeppelinapplication.md @@ -0,0 +1,185 @@ +--- +layout: page +title: "Writing Zeppelin Application" +description: "" +group: development +--- + +{% include JB/setup %} + +# What is Zeppelin Application (Experimental) + +Apache Zeppelin Application is a package that runs on Interpreter process and displays it's output inside of the notebook. While application runs on Interpreter process, it's able to access resources provided by Interpreter through ResourcePool. Output is always rendered by AngularDisplaySystem. Therefore application provides all the possiblities of making interactive graphical application that uses data and processing power of any Interpreter. + + + +## Writing your own Application + +Writing Application means extending `org.apache.zeppelin.helium.Application`. You can use your favorite IDE and language while Java class files are packaged into jar. `Application` class looks like + +```java + +/** + * Constructor. Invoked when application is loaded + */ +public Application(ApplicationContext context); + +/** + * Invoked when there're (possible) updates in required resource set. + * i.e. invoked after application load and after paragraph finishes. + */ +public abstract void run(ResourceSet args); + +/** + * Invoked before application unload. + * Application is automatically unloaded with paragraph/notebook removal + */ +public abstract void unload(); +``` + + +You can check example applications under [./zeppelin-examples](https://github.com/apache/incubator-zeppelin/tree/master/zeppelin-examples) directory. + + +## Development mode + +In the development mode, you can run your Application in your IDE as a normal java application and see the result inside of Zeppelin notebook. + +org.apache.zeppelin.interpreter.dev.ZeppelinApplicationDevServer can run Zeppelin Application in development mode. + +```java + +// entry point for development mode +public static void main(String[] args) throws Exception { + + // add resources for development mode + LocalResourcePool pool = new LocalResourcePool("dev"); + pool.put("date", new Date()); + + // run application in devlopment mode with give resource + // in this case, Clock.class.getName() will be the application class name + ZeppelinApplicationDevServer devServer = new ZeppelinApplicationDevServer( + Clock.class.getName(), + pool.getAll()); + + // start development mode + devServer.start(); + devServer.join(); +} +``` + + +In the Zeppelin notebook, run `%dev run` will connect to application running in development mode. + + + + +## Package file + +Package file is a json file that provides information about the application. +Json file contains following informations + +``` +{ + name : "[organization].[name]", + description : "Description", + artifact : "groupId:artifactId:version", + className : "your.package.name.YourApplicationClass", + resources : [ + ["resource.name", ":resource.class.name"], + ["alternative.resource.name", ":alternative.class.name"] + ], + icon : "" +} + +``` + +#### name + +Name is a string in '[group].[name]' format. +[group] and [name] allows only [A-Za-z0-9_]. +Group is normally organization name who creates this application. + +#### description + +Short description. about application + +#### artifact + +Location of the jar artifact. +"groupId:artifactId:version" will make load artifact from maven repository. +If jar is in local filesystem, absolute/relative can be used. + +e.g. + +When artifact exists in Maven repository + +`artifact: "org.apache.zeppelin:zeppelin-examples:0.6.0"` + +When artifact exists in local filesystem + +`artifact: "zeppelin-example/target/zeppelin-example-0.6.0.jar"` + + +#### className + +Entry point. Class that extends `org.apache.zeppelin.helium.Application` + + +#### resources + +Two dimensional array that defines required resources by name or by className. Helium Application launcher will compare resources in the ResourcePool with informations in this field and suggest application only when all required resources are available in the ResourcePool. + +Resouce name is a string which will be compared with name of objects in the ResourcePool. className is a string with ":" prepended, which will be compared with className of the objects in the ResourcePool. + +Application may require two or more resources. Required resource can be listed inside of json array. For example, if application requires object "name1", "name2" and "className1" type of object to run, resources field can be + +``` +resources: [ + [ "name1", "name2", ":className1", ...] +] +``` + +If Application can handle alternative combination of required resource, alternative set can be listed as below. + +``` +resources: [ + [ "name", ":className"], + [ "altName", ":altClassName1"], + ... +] +``` + +Easier way of understanding this scheme is + +``` +resources: [ + [ 'resource' AND 'resource' AND ... ] OR + [ 'resource' AND 'resource' AND ... ] OR + ... +] +``` + + +#### icon + +Icon to be used on the application button. String in this field will be rendered as a html. + +e.g. + +``` +icon: "" +``` + diff --git a/pom.xml b/pom.xml index 602b06e878f..d1dda136364 100755 --- a/pom.xml +++ b/pom.xml @@ -667,6 +667,13 @@ + + examples + + zeppelin-examples + + + build-distr diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 6783378efda..53bfc02b370 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -50,6 +50,8 @@ import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.InterpreterUtils; import org.apache.zeppelin.interpreter.WrappedInterpreter; +import org.apache.zeppelin.resource.ResourcePool; +import org.apache.zeppelin.resource.WellKnownResourceName; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; @@ -533,7 +535,9 @@ public void open() { binder.put("sc", sc); binder.put("sqlc", sqlc); binder.put("z", z); - + binder.put("intp", intp); + intp.interpret("@transient val intp = _binder.get(\"intp\").asInstanceOf[org.apache.spark" + + ".repl.SparkIMain]"); intp.interpret("@transient val z = " + "_binder.get(\"z\").asInstanceOf[org.apache.zeppelin.spark.ZeppelinContext]"); intp.interpret("@transient val sc = " @@ -759,13 +763,10 @@ public InterpreterResult interpret(String[] lines, InterpreterContext context) { public InterpreterResult interpretInput(String[] lines, InterpreterContext context) { SparkEnv.set(env); - // add print("") to make sure not finishing with comment - // see https://github.com/NFLabs/zeppelin/issues/151 - String[] linesToRun = new String[lines.length + 1]; + String[] linesToRun = new String[lines.length]; for (int i = 0; i < lines.length; i++) { linesToRun[i] = lines[i]; } - linesToRun[lines.length] = "print(\"\")"; Console.setOut(context.out); out.setInterpreterOutput(context.out); @@ -828,17 +829,39 @@ public InterpreterResult interpretInput(String[] lines, InterpreterContext conte } } + // make sure code does not finish with comment + if (r == Code.INCOMPLETE) { + scala.tools.nsc.interpreter.Results.Result res; + res = intp.interpret(incomplete + "\nprint(\"\")"); + r = getResultCode(res); + } + if (r == Code.INCOMPLETE) { sc.clearJobGroup(); out.setInterpreterOutput(null); return new InterpreterResult(r, "Incomplete expression"); } else { sc.clearJobGroup(); + putLatestVarInResourcePool(context); out.setInterpreterOutput(null); return new InterpreterResult(Code.SUCCESS); } } + private void putLatestVarInResourcePool(InterpreterContext context) { + String varName = intp.mostRecentVar(); + if (varName == null || varName.isEmpty()) { + return; + } + + Object lastObj = getValue(varName); + if (lastObj != null) { + ResourcePool resourcePool = context.getResourcePool(); + resourcePool.put(context.getNoteId(), context.getParagraphId(), + WellKnownResourceName.ZeppelinReplResult.toString(), lastObj); + } + }; + @Override public void cancel(InterpreterContext context) { diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java index eb8d876021a..90f311ed5c3 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java @@ -29,6 +29,7 @@ import org.apache.spark.SparkConf; import org.apache.spark.SparkContext; import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.resource.LocalResourcePool; import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.*; @@ -95,7 +96,7 @@ public void setUp() throws Exception { new HashMap(), new GUI(), new AngularObjectRegistry(intpGroup.getId(), null), - null, + new LocalResourcePool("id"), new LinkedList(), new InterpreterOutput(new InterpreterOutputListener() { @Override diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java index 3196cf56bd8..73d5e8abbc8 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java @@ -24,6 +24,7 @@ import java.util.Properties; import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.resource.LocalResourcePool; import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.*; @@ -77,7 +78,7 @@ public void setUp() throws Exception { context = new InterpreterContext("note", "id", "title", "text", new AuthenticationInfo(), new HashMap(), new GUI(), new AngularObjectRegistry(intpGroup.getId(), null), - null, + new LocalResourcePool("id"), new LinkedList(), new InterpreterOutput(new InterpreterOutputListener() { @Override public void onAppend(InterpreterOutput out, byte[] line) { diff --git a/zeppelin-examples/pom.xml b/zeppelin-examples/pom.xml new file mode 100644 index 00000000000..a017d3c8ef5 --- /dev/null +++ b/zeppelin-examples/pom.xml @@ -0,0 +1,65 @@ + + + + + 4.0.0 + + + zeppelin + org.apache.zeppelin + 0.6.0-SNAPSHOT + .. + + + org.apache.zeppelin + zeppelin-examples + pom + 0.6.0-SNAPSHOT + Zeppelin: Examples + Zeppelin examples + http://zeppelin.apache.org + + + zeppelin-example-clock + zeppelin-example-horizontalbar + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.7 + + true + + + + + maven-enforcer-plugin + 1.3.1 + + + enforce + none + + + + + + diff --git a/zeppelin-examples/zeppelin-example-clock/pom.xml b/zeppelin-examples/zeppelin-example-clock/pom.xml new file mode 100644 index 00000000000..7ed5fce1b31 --- /dev/null +++ b/zeppelin-examples/zeppelin-example-clock/pom.xml @@ -0,0 +1,111 @@ + + + + + 4.0.0 + + + zeppelin-examples + org.apache.zeppelin + 0.6.0-SNAPSHOT + .. + + + org.apache.zeppelin + zeppelin-example-clock + jar + 0.6.0-SNAPSHOT + Zeppelin: Example application - Clock + http://zeppelin.apache.org + + + + ${project.groupId} + zeppelin-interpreter + ${project.version} + + + + org.slf4j + slf4j-api + + + + org.slf4j + slf4j-log4j12 + + + + junit + junit + test + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.7 + + true + + + + + maven-clean-plugin + + + + ${project.basedir}/../../helium + + ${project.artifactId}.json + + + + + + + + maven-resources-plugin + 2.7 + + + generate-resources + + copy-resources + + + + ${project.basedir}/../../helium/ + + + ${project.basedir} + + ${project.artifactId}.json + + + + + + + + + + diff --git a/zeppelin-examples/zeppelin-example-clock/src/main/java/org/apache/zeppelin/example/app/clock/Clock.java b/zeppelin-examples/zeppelin-example-clock/src/main/java/org/apache/zeppelin/example/app/clock/Clock.java new file mode 100644 index 00000000000..632b35ff251 --- /dev/null +++ b/zeppelin-examples/zeppelin-example-clock/src/main/java/org/apache/zeppelin/example/app/clock/Clock.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.example.app.clock; + +import org.apache.zeppelin.helium.Application; +import org.apache.zeppelin.helium.ApplicationContext; +import org.apache.zeppelin.helium.ApplicationException; +import org.apache.zeppelin.interpreter.dev.ZeppelinApplicationDevServer; +import org.apache.zeppelin.resource.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Date; + +/** + * Basic example application. + * Get java.util.Date from resource pool and display it + */ +public class Clock extends Application { + private final Logger logger = LoggerFactory.getLogger(Clock.class); + + Date date; + boolean shutdown = false; + private Thread updateThread; + + public Clock(ApplicationContext context) { + super(context); + } + + @Override + public void run(ResourceSet resources) throws ApplicationException { + // Get data from resource args + date = (Date) resources.get(0).get(); + + // print view template + try { + context().out.writeResource("example/app/clock/clock.html"); + } catch (IOException e) { + throw new ApplicationException(e); + } + + if (updateThread == null) { + start(); + } + } + + + public void start() { + updateThread = new Thread() { + public void run() { + while (!shutdown) { + // format date + SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + // put formatted string to angular object. + context().getAngularObjectRegistry().add("date", df.format(date)); + + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // nothing todo + } + date = new Date(date.getTime() + 1000); + } + } + }; + + updateThread.start(); + } + + + @Override + public void unload() throws ApplicationException { + shutdown = true; + try { + updateThread.join(); + } catch (InterruptedException e) { + // nothing to do + } + context().getAngularObjectRegistry().remove("date"); + } + + /** + * Development mode + */ + public static void main(String[] args) throws Exception { + LocalResourcePool pool = new LocalResourcePool("dev"); + pool.put("date", new Date()); + + ZeppelinApplicationDevServer devServer = new ZeppelinApplicationDevServer( + Clock.class.getName(), + pool.getAll()); + + devServer.start(); + devServer.join(); + } +} diff --git a/zeppelin-examples/zeppelin-example-clock/src/main/resources/example/app/clock/clock.html b/zeppelin-examples/zeppelin-example-clock/src/main/resources/example/app/clock/clock.html new file mode 100644 index 00000000000..ff492f03e5c --- /dev/null +++ b/zeppelin-examples/zeppelin-example-clock/src/main/resources/example/app/clock/clock.html @@ -0,0 +1,14 @@ + +

    {{date}}

    \ No newline at end of file diff --git a/zeppelin-examples/zeppelin-example-clock/zeppelin-example-clock.json b/zeppelin-examples/zeppelin-example-clock/zeppelin-example-clock.json new file mode 100644 index 00000000000..7cb05fec56c --- /dev/null +++ b/zeppelin-examples/zeppelin-example-clock/zeppelin-example-clock.json @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +{ + "type" : "APPLICATION", + "name" : "zeppelin.clock", + "description" : "Clock (example)", + "artifact" : "zeppelin-examples/zeppelin-example-clock/target/zeppelin-example-clock-0.6.0-SNAPSHOT.jar", + "className" : "org.apache.zeppelin.example.app.clock.Clock", + "resources" : [[":java.util.Date"]], + "icon" : '' +} diff --git a/zeppelin-examples/zeppelin-example-horizontalbar/pom.xml b/zeppelin-examples/zeppelin-example-horizontalbar/pom.xml new file mode 100644 index 00000000000..f34c55ed7a6 --- /dev/null +++ b/zeppelin-examples/zeppelin-example-horizontalbar/pom.xml @@ -0,0 +1,121 @@ + + + + + 4.0.0 + + + zeppelin-examples + org.apache.zeppelin + 0.6.0-SNAPSHOT + .. + + + org.apache.zeppelin + zeppelin-example-horizontalbar + jar + 0.6.0-SNAPSHOT + Zeppelin: Example application - Horizontal Bar chart + http://zeppelin.apache.org + + + + ${project.groupId} + zeppelin-interpreter + ${project.version} + + + + org.slf4j + slf4j-api + + + + org.slf4j + slf4j-log4j12 + + + + junit + junit + test + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.7 + + true + + + + + maven-clean-plugin + + + + ${project.basedir}/../../helium + + ${project.artifactId}.json + + + + + + + + maven-resources-plugin + 2.7 + + + generate-resources + + copy-resources + + + + ${project.basedir}/../../helium/ + + + ${project.basedir} + + ${project.artifactId}.json + + + + + + + + + + org.apache.rat + apache-rat-plugin + + + **/horizontalbar_mockdata.txt + + + + + + diff --git a/zeppelin-examples/zeppelin-example-horizontalbar/src/main/java/org/apache/zeppelin/example/app/horizontalbar/HorizontalBar.java b/zeppelin-examples/zeppelin-example-horizontalbar/src/main/java/org/apache/zeppelin/example/app/horizontalbar/HorizontalBar.java new file mode 100644 index 00000000000..fa209f60499 --- /dev/null +++ b/zeppelin-examples/zeppelin-example-horizontalbar/src/main/java/org/apache/zeppelin/example/app/horizontalbar/HorizontalBar.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.example.app.horizontalbar; + +import org.apache.commons.io.IOUtils; +import org.apache.zeppelin.helium.Application; +import org.apache.zeppelin.helium.ApplicationContext; +import org.apache.zeppelin.helium.ApplicationException; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.dev.ZeppelinApplicationDevServer; +import org.apache.zeppelin.resource.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Random; + +/** + * Basic example application. + * TableData for input + */ +public class HorizontalBar extends Application { + private final Logger logger = LoggerFactory.getLogger(HorizontalBar.class); + + InterpreterResult result; + + public HorizontalBar(ApplicationContext context) { + super(context); + } + + @Override + public void run(ResourceSet resources) throws ApplicationException, IOException { + // Get data from resource args + result = (InterpreterResult) resources.get(0).get(); + + // create element + println(String.format( + "
    ", + context().getApplicationInstanceId())); + // write js + printResourceAsJavascript("example/app/horizontalbar/horizontalbar.js"); + } + + @Override + public void unload() throws ApplicationException { + } + + /** + * Development mode + */ + public static void main(String[] args) throws Exception { + LocalResourcePool pool = new LocalResourcePool("dev"); + InputStream ins = ClassLoader.getSystemResourceAsStream( + "example/app/horizontalbar/horizontalbar_mockdata.txt"); + InterpreterResult result = new InterpreterResult( + InterpreterResult.Code.SUCCESS, + InterpreterResult.Type.TABLE, + IOUtils.toString(ins)); + pool.put(WellKnownResourceName.ZeppelinTableResult.name(), result); + + ZeppelinApplicationDevServer devServer = new ZeppelinApplicationDevServer( + HorizontalBar.class.getName(), + pool.getAll()); + + devServer.start(); + devServer.join(); + } +} diff --git a/zeppelin-examples/zeppelin-example-horizontalbar/src/main/resources/example/app/horizontalbar/horizontalbar.js b/zeppelin-examples/zeppelin-example-horizontalbar/src/main/resources/example/app/horizontalbar/horizontalbar.js new file mode 100644 index 00000000000..fac2c8eb877 --- /dev/null +++ b/zeppelin-examples/zeppelin-example-horizontalbar/src/main/resources/example/app/horizontalbar/horizontalbar.js @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +var data = []; +_.forEach($z.result.columnNames, function(col, series) { + if (series == 0) return; + var values = _.map($z.result.rows, function(row) { + return { + label: row[0], + value : parseFloat(row[series]) + } + }); + + data.push({ + key : col.name, + values : values + }) +}); + +nv.addGraph(function() { + var chart = nv.models.multiBarHorizontalChart() + .x(function(d) { return d.label }) + .y(function(d) { return d.value }) + .margin({top: 30, right: 20, bottom: 50, left: 175}) + .showValues(true) //Show bar value next to each bar. + .tooltips(true) //Show tooltips on hover. + .showControls(true); //Allow user to switch between "Grouped" and "Stacked" mode. + + chart.yAxis + .tickFormat(d3.format(',.2f')); + + d3.select('#horizontalbar_' + $z.id + ' svg') + .datum(data) + .call(chart); + + nv.utils.windowResize(chart.update); + + return chart; +}); + + + diff --git a/zeppelin-examples/zeppelin-example-horizontalbar/src/main/resources/example/app/horizontalbar/horizontalbar_mockdata.txt b/zeppelin-examples/zeppelin-example-horizontalbar/src/main/resources/example/app/horizontalbar/horizontalbar_mockdata.txt new file mode 100644 index 00000000000..adf322d547c --- /dev/null +++ b/zeppelin-examples/zeppelin-example-horizontalbar/src/main/resources/example/app/horizontalbar/horizontalbar_mockdata.txt @@ -0,0 +1,10 @@ +Label Series1 Series2 +GroupA -1.8746444827653 25.307646510375 +GroupB -8.0961543492239 16.756779544553 +GroupC -0.57072943117674 18.451534877007 +GroupD -2.4174010336624 8.6142352811805 +GroupE -0.72009071426284 7.8082472075876 +GroupF -0.77154485523777 5.259101026956 +GroupG -0.90152097798131 0.30947953487127 +GroupH -0.91445417330854 0 +GroupI -0.055746319141851 0 diff --git a/zeppelin-examples/zeppelin-example-horizontalbar/zeppelin-example-horizontalbar.json b/zeppelin-examples/zeppelin-example-horizontalbar/zeppelin-example-horizontalbar.json new file mode 100644 index 00000000000..afac7371c2d --- /dev/null +++ b/zeppelin-examples/zeppelin-example-horizontalbar/zeppelin-example-horizontalbar.json @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +{ + "type" : "APPLICATION", + "name" : "zeppelin.horizontalbar", + "description" : "Horizontal Bar chart (example)", + "artifact" : "zeppelin-examples/zeppelin-example-horizontalbar/target/zeppelin-example-horizontalbar-0.6.0-SNAPSHOT.jar", + "className" : "org.apache.zeppelin.example.app.horizontalbar.HorizontalBar", + "resources" : [[":org.apache.zeppelin.interpreter.InterpreterResult"]], + "icon" : '' +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java new file mode 100644 index 00000000000..291899877db --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import org.apache.zeppelin.annotation.Experimental; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.resource.ResourceSet; + +import java.io.IOException; + +/** + * Base class for pluggable application (e.g. visualization) + * Application can access resources from ResourcePool and interact with front-end using + * AngularDisplay system + */ +@Experimental +public abstract class Application { + + private final ApplicationContext context; + + public Application(ApplicationContext context) { + this.context = context; + } + + public ApplicationContext context() { + return context; + } + + /** + * This method can be invoked multiple times before unload(), + * Either just after application selected or when paragraph re-run after application load + */ + @Experimental + public abstract void run(ResourceSet args) + throws ApplicationException, IOException; + + + /** + * this method is invoked just before application is removed + */ + @Experimental + public abstract void unload() throws ApplicationException; + + /** + * Print string on the notebook + * @param string + * @throws IOException + */ + @Experimental + public void print(String string) throws IOException { + context.out.write(string); + } + + /** + * Print string on the notebook with newline + * @param string + * @throws IOException + */ + @Experimental + public void println(String string) throws IOException { + print(string + "\n"); + } + + /** + * Print resource on the notebook + * @param resourceName + * @throws IOException + */ + @Experimental + public void printResource(String resourceName) throws IOException { + context.out.writeResource(resourceName); + } + + /** + * Print resource as a javascript + * + * Using this method does not require print javascript inside of tag. + * Javascript printed using this method will be run in the un-named function. + * i.e. each method call will creates different variable scope for the javascript code. + * + * This method inject '$z' into the variable scope for convenience. + * + * $z.scope : angularjs scope object for this application + * $z.id : unique id for this application instance + * + * @param resourceName + * @throws IOException + */ + @Experimental + public void printResourceAsJavascript(String resourceName) throws IOException { + beginJavascript(); + context.out.writeResource(resourceName); + endJavascript(); + } + + /** + * Print string as a javascript + * + * Using this method does not require print javascript inside of tag. + * Javascript printed using this method will be run in the un-named function. + * i.e. each method call will creates different variable scope for the javascript code. + * + * This method inject '$z' into the variable scope for convenience. + * + * $z.scope : angularjs scope object for this application + * $z.id : unique id for this application instance + * + * @param js + * @throws IOException + */ + @Experimental + public void printStringAsJavascript(String js) throws IOException { + beginJavascript(); + context.out.write(js); + endJavascript(); + } + + private void beginJavascript() throws IOException { + StringBuffer js = new StringBuffer(); + js.append("\n\n"); + context.out.write(js.toString()); + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java new file mode 100644 index 00000000000..e0ea94cb01f --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.interpreter.InterpreterOutput; + +/** + * ApplicationContext + */ +public class ApplicationContext { + private final String noteId; + private final String paragraphId; + private final String applicationInstanceId; + private final HeliumAppAngularObjectRegistry angularObjectRegistry; + public final InterpreterOutput out; + + + public ApplicationContext(String noteId, + String paragraphId, + String applicationInstanceId, + HeliumAppAngularObjectRegistry angularObjectRegistry, + InterpreterOutput out) { + this.noteId = noteId; + this.paragraphId = paragraphId; + this.applicationInstanceId = applicationInstanceId; + this.angularObjectRegistry = angularObjectRegistry; + this.out = out; + } + + public String getNoteId() { + return noteId; + } + + public String getParagraphId() { + return paragraphId; + } + + public String getApplicationInstanceId() { + return applicationInstanceId; + } + + public HeliumAppAngularObjectRegistry getAngularObjectRegistry() { + return angularObjectRegistry; + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationEventListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationEventListener.java new file mode 100644 index 00000000000..f89bb211265 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationEventListener.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +/** + * Event from HeliumApplication running on remote interpreter process + */ +public interface ApplicationEventListener { + public void onOutputAppend(String noteId, String paragraphId, String appId, String output); + public void onOutputUpdated(String noteId, String paragraphId, String appId, String output); + public void onLoad(String noteId, String paragraphId, String appId, HeliumPackage pkg); + public void onStatusChange(String noteId, String paragraphId, String appId, String status); +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationException.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationException.java new file mode 100644 index 00000000000..4bf0ac28b94 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationException.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +/** + * Application exception + */ +public class ApplicationException extends Exception { + public ApplicationException(String s) { + super(s); + } + + public ApplicationException(Exception e) { + super(e); + } + + public ApplicationException() { + + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java new file mode 100644 index 00000000000..be36ad96fec --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import org.apache.zeppelin.dep.DependencyResolver; +import org.apache.zeppelin.resource.DistributedResourcePool; +import org.apache.zeppelin.resource.Resource; +import org.apache.zeppelin.resource.ResourcePool; +import org.apache.zeppelin.resource.ResourceSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.lang.reflect.Constructor; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.*; + +/** + * Load application + */ +public class ApplicationLoader { + Logger logger = LoggerFactory.getLogger(ApplicationLoader.class); + + private final DependencyResolver depResolver; + private final ResourcePool resourcePool; + private final Map> cached; + + public ApplicationLoader(ResourcePool resourcePool, DependencyResolver depResolver) { + this.depResolver = depResolver; + this.resourcePool = resourcePool; + cached = Collections.synchronizedMap( + new HashMap>()); + } + + /** + * Information of loaded application + */ + private static class RunningApplication { + HeliumPackage packageInfo; + String noteId; + String paragraphId; + + public RunningApplication(HeliumPackage packageInfo, String noteId, String paragraphId) { + this.packageInfo = packageInfo; + this.noteId = noteId; + this.paragraphId = paragraphId; + } + + public HeliumPackage getPackageInfo() { + return packageInfo; + } + + public String getNoteId() { + return noteId; + } + + public String getParagraphId() { + return paragraphId; + } + + @Override + public int hashCode() { + return (paragraphId + noteId + packageInfo.getArtifact() + packageInfo.getClassName()) + .hashCode(); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof RunningApplication)) { + return false; + } + + RunningApplication r = (RunningApplication) o; + return packageInfo.equals(r.getPackageInfo()) && paragraphId.equals(r.getParagraphId()) && + noteId.equals(r.getNoteId()); + } + } + + /** + * + * Instantiate application + * + * @param packageInfo + * @param context + * @return + * @throws Exception + */ + public Application load(HeliumPackage packageInfo, ApplicationContext context) + throws Exception { + if (packageInfo.getType() != HeliumPackage.Type.APPLICATION) { + throw new ApplicationException( + "Can't instantiate " + packageInfo.getType() + " package using ApplicationLoader"); + } + + // check if already loaded + RunningApplication key = + new RunningApplication(packageInfo, context.getNoteId(), context.getParagraphId()); + + // get resource required by this package + ResourceSet resources = findRequiredResourceSet(packageInfo.getResources(), + context.getNoteId(), context.getParagraphId()); + + // load class + Class appClass = loadClass(packageInfo); + + // instantiate + ClassLoader oldcl = Thread.currentThread().getContextClassLoader(); + ClassLoader cl = appClass.getClassLoader(); + Thread.currentThread().setContextClassLoader(cl); + try { + Constructor constructor = appClass.getConstructor(ApplicationContext.class); + + Application app = new ClassLoaderApplication(constructor.newInstance(context), cl); + return app; + } catch (Exception e) { + throw new ApplicationException(e); + } finally { + Thread.currentThread().setContextClassLoader(oldcl); + } + } + + public ResourceSet findRequiredResourceSet( + String [][] requiredResources, String noteId, String paragraphId) { + if (requiredResources == null || requiredResources.length == 0) { + return new ResourceSet(); + } + + ResourceSet allResources; + if (resourcePool instanceof DistributedResourcePool) { + allResources = ((DistributedResourcePool) resourcePool).getAll(false); + } else { + allResources = resourcePool.getAll(); + } + + return findRequiredResourceSet(requiredResources, noteId, paragraphId, allResources); + } + + static ResourceSet findRequiredResourceSet(String [][] requiredResources, + String noteId, + String paragraphId, + ResourceSet resources) { + ResourceSet args = new ResourceSet(); + if (requiredResources == null || requiredResources.length == 0) { + return args; + } + + resources = resources.filterByNoteId(noteId).filterByParagraphId(paragraphId); + + for (String [] requires : requiredResources) { + args.clear(); + + for (String require : requires) { + boolean found = false; + + for (Resource r : resources) { + if (require.startsWith(":") && r.getClassName().equals(require.substring(1))) { + found = true; + } else if (r.getResourceId().getName().equals(require)) { + found = true; + } + + if (found) { + args.add(r); + break; + } + } + + if (found == false) { + break; + } + } + + if (args.size() == requires.length) { + return args; + } + } + + return null; + } + + + private Class loadClass(HeliumPackage packageInfo) throws Exception { + if (cached.containsKey(packageInfo)) { + return cached.get(packageInfo); + } + + // Create Application classloader + List urlList = new LinkedList(); + + // load artifact + if (packageInfo.getArtifact() != null) { + List paths = depResolver.load(packageInfo.getArtifact()); + + if (paths != null) { + + for (File path : paths) { + urlList.add(path.toURI().toURL()); + } + } + } + URLClassLoader applicationClassLoader = + new URLClassLoader( + urlList.toArray(new URL[]{}), + Thread.currentThread().getContextClassLoader()); + + Class cls = + (Class) applicationClassLoader.loadClass(packageInfo.getClassName()); + cached.put(packageInfo, cls); + return cls; + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ClassLoaderApplication.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ClassLoaderApplication.java new file mode 100644 index 00000000000..272a152448e --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ClassLoaderApplication.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import org.apache.zeppelin.resource.ResourceSet; + +/** + * Application wrapper + */ +public class ClassLoaderApplication extends Application { + Application app; + ClassLoader cl; + public ClassLoaderApplication(Application app, ClassLoader cl) throws ApplicationException { + super(app.context()); + this.app = app; + this.cl = cl; + } + + @Override + public void run(ResourceSet args) throws ApplicationException { + // instantiate + ClassLoader oldcl = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(cl); + try { + app.run(args); + } catch (ApplicationException e) { + throw e; + } catch (Exception e) { + throw new ApplicationException(e); + } finally { + Thread.currentThread().setContextClassLoader(oldcl); + } + } + + @Override + public void unload() throws ApplicationException { + // instantiate + ClassLoader oldcl = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(cl); + try { + app.unload(); + } catch (ApplicationException e) { + throw e; + } catch (Exception e) { + throw new ApplicationException(e); + } finally { + Thread.currentThread().setContextClassLoader(oldcl); + } + } + + public ClassLoader getClassLoader() { + return cl; + } + + public Application getInnerApplication() { + return app; + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumAppAngularObjectRegistry.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumAppAngularObjectRegistry.java new file mode 100644 index 00000000000..dedb603add8 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumAppAngularObjectRegistry.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import org.apache.zeppelin.display.AngularObject; +import org.apache.zeppelin.display.AngularObjectRegistry; + +import java.util.List; + +/** + * Angular Registry for helium app + */ +public class HeliumAppAngularObjectRegistry { + private final String noteId; + private final String appId; + private final AngularObjectRegistry angularObjectRegistry; + + public HeliumAppAngularObjectRegistry(AngularObjectRegistry angularObjectRegistry, + String noteId, + String appId) { + this.angularObjectRegistry = angularObjectRegistry; + this.noteId = noteId; + this.appId = appId; + } + + public AngularObject add(String name, Object o) { + return angularObjectRegistry.add(name, o, noteId, appId); + } + + public AngularObject remove(String name) { + return angularObjectRegistry.remove(name, noteId, appId); + } + + public AngularObject get(String name) { + return angularObjectRegistry.get(name, noteId, appId); + } + + public List getAll() { + return angularObjectRegistry.getAll(noteId, appId); + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java new file mode 100644 index 00000000000..13526420e39 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import org.apache.zeppelin.annotation.Experimental; + +/** + * Helium package definition + */ +@Experimental +public class HeliumPackage { + private Type type; + private String name; // user friendly name of this application + private String description; // description + private String artifact; // artifact name e.g) groupId:artifactId:versionId + private String className; // entry point + private String [][] resources; // resource classnames that requires + // [[ .. and .. and .. ] or [ .. and .. and ..] ..] + private String icon; + /** + * Type of package + */ + public static enum Type { + INTERPRETER, + NOTEBOOK_REPO, + APPLICATION + } + + public HeliumPackage(Type type, + String name, + String description, + String artifact, + String className, + String[][] resources) { + this.type = type; + this.name = name; + this.description = description; + this.artifact = artifact; + this.className = className; + this.resources = resources; + } + + @Override + public int hashCode() { + return (type.toString() + artifact + className).hashCode(); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof HeliumPackage)) { + return false; + } + + HeliumPackage info = (HeliumPackage) o; + return type == info.type && artifact.equals(info.artifact) && className.equals(info.className); + } + + public Type getType() { + return type; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public String getArtifact() { + return artifact; + } + + public String getClassName() { + return className; + } + + public String[][] getResources() { + return resources; + } + + public String getIcon() { + return icon; + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java index b5d5863ab5c..bc56784b15b 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java @@ -23,7 +23,6 @@ import org.apache.log4j.Logger; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; -import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; import org.apache.zeppelin.resource.ResourcePool; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java index 42ebe485e1b..df0c210b8d9 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java @@ -21,9 +21,12 @@ import org.slf4j.LoggerFactory; import java.io.*; +import java.net.URISyntaxException; import java.net.URL; +import java.util.Collections; import java.util.LinkedList; import java.util.List; +import java.util.concurrent.ConcurrentHashMap; /** * InterpreterOutput is OutputStream that supposed to print content on notebook @@ -32,6 +35,7 @@ public class InterpreterOutput extends OutputStream { Logger logger = LoggerFactory.getLogger(InterpreterOutput.class); private final int NEW_LINE_CHAR = '\n'; + private List resourceSearchPaths = Collections.synchronizedList(new LinkedList()); ByteArrayOutputStream buffer = new ByteArrayOutputStream(); @@ -61,7 +65,6 @@ public InterpreterResult.Type getType() { public void setType(InterpreterResult.Type type) { if (this.type != type) { clear(); - flushListener.onUpdate(this, new byte[]{}); this.type = type; } } @@ -74,6 +77,8 @@ public void clear() { if (watcher != null) { watcher.clear(); } + + flushListener.onUpdate(this, new byte[]{}); } } @@ -149,33 +154,33 @@ public void write(String string) throws IOException { * @throws IOException */ public void write(URL url) throws IOException { - if ("file".equals(url.getProtocol())) { - write(new File(url.getPath())); - } else { - outList.add(url); - } + outList.add(url); + } + + public void addResourceSearchPath(String path) { + resourceSearchPaths.add(path); } public void writeResource(String resourceName) throws IOException { - // search file under resource dir first for dev mode - File mainResource = new File("./src/main/resources/" + resourceName); - File testResource = new File("./src/test/resources/" + resourceName); - if (mainResource.isFile()) { - write(mainResource); - } else if (testResource.isFile()) { - write(testResource); - } else { - // search from classpath - ClassLoader cl = Thread.currentThread().getContextClassLoader(); - if (cl == null) { - cl = this.getClass().getClassLoader(); - } - if (cl == null) { - cl = ClassLoader.getSystemClassLoader(); + // search file under provided paths first, for dev mode + for (String path : resourceSearchPaths) { + File res = new File(path + "/" + resourceName); + if (res.isFile()) { + write(res); + return; } + } - write(cl.getResource(resourceName)); + // search from classpath + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + if (cl == null) { + cl = this.getClass().getClassLoader(); } + if (cl == null) { + cl = ClassLoader.getSystemClassLoader(); + } + + write(cl.getResource(resourceName)); } public byte[] toByteArray() throws IOException { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/DevInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/DevInterpreter.java new file mode 100644 index 00000000000..a972b91efdc --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/DevInterpreter.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.interpreter.dev; + +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterContextRunner; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; + +/** + * Dummy interpreter to support development mode for Zeppelin app + */ +public class DevInterpreter extends Interpreter { + static { + Interpreter.register( + "dev", + "dev", + DevInterpreter.class.getName(), + new InterpreterPropertyBuilder().build()); + } + + private InterpreterEvent interpreterEvent; + private InterpreterContext context; + + public static boolean isInterpreterName(String replName) { + return replName.equals("dev"); + } + + /** + * event handler for ZeppelinApplicationDevServer + */ + public static interface InterpreterEvent { + public InterpreterResult interpret(String st, InterpreterContext context); + } + + public DevInterpreter(Properties property) { + super(property); + } + + public DevInterpreter(Properties property, InterpreterEvent interpreterEvent) { + super(property); + this.interpreterEvent = interpreterEvent; + } + + @Override + public void open() { + } + + @Override + public void close() { + } + + public void rerun() { + for (InterpreterContextRunner r : context.getRunners()) { + if (context.getParagraphId().equals(r.getParagraphId())) { + r.run(); + } + } + } + + @Override + public InterpreterResult interpret(String st, InterpreterContext context) { + this.context = context; + try { + return interpreterEvent.interpret(st, context); + } catch (Exception e) { + throw new InterpreterException(e); + } + } + + @Override + public void cancel(InterpreterContext context) { + } + + @Override + public FormType getFormType() { + return FormType.NATIVE; + } + + @Override + public int getProgress(InterpreterContext context) { + return 0; + } + + @Override + public List completion(String buf, int cursor) { + return new LinkedList(); + } + + public InterpreterContext getLastInterpretContext() { + return context; + } + + public void setInterpreterEvent(InterpreterEvent event) { + this.interpreterEvent = event; + } + + public InterpreterEvent getInterpreterEvent() { + return interpreterEvent; + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java new file mode 100644 index 00000000000..941fdfe1f06 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.interpreter.dev; + +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.net.URL; + +import com.google.gson.Gson; +import org.apache.log4j.ConsoleAppender; +import org.apache.log4j.Level; +import org.apache.log4j.PatternLayout; +import org.apache.zeppelin.helium.*; +import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventClient; +import org.apache.zeppelin.resource.ResourceSet; +import org.apache.zeppelin.resource.WellKnownResourceName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Run this server for development mode. + */ +public class ZeppelinApplicationDevServer extends ZeppelinDevServer { + final Logger logger = LoggerFactory.getLogger(ZeppelinApplicationDevServer.class); + + private final String className; + private final ResourceSet resourceSet; + private Application app; + private InterpreterOutput out; + + public ZeppelinApplicationDevServer(final String className, ResourceSet resourceSet) throws + Exception { + this(ZeppelinDevServer.DEFAULT_TEST_INTERPRETER_PORT, className, resourceSet); + } + + public ZeppelinApplicationDevServer(int port, String className, ResourceSet resourceSet) throws + Exception { + super(port); + this.className = className; + this.resourceSet = resourceSet; + setLogger(); + }; + + void setLogger() { + ConsoleAppender console = new ConsoleAppender(); //create appender + //configure the appender + String PATTERN = "%d [%p|%c|%C{1}] %m%n"; + console.setLayout(new PatternLayout(PATTERN)); + console.setThreshold(Level.DEBUG); + console.activateOptions(); + //add appender to any Logger (here is root) + org.apache.log4j.Logger.getRootLogger().addAppender(console); + } + + + @Override + public InterpreterResult interpret(String st, InterpreterContext context) { + if (app == null) { + logger.info("Create instance " + className); + try { + Class appClass = ClassLoader.getSystemClassLoader().loadClass(className); + Constructor constructor = appClass.getConstructor(ApplicationContext.class); + + // classPath will be ..../target/classes in dev mode most cases + String classPath = appClass.getProtectionDomain().getCodeSource().getLocation().getPath(); + + context.out.addResourceSearchPath(classPath + "../../src/main/resources/"); + context.out.addResourceSearchPath(classPath + "../../src/test/resources/"); + + ApplicationContext appContext = getApplicationContext(context); + app = (Application) constructor.newInstance(appContext); + } catch (Exception e) { + logger.error(e.getMessage(), e); + return new InterpreterResult(Code.ERROR, e.getMessage()); + } + } + + try { + logger.info("Run " + className); + app.context().out.clear(); + app.context().out.setType(InterpreterResult.Type.ANGULAR); + transferTableResultDataToFrontend(); + app.run(resourceSet); + } catch (IOException | ApplicationException e) { + logger.error(e.getMessage(), e); + return new InterpreterResult(Code.ERROR, e.getMessage()); + } + return new InterpreterResult(Code.SUCCESS, ""); + } + + private void transferTableResultDataToFrontend() throws IOException { + ResourceSet results = resourceSet.filterByClassname(InterpreterResult.class.getName()); + if (results.size() == 0) { + return; + } + + InterpreterResult result = (InterpreterResult) results.get(0).get(); + Gson gson = new Gson(); + String resultJson = gson.toJson(result); + StringBuffer transferResult = new StringBuffer(); + transferResult.append("$z.result = " + resultJson + ";\n"); + if (result.type() == InterpreterResult.Type.TABLE) { + transferResult.append("$z.scope.loadTableData($z.result);\n"); + } + transferResult.append("$z.scope._devmodeResult = $z.result;\n"); + app.printStringAsJavascript(transferResult.toString()); + } + + ApplicationContext getApplicationContext(InterpreterContext interpreterContext) { + return new ApplicationContext( + interpreterContext.getNoteId(), + interpreterContext.getParagraphId(), + "app_" + this.hashCode(), + new HeliumAppAngularObjectRegistry( + interpreterContext.getAngularObjectRegistry(), + interpreterContext.getNoteId(), + interpreterContext.getParagraphId()), + interpreterContext.out); + } + + @Override + protected InterpreterOutput createInterpreterOutput( + final String noteId, final String paragraphId) { + if (out == null) { + final RemoteInterpreterEventClient eventClient = getEventClient(); + try { + out = new InterpreterOutput(new InterpreterOutputListener() { + @Override + public void onAppend(InterpreterOutput out, byte[] line) { + eventClient.onInterpreterOutputAppend(noteId, paragraphId, new String(line)); + } + + @Override + public void onUpdate(InterpreterOutput out, byte[] output) { + eventClient.onInterpreterOutputUpdate(noteId, paragraphId, new String(output)); + } + }, this); + } catch (IOException e) { + return null; + } + } + + return out; + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinDevServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinDevServer.java new file mode 100644 index 00000000000..9f409230a8c --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinDevServer.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.interpreter.dev; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Properties; + +import org.apache.thrift.TException; +import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.interpreter.dev.DevInterpreter.InterpreterEvent; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventClient; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Interpreter development server + */ +public class ZeppelinDevServer extends + RemoteInterpreterServer implements InterpreterEvent, InterpreterOutputChangeListener { + final Logger logger = LoggerFactory.getLogger(ZeppelinDevServer.class); + public static final int DEFAULT_TEST_INTERPRETER_PORT = 29914; + + DevInterpreter interpreter = null; + InterpreterOutput out; + public ZeppelinDevServer(int port) throws TException { + super(port); + } + + @Override + protected Interpreter getInterpreter(String noteId, String className) throws TException { + synchronized (this) { + InterpreterGroup interpreterGroup = getInterpreterGroup(); + if (interpreterGroup == null) { + createInterpreter( + "dev", + noteId, + DevInterpreter.class.getName(), + new HashMap()); + + Interpreter intp = super.getInterpreter(noteId, className); + interpreter = (DevInterpreter) ( + ((LazyOpenInterpreter) intp).getInnerInterpreter()); + interpreter.setInterpreterEvent(this); + notify(); + } + } + return super.getInterpreter(noteId, className); + } + + @Override + protected InterpreterOutput createInterpreterOutput( + final String noteId, final String paragraphId) { + if (out == null) { + final RemoteInterpreterEventClient eventClient = getEventClient(); + try { + out = new InterpreterOutput(new InterpreterOutputListener() { + @Override + public void onAppend(InterpreterOutput out, byte[] line) { + eventClient.onInterpreterOutputAppend(noteId, paragraphId, new String(line)); + } + + @Override + public void onUpdate(InterpreterOutput out, byte[] output) { + eventClient.onInterpreterOutputUpdate(noteId, paragraphId, new String(output)); + } + }, this); + } catch (IOException e) { + return null; + } + } + + out.clear(); + return out; + } + + @Override + public void fileChanged(File file) { + refresh(); + } + + @Override + public InterpreterResult interpret(String st, InterpreterContext context) { + waitForConnected(); + return new InterpreterResult(InterpreterResult.Code.SUCCESS, ""); + } + + public void refresh() { + interpreter.rerun(); + } + + /** + * Wait until %dev paragraph is executed and connected to this process + */ + public void waitForConnected() { + synchronized (this) { + while (!isConnected()) { + try { + this.wait(10 * 1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + } + + public boolean isConnected() { + return !(interpreter == null || interpreter.getLastInterpretContext() == null); + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java index 6126f75d3db..809c574cbc8 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java @@ -76,4 +76,9 @@ public void destroyObject(PooledObject p) { } } } + + @Override + public boolean validateObject(PooledObject p) { + return p.getObject().getOutputProtocol().getTransport().isOpen(); + } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java index b80a2526e92..0ac71165348 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java @@ -23,9 +23,7 @@ import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.AngularObjectRegistryListener; -import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterGroup; -import org.apache.zeppelin.interpreter.WrappedInterpreter; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -96,7 +94,7 @@ public AngularObject addAndNotifyRemoteProcess(String name, Object o, String not public AngularObject removeAndNotifyRemoteProcess(String name, String noteId, String paragraphId) { RemoteInterpreterProcess remoteInterpreterProcess = getRemoteInterpreterProcess(); - if (!remoteInterpreterProcess.isRunning()) { + if (remoteInterpreterProcess == null || !remoteInterpreterProcess.isRunning()) { return super.remove(name, noteId, paragraphId); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index e18edbde4e4..db740f4873a 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -23,6 +23,7 @@ import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; +import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.display.Input; import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.interpreter.InterpreterResult.Type; @@ -43,6 +44,7 @@ */ public class RemoteInterpreter extends Interpreter { private final RemoteInterpreterProcessListener remoteInterpreterProcessListener; + private final ApplicationEventListener applicationEventListener; Logger logger = LoggerFactory.getLogger(RemoteInterpreter.class); Gson gson = new Gson(); private String interpreterRunner; @@ -55,17 +57,22 @@ public class RemoteInterpreter extends Interpreter { private Map env; private int connectTimeout; private int maxPoolSize; - private static String schedulerName; + private String host; + private int port; + /** + * Remote interpreter and manage interpreter process + */ public RemoteInterpreter(Properties property, - String noteId, - String className, - String interpreterRunner, - String interpreterPath, - String localRepoPath, - int connectTimeout, - int maxPoolSize, - RemoteInterpreterProcessListener remoteInterpreterProcessListener) { + String noteId, + String className, + String interpreterRunner, + String interpreterPath, + String localRepoPath, + int connectTimeout, + int maxPoolSize, + RemoteInterpreterProcessListener remoteInterpreterProcessListener, + ApplicationEventListener appListener) { super(property); this.noteId = noteId; this.className = className; @@ -77,9 +84,39 @@ public RemoteInterpreter(Properties property, this.connectTimeout = connectTimeout; this.maxPoolSize = maxPoolSize; this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; + this.applicationEventListener = appListener; } - public RemoteInterpreter(Properties property, + + /** + * Connect to existing process + */ + public RemoteInterpreter( + Properties property, + String noteId, + String className, + String host, + int port, + int connectTimeout, + int maxPoolSize, + RemoteInterpreterProcessListener remoteInterpreterProcessListener, + ApplicationEventListener appListener) { + super(property); + this.noteId = noteId; + this.className = className; + initialized = false; + this.host = host; + this.port = port; + this.connectTimeout = connectTimeout; + this.maxPoolSize = maxPoolSize; + this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; + this.applicationEventListener = appListener; + } + + + // VisibleForTesting + public RemoteInterpreter( + Properties property, String noteId, String className, String interpreterRunner, @@ -87,7 +124,8 @@ public RemoteInterpreter(Properties property, String localRepoPath, Map env, int connectTimeout, - RemoteInterpreterProcessListener remoteInterpreterProcessListener) { + RemoteInterpreterProcessListener remoteInterpreterProcessListener, + ApplicationEventListener appListener) { super(property); this.className = className; this.noteId = noteId; @@ -99,6 +137,7 @@ public RemoteInterpreter(Properties property, this.connectTimeout = connectTimeout; this.maxPoolSize = 10; this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; + this.applicationEventListener = appListener; } private Map getEnvFromInterpreterProperty(Properties property) { @@ -124,6 +163,10 @@ public String getClassName() { return className; } + private boolean connectToExistingProcess() { + return host != null && port > 0; + } + public RemoteInterpreterProcess getInterpreterProcess() { InterpreterGroup intpGroup = getInterpreterGroup(); if (intpGroup == null) { @@ -132,10 +175,20 @@ public RemoteInterpreterProcess getInterpreterProcess() { synchronized (intpGroup) { if (intpGroup.getRemoteInterpreterProcess() == null) { - // create new remote process - RemoteInterpreterProcess remoteProcess = new RemoteInterpreterProcess( - interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout, - remoteInterpreterProcessListener); + RemoteInterpreterProcess remoteProcess; + if (connectToExistingProcess()) { + remoteProcess = new RemoteInterpreterRunningProcess( + connectTimeout, + remoteInterpreterProcessListener, + applicationEventListener, + host, + port); + } else { + // create new remote process + remoteProcess = new RemoteInterpreterManagedProcess( + interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout, + remoteInterpreterProcessListener, applicationEventListener); + } intpGroup.setRemoteInterpreterProcess(remoteProcess); } @@ -168,7 +221,9 @@ public synchronized void init() { boolean broken = false; try { logger.info("Create remote interpreter {}", getClassName()); - property.put("zeppelin.interpreter.localRepo", localRepoPath); + if (localRepoPath != null) { + property.put("zeppelin.interpreter.localRepo", localRepoPath); + } client.createInterpreter(groupId, noteId, getClassName(), (Map) property); @@ -462,4 +517,12 @@ void pushAngularObjectRegistryToRemote(Client client) throws TException { client.angularRegistryPush(gson.toJson(registry, registryType)); } } + + public Map getEnv() { + return env; + } + + public void setEnv(Map env) { + this.env = env; + } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java index 158f145de45..6f26ffd959b 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java @@ -242,4 +242,55 @@ private void sendEvent(RemoteInterpreterEvent event) { } } + public void onAppOutputAppend(String noteId, String paragraphId, String appId, String output) { + Map appendOutput = new HashMap(); + appendOutput.put("noteId", noteId); + appendOutput.put("paragraphId", paragraphId); + appendOutput.put("appId", appId); + appendOutput.put("data", output); + + sendEvent(new RemoteInterpreterEvent( + RemoteInterpreterEventType.OUTPUT_APPEND, + gson.toJson(appendOutput))); + } + + + public void onAppOutputUpdate(String noteId, String paragraphId, String appId, String output) { + Map appendOutput = new HashMap(); + appendOutput.put("noteId", noteId); + appendOutput.put("paragraphId", paragraphId); + appendOutput.put("appId", appId); + appendOutput.put("data", output); + + sendEvent(new RemoteInterpreterEvent( + RemoteInterpreterEventType.OUTPUT_UPDATE, + gson.toJson(appendOutput))); + } + + public void onAppStatusUpdate(String noteId, String paragraphId, String appId, String status) { + Map appendOutput = new HashMap(); + appendOutput.put("noteId", noteId); + appendOutput.put("paragraphId", paragraphId); + appendOutput.put("appId", appId); + appendOutput.put("status", status); + + sendEvent(new RemoteInterpreterEvent( + RemoteInterpreterEventType.APP_STATUS_UPDATE, + gson.toJson(appendOutput))); + } + + /** + * Wait for eventQueue becomes empty + */ + public void waitForEventQueueBecomesEmpty() { + synchronized (eventQueue) { + while (!eventQueue.isEmpty()) { + try { + eventQueue.wait(100); + } catch (InterruptedException e) { + // ignore exception + } + } + } + } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java index 48e79de8109..48c14d50bde 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java @@ -22,9 +22,9 @@ import org.apache.thrift.TException; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.InterpreterContextRunner; import org.apache.zeppelin.interpreter.InterpreterGroup; -import org.apache.zeppelin.interpreter.InterpreterOutputListener; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; @@ -36,7 +36,6 @@ import org.slf4j.LoggerFactory; import java.nio.ByteBuffer; -import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -47,14 +46,18 @@ public class RemoteInterpreterEventPoller extends Thread { private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterEventPoller.class); private final RemoteInterpreterProcessListener listener; + private final ApplicationEventListener appListener; private volatile boolean shutdown; private RemoteInterpreterProcess interpreterProcess; private InterpreterGroup interpreterGroup; - public RemoteInterpreterEventPoller(RemoteInterpreterProcessListener listener) { + public RemoteInterpreterEventPoller( + RemoteInterpreterProcessListener listener, + ApplicationEventListener appListener) { this.listener = listener; + this.appListener = appListener; shutdown = false; } @@ -70,7 +73,17 @@ public void setInterpreterGroup(InterpreterGroup interpreterGroup) { public void run() { Client client = null; - while (!shutdown && interpreterProcess.isRunning()) { + while (!shutdown) { + // wait and retry + if (!interpreterProcess.isRunning()) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // nothing to do + } + continue; + } + try { client = interpreterProcess.getClient(); } catch (Exception e1) { @@ -141,17 +154,38 @@ public void run() { String noteId = outputAppend.get("noteId"); String paragraphId = outputAppend.get("paragraphId"); String outputToAppend = outputAppend.get("data"); + String appId = outputAppend.get("appId"); - listener.onOutputAppend(noteId, paragraphId, outputToAppend); + if (appId == null) { + listener.onOutputAppend(noteId, paragraphId, outputToAppend); + } else { + appListener.onOutputAppend(noteId, paragraphId, appId, outputToAppend); + } } else if (event.getType() == RemoteInterpreterEventType.OUTPUT_UPDATE) { // on output update Map outputAppend = gson.fromJson( - event.getData(), new TypeToken>() {}.getType()); + event.getData(), new TypeToken>() {}.getType()); String noteId = outputAppend.get("noteId"); String paragraphId = outputAppend.get("paragraphId"); String outputToUpdate = outputAppend.get("data"); + String appId = outputAppend.get("appId"); + + if (appId == null) { + listener.onOutputUpdated(noteId, paragraphId, outputToUpdate); + } else { + appListener.onOutputUpdated(noteId, paragraphId, appId, outputToUpdate); + } + } else if (event.getType() == RemoteInterpreterEventType.APP_STATUS_UPDATE) { + // on output update + Map appStatusUpdate = gson.fromJson( + event.getData(), new TypeToken>() {}.getType()); + + String noteId = appStatusUpdate.get("noteId"); + String paragraphId = appStatusUpdate.get("paragraphId"); + String appId = appStatusUpdate.get("appId"); + String status = appStatusUpdate.get("status"); - listener.onOutputUpdated(noteId, paragraphId, outputToUpdate); + appListener.onStatusChange(noteId, paragraphId, appId, status); } logger.debug("Event from remoteproceess {}", event.getType()); } catch (Exception e) { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java new file mode 100644 index 00000000000..098a9d48431 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter.remote; + +import org.apache.commons.exec.*; +import org.apache.commons.exec.environment.EnvironmentUtils; +import org.apache.zeppelin.helium.ApplicationEventListener; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; + +/** + * This class manages start / stop of remote interpreter process + */ +public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess + implements ExecuteResultHandler { + private static final Logger logger = LoggerFactory.getLogger( + RemoteInterpreterManagedProcess.class); + private final String interpreterRunner; + + private DefaultExecutor executor; + private ExecuteWatchdog watchdog; + boolean running = false; + private int port = -1; + private final String interpreterDir; + private final String localRepoDir; + + private Map env; + + public RemoteInterpreterManagedProcess( + String intpRunner, + String intpDir, + String localRepoDir, + Map env, + int connectTimeout, + RemoteInterpreterProcessListener listener, + ApplicationEventListener appListener) { + super(new RemoteInterpreterEventPoller(listener, appListener), + connectTimeout); + this.interpreterRunner = intpRunner; + this.env = env; + this.interpreterDir = intpDir; + this.localRepoDir = localRepoDir; + } + + RemoteInterpreterManagedProcess(String intpRunner, + String intpDir, + String localRepoDir, + Map env, + RemoteInterpreterEventPoller remoteInterpreterEventPoller, + int connectTimeout) { + super(remoteInterpreterEventPoller, + connectTimeout); + this.interpreterRunner = intpRunner; + this.env = env; + this.interpreterDir = intpDir; + this.localRepoDir = localRepoDir; + } + + @Override + public String getHost() { + return "localhost"; + } + + @Override + public int getPort() { + return port; + } + + @Override + public void start() { + // start server process + try { + port = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); + } catch (IOException e1) { + throw new InterpreterException(e1); + } + + CommandLine cmdLine = CommandLine.parse(interpreterRunner); + cmdLine.addArgument("-d", false); + cmdLine.addArgument(interpreterDir, false); + cmdLine.addArgument("-p", false); + cmdLine.addArgument(Integer.toString(port), false); + cmdLine.addArgument("-l", false); + cmdLine.addArgument(localRepoDir, false); + + executor = new DefaultExecutor(); + + watchdog = new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT); + executor.setWatchdog(watchdog); + + try { + Map procEnv = EnvironmentUtils.getProcEnvironment(); + procEnv.putAll(env); + + logger.info("Run interpreter process {}", cmdLine); + executor.execute(cmdLine, procEnv, this); + running = true; + } catch (IOException e) { + running = false; + throw new InterpreterException(e); + } + + + long startTime = System.currentTimeMillis(); + while (System.currentTimeMillis() - startTime < getConnectTimeout()) { + if (RemoteInterpreterUtils.checkIfRemoteEndpointAccessible("localhost", port)) { + break; + } else { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + logger.error("Exception in RemoteInterpreterProcess while synchronized reference " + + "Thread.sleep", e); + } + } + } + } + + public void stop() { + if (isRunning()) { + logger.info("kill interpreter process"); + watchdog.destroyProcess(); + } + + executor = null; + watchdog = null; + running = false; + logger.info("Remote process terminated"); + } + + @Override + public void onProcessComplete(int exitValue) { + logger.info("Interpreter process exited {}", exitValue); + running = false; + + } + + @Override + public void onProcessFailed(ExecuteException e) { + logger.info("Interpreter process failed {}", e); + running = false; + } + + public boolean isRunning() { + return running; + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java index 05baf623809..aef6c2bd809 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java @@ -14,160 +14,72 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.zeppelin.interpreter.remote; import com.google.gson.Gson; import org.apache.commons.exec.*; -import org.apache.commons.exec.environment.EnvironmentUtils; import org.apache.commons.pool2.impl.GenericObjectPool; import org.apache.thrift.TException; +import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.Constants; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.Properties; /** - * + * Abstract class for interpreter process */ -public class RemoteInterpreterProcess implements ExecuteResultHandler { +public abstract class RemoteInterpreterProcess { private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterProcess.class); private final AtomicInteger referenceCount; - private DefaultExecutor executor; private ExecuteWatchdog watchdog; - boolean running = false; - private int port = -1; - private final String interpreterRunner; - private final String interpreterDir; - private final String localRepoDir; private GenericObjectPool clientPool; - private Map env; private final RemoteInterpreterEventPoller remoteInterpreterEventPoller; private final InterpreterContextRunnerPool interpreterContextRunnerPool; private int connectTimeout; String host = "localhost"; boolean isInterpreterAlreadyExecuting = false; - public RemoteInterpreterProcess(String intpRunner, - String intpDir, - String localRepoDir, - Map env, + public RemoteInterpreterProcess( int connectTimeout, - RemoteInterpreterProcessListener listener) { - this(intpRunner, - intpDir, - localRepoDir, - env, - new RemoteInterpreterEventPoller(listener), + RemoteInterpreterProcessListener listener, + ApplicationEventListener appListener) { + this(new RemoteInterpreterEventPoller(listener, appListener), connectTimeout); } - RemoteInterpreterProcess(String intpRunner, - String intpDir, - String localRepoDir, - Map env, - RemoteInterpreterEventPoller remoteInterpreterEventPoller, - int connectTimeout) { - this.interpreterRunner = intpRunner; - this.interpreterDir = intpDir; - this.localRepoDir = localRepoDir; - this.env = env; + RemoteInterpreterProcess(RemoteInterpreterEventPoller remoteInterpreterEventPoller, + int connectTimeout) { this.interpreterContextRunnerPool = new InterpreterContextRunnerPool(); referenceCount = new AtomicInteger(0); this.remoteInterpreterEventPoller = remoteInterpreterEventPoller; this.connectTimeout = connectTimeout; } + public abstract String getHost(); + public abstract int getPort(); + public abstract void start(); + public abstract void stop(); + public abstract boolean isRunning(); - public int getPort() { - return port; + public int getConnectTimeout() { + return connectTimeout; } public int reference(InterpreterGroup interpreterGroup) { synchronized (referenceCount) { - if (executor == null) { - if (interpreterGroup.containsKey(Constants.EXISTING_PROCESS)) { - Properties properties = interpreterGroup.getProperty(); - isInterpreterAlreadyExecuting = true; - if (isInterpreterAlreadyExecuting) { - if (properties.containsKey(Constants.ZEPPELIN_INTERPRETER_HOST)) { - host = properties.getProperty(Constants.ZEPPELIN_INTERPRETER_HOST); - - } else { - throw new InterpreterException("Can't find value for option Host." - + "Please specify the host on which interpreter is executing"); - } - if (properties.containsKey(Constants.ZEPPELIN_INTERPRETER_PORT)) { - port = Integer.parseInt( - interpreterGroup.getProperty().getProperty(Constants.ZEPPELIN_INTERPRETER_PORT)); - } else { - throw new InterpreterException("Can't find value for option Port." - + "Please specify the port on which interpreter is listening"); - } - } - running = true; - } - - if (!isInterpreterAlreadyExecuting) { - try { - port = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); - } catch (IOException e1) { - throw new InterpreterException(e1); - } - CommandLine cmdLine = CommandLine.parse(interpreterRunner); - cmdLine.addArgument("-d", false); - cmdLine.addArgument(interpreterDir, false); - cmdLine.addArgument("-p", false); - cmdLine.addArgument(Integer.toString(port), false); - cmdLine.addArgument("-l", false); - cmdLine.addArgument(localRepoDir, false); - - executor = new DefaultExecutor(); - - watchdog = new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT); - executor.setWatchdog(watchdog); - - running = true; - try { - Map procEnv = EnvironmentUtils.getProcEnvironment(); - procEnv.putAll(env); - - logger.info("Run interpreter process {}", cmdLine); - executor.execute(cmdLine, procEnv, this); - - } catch (IOException e) { - running = false; - throw new InterpreterException(e); - } - - } else { - logger.info( - "Not starting interpreter as \"isExistingProcess\" is enabled"); - } - - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() - startTime < connectTimeout) { - if (RemoteInterpreterUtils.checkIfRemoteEndpointAccessible(host, port)) { - break; - } else { - try { - Thread.sleep(500); - } catch (InterruptedException e) { - logger.error("Exception in RemoteInterpreterProcess while synchronized reference " - + "Thread.sleep", e); - } - } - } + if (!isRunning()) { + start(); + } - clientPool = new GenericObjectPool(new ClientFactory(host, port)); + if (clientPool == null) { + clientPool = new GenericObjectPool(new ClientFactory(getHost(), getPort())); + clientPool.setTestOnBorrow(true); remoteInterpreterEventPoller.setInterpreterGroup(interpreterGroup); remoteInterpreterEventPoller.setInterpreterProcess(this); @@ -250,16 +162,6 @@ public int dereference() { break; } } - - if (isRunning()) { - logger.info("kill interpreter process"); - watchdog.destroyProcess(); - } - - executor = null; - watchdog = null; - running = false; - logger.info("Remote process terminated"); } return r; } @@ -271,23 +173,6 @@ public int referenceCount() { } } - @Override - public void onProcessComplete(int exitValue) { - logger.info("Interpreter process exited {}", exitValue); - running = false; - - } - - @Override - public void onProcessFailed(ExecuteException e) { - logger.info("Interpreter process failed {}", e); - running = false; - } - - public boolean isRunning() { - return running; - } - public int getNumActiveClient() { if (clientPool == null) { return 0; diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java new file mode 100644 index 00000000000..42e6250e3ca --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.interpreter.remote; + +import org.apache.zeppelin.helium.ApplicationEventListener; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class connects to existing process + */ +public class RemoteInterpreterRunningProcess extends RemoteInterpreterProcess { + private final Logger logger = LoggerFactory.getLogger(RemoteInterpreterRunningProcess.class); + private final String host; + private final int port; + + public RemoteInterpreterRunningProcess( + int connectTimeout, + RemoteInterpreterProcessListener listener, + ApplicationEventListener appListener, + String host, + int port + ) { + super(connectTimeout, listener, appListener); + this.host = host; + this.port = port; + } + + @Override + public String getHost() { + return host; + } + + @Override + public int getPort() { + return port; + } + + @Override + public void start() { + // assume process is externally managed. nothing to do + } + + @Override + public void stop() { + // assume process is externally managed. nothing to do + } + + @Override + public boolean isRunning() { + return RemoteInterpreterUtils.checkIfRemoteEndpointAccessible(getHost(), getPort()); + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index 6b4edc4f5ae..173026562f3 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -29,7 +29,9 @@ import org.apache.thrift.server.TThreadPoolServer; import org.apache.thrift.transport.TServerSocket; import org.apache.thrift.transport.TTransportException; +import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.display.*; +import org.apache.zeppelin.helium.*; import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.thrift.*; @@ -58,6 +60,8 @@ public class RemoteInterpreterServer InterpreterGroup interpreterGroup; AngularObjectRegistry angularObjectRegistry; DistributedResourcePool resourcePool; + private ApplicationLoader appLoader; + Gson gson = new Gson(); RemoteInterpreterService.Processor processor; @@ -66,6 +70,10 @@ public class RemoteInterpreterServer private TThreadPoolServer server; RemoteInterpreterEventClient eventClient = new RemoteInterpreterEventClient(); + private DependencyResolver depLoader; + + private final Map runningApplications = + Collections.synchronizedMap(new HashMap()); public RemoteInterpreterServer(int port) throws TTransportException { this.port = port; @@ -84,6 +92,7 @@ public void run() { @Override public void shutdown() throws TException { + eventClient.waitForEventQueueBecomesEmpty(); if (interpreterGroup != null) { interpreterGroup.close(); interpreterGroup.destroy(); @@ -134,14 +143,17 @@ public static void main(String[] args) @Override public void createInterpreter(String interpreterGroupId, String noteId, String - className, - Map properties) throws TException { + className, Map properties) throws TException { if (interpreterGroup == null) { interpreterGroup = new InterpreterGroup(interpreterGroupId); angularObjectRegistry = new AngularObjectRegistry(interpreterGroup.getId(), this); resourcePool = new DistributedResourcePool(interpreterGroup.getId(), eventClient); interpreterGroup.setAngularObjectRegistry(angularObjectRegistry); interpreterGroup.setResourcePool(resourcePool); + + String localRepoPath = properties.get("zeppelin.interpreter.localRepo"); + depLoader = new DependencyResolver(localRepoPath); + appLoader = new ApplicationLoader(resourcePool, depLoader); } try { @@ -176,6 +188,18 @@ public void createInterpreter(String interpreterGroupId, String noteId, String } } + protected InterpreterGroup getInterpreterGroup() { + return interpreterGroup; + } + + protected ResourcePool getResourcePool() { + return resourcePool; + } + + protected RemoteInterpreterEventClient getEventClient() { + return eventClient; + } + private void setSystemProperty(Properties properties) { for (Object key : properties.keySet()) { if (!RemoteInterpreter.isEnvString((String) key)) { @@ -189,7 +213,7 @@ private void setSystemProperty(Properties properties) { } } - private Interpreter getInterpreter(String noteId, String className) throws TException { + protected Interpreter getInterpreter(String noteId, String className) throws TException { if (interpreterGroup == null) { throw new TException( new InterpreterException("Interpreter instance " + className + " not created")); @@ -218,6 +242,24 @@ public void open(String noteId, String className) throws TException { @Override public void close(String noteId, String className) throws TException { + // unload all applications + for (String appId : runningApplications.keySet()) { + RunningApplication appInfo = runningApplications.get(appId); + + // see NoteInterpreterLoader.SHARED_SESSION + if (appInfo.noteId.equals(noteId) || noteId.equals("shared_session")) { + try { + logger.info("Unload App {} ", appInfo.pkg.getName()); + appInfo.app.unload(); + // see ApplicationState.Status.UNLOADED + eventClient.onAppStatusUpdate(appInfo.noteId, appInfo.paragraphId, appId, "UNLOADED"); + } catch (ApplicationException e) { + logger.error(e.getMessage(), e); + } + } + } + + // close interpreters synchronized (interpreterGroup) { List interpreters = interpreterGroup.get(noteId); if (interpreters != null) { @@ -346,7 +388,6 @@ protected Object jobRun() throws Throwable { context.out.flush(); InterpreterResult.Type outputType = context.out.getType(); byte[] interpreterOutput = context.out.toByteArray(); - context.out.clear(); if (interpreterOutput != null && interpreterOutput.length > 0) { message = new String(interpreterOutput); @@ -363,11 +404,13 @@ protected Object jobRun() throws Throwable { } // put result into resource pool - context.getResourcePool().put( - context.getNoteId(), - context.getParagraphId(), - WellKnownResourceName.ParagraphResult.toString(), - combinedResult); + if (combinedResult.type() == InterpreterResult.Type.TABLE) { + context.getResourcePool().put( + context.getNoteId(), + context.getParagraphId(), + WellKnownResourceName.ZeppelinTableResult.toString(), + combinedResult); + } return combinedResult; } finally { InterpreterContext.remove(); @@ -392,7 +435,7 @@ public void cancel(String noteId, String className, RemoteInterpreterContext int if (job != null) { job.setStatus(Status.ABORT); } else { - intp.cancel(convert(interpreterContext)); + intp.cancel(convert(interpreterContext, null)); } } @@ -401,7 +444,7 @@ public int getProgress(String noteId, String className, RemoteInterpreterContext interpreterContext) throws TException { Interpreter intp = getInterpreter(noteId, className); - return intp.getProgress(convert(interpreterContext)); + return intp.getProgress(convert(interpreterContext, null)); } @@ -421,6 +464,10 @@ public List completion(String noteId, } private InterpreterContext convert(RemoteInterpreterContext ric) { + return convert(ric, createInterpreterOutput(ric.getNoteId(), ric.getParagraphId())); + } + + private InterpreterContext convert(RemoteInterpreterContext ric, InterpreterOutput output) { List contextRunners = new LinkedList(); List runners = gson.fromJson(ric.getRunners(), new TypeToken>() { @@ -441,11 +488,12 @@ private InterpreterContext convert(RemoteInterpreterContext ric) { gson.fromJson(ric.getGui(), GUI.class), interpreterGroup.getAngularObjectRegistry(), interpreterGroup.getResourcePool(), - contextRunners, createInterpreterOutput(ric.getNoteId(), ric.getParagraphId())); + contextRunners, output); } - private InterpreterOutput createInterpreterOutput(final String noteId, final String paragraphId) { + protected InterpreterOutput createInterpreterOutput(final String noteId, final String + paragraphId) { return new InterpreterOutput(new InterpreterOutputListener() { @Override public void onAppend(InterpreterOutput out, byte[] line) { @@ -660,9 +708,14 @@ public void resourceResponseGet(String resourceId, ByteBuffer object) throws TEx @Override public List resourcePoolGetAll() throws TException { logger.debug("Request getAll from ZeppelinServer"); + List result = new LinkedList(); + + if (resourcePool == null) { + return result; + } ResourceSet resourceSet = resourcePool.getAll(false); - List result = new LinkedList(); + Gson gson = new Gson(); for (Resource r : resourceSet) { @@ -708,4 +761,134 @@ public void angularRegistryPush(String registryAsString) throws TException { logger.info("Exception in RemoteInterpreterServer while angularRegistryPush, nolock", e); } } + + protected InterpreterOutput createAppOutput(final String noteId, + final String paragraphId, + final String appId) { + return new InterpreterOutput(new InterpreterOutputListener() { + @Override + public void onAppend(InterpreterOutput out, byte[] line) { + eventClient.onAppOutputAppend(noteId, paragraphId, appId, new String(line)); + } + + @Override + public void onUpdate(InterpreterOutput out, byte[] output) { + eventClient.onAppOutputUpdate(noteId, paragraphId, appId, new String(output)); + } + }); + } + + private ApplicationContext getApplicationContext( + HeliumPackage packageInfo, String noteId, String paragraphId, String applicationInstanceId) { + InterpreterOutput out = createAppOutput(noteId, paragraphId, applicationInstanceId); + return new ApplicationContext( + noteId, + paragraphId, + applicationInstanceId, + new HeliumAppAngularObjectRegistry(angularObjectRegistry, noteId, applicationInstanceId), + out); + } + + @Override + public RemoteApplicationResult loadApplication( + String applicationInstanceId, String packageInfo, String noteId, String paragraphId) + throws TException { + if (runningApplications.containsKey(applicationInstanceId)) { + logger.warn("Application instance {} is already running"); + return new RemoteApplicationResult(true, ""); + } + HeliumPackage pkgInfo = gson.fromJson(packageInfo, HeliumPackage.class); + ApplicationContext context = getApplicationContext( + pkgInfo, noteId, paragraphId, applicationInstanceId); + try { + Application app = null; + logger.info( + "Loading application {}({}), artifact={}, className={} into note={}, paragraph={}", + pkgInfo.getName(), + applicationInstanceId, + pkgInfo.getArtifact(), + pkgInfo.getClassName(), + noteId, + paragraphId); + app = appLoader.load(pkgInfo, context); + runningApplications.put( + applicationInstanceId, + new RunningApplication(pkgInfo, app, noteId, paragraphId)); + return new RemoteApplicationResult(true, ""); + } catch (Exception e) { + logger.error(e.getMessage(), e); + return new RemoteApplicationResult(false, e.getMessage()); + } + } + + @Override + public RemoteApplicationResult unloadApplication(String applicationInstanceId) + throws TException { + RunningApplication runningApplication = runningApplications.remove(applicationInstanceId); + if (runningApplication != null) { + try { + logger.info("Unloading application {}", applicationInstanceId); + runningApplication.app.unload(); + } catch (ApplicationException e) { + logger.error(e.getMessage(), e); + return new RemoteApplicationResult(false, e.getMessage()); + } + } + return new RemoteApplicationResult(true, ""); + } + + @Override + public RemoteApplicationResult runApplication(String applicationInstanceId) + throws TException { + logger.info("run application {}", applicationInstanceId); + + RunningApplication runningApp = runningApplications.get(applicationInstanceId); + if (runningApp == null) { + logger.error("Application instance {} not exists", applicationInstanceId); + return new RemoteApplicationResult(false, "Application instance does not exists"); + } else { + ApplicationContext context = runningApp.app.context(); + try { + context.out.clear(); + context.out.setType(InterpreterResult.Type.ANGULAR); + ResourceSet resource = appLoader.findRequiredResourceSet( + runningApp.pkg.getResources(), + context.getNoteId(), + context.getParagraphId()); + for (Resource res : resource) { + System.err.println("Resource " + res.get()); + } + runningApp.app.run(resource); + String output = new String(context.out.toByteArray()); + eventClient.onAppOutputUpdate( + context.getNoteId(), + context.getParagraphId(), + applicationInstanceId, + output); + return new RemoteApplicationResult(true, ""); + } catch (ApplicationException | IOException e) { + return new RemoteApplicationResult(false, e.getMessage()); + } + } + + + + } + + private static class RunningApplication { + public final Application app; + public final HeliumPackage pkg; + public final String noteId; + public final String paragraphId; + + public RunningApplication(HeliumPackage pkg, + Application app, + String noteId, + String paragraphId) { + this.app = app; + this.pkg = pkg; + this.noteId = noteId; + this.paragraphId = paragraphId; + } + }; } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteApplicationResult.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteApplicationResult.java new file mode 100644 index 00000000000..a192899daa1 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteApplicationResult.java @@ -0,0 +1,518 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Autogenerated by Thrift Compiler (0.9.2) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.zeppelin.interpreter.thrift; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import org.apache.thrift.async.AsyncMethodCallback; +import org.apache.thrift.server.AbstractNonblockingServer.*; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import javax.annotation.Generated; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) +@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-5-7") +public class RemoteApplicationResult implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteApplicationResult"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.BOOL, (short)1); + private static final org.apache.thrift.protocol.TField MSG_FIELD_DESC = new org.apache.thrift.protocol.TField("msg", org.apache.thrift.protocol.TType.STRING, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new RemoteApplicationResultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new RemoteApplicationResultTupleSchemeFactory()); + } + + public boolean success; // required + public String msg; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)1, "success"), + MSG((short)2, "msg"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // SUCCESS + return SUCCESS; + case 2: // MSG + return MSG; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __SUCCESS_ISSET_ID = 0; + private byte __isset_bitfield = 0; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); + tmpMap.put(_Fields.MSG, new org.apache.thrift.meta_data.FieldMetaData("msg", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(RemoteApplicationResult.class, metaDataMap); + } + + public RemoteApplicationResult() { + } + + public RemoteApplicationResult( + boolean success, + String msg) + { + this(); + this.success = success; + setSuccessIsSet(true); + this.msg = msg; + } + + /** + * Performs a deep copy on other. + */ + public RemoteApplicationResult(RemoteApplicationResult other) { + __isset_bitfield = other.__isset_bitfield; + this.success = other.success; + if (other.isSetMsg()) { + this.msg = other.msg; + } + } + + public RemoteApplicationResult deepCopy() { + return new RemoteApplicationResult(this); + } + + @Override + public void clear() { + setSuccessIsSet(false); + this.success = false; + this.msg = null; + } + + public boolean isSuccess() { + return this.success; + } + + public RemoteApplicationResult setSuccess(boolean success) { + this.success = success; + setSuccessIsSet(true); + return this; + } + + public void unsetSuccess() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __SUCCESS_ISSET_ID); + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return EncodingUtils.testBit(__isset_bitfield, __SUCCESS_ISSET_ID); + } + + public void setSuccessIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __SUCCESS_ISSET_ID, value); + } + + public String getMsg() { + return this.msg; + } + + public RemoteApplicationResult setMsg(String msg) { + this.msg = msg; + return this; + } + + public void unsetMsg() { + this.msg = null; + } + + /** Returns true if field msg is set (has been assigned a value) and false otherwise */ + public boolean isSetMsg() { + return this.msg != null; + } + + public void setMsgIsSet(boolean value) { + if (!value) { + this.msg = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((Boolean)value); + } + break; + + case MSG: + if (value == null) { + unsetMsg(); + } else { + setMsg((String)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return Boolean.valueOf(isSuccess()); + + case MSG: + return getMsg(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case MSG: + return isSetMsg(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof RemoteApplicationResult) + return this.equals((RemoteApplicationResult)that); + return false; + } + + public boolean equals(RemoteApplicationResult that) { + if (that == null) + return false; + + boolean this_present_success = true; + boolean that_present_success = true; + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (this.success != that.success) + return false; + } + + boolean this_present_msg = true && this.isSetMsg(); + boolean that_present_msg = true && that.isSetMsg(); + if (this_present_msg || that_present_msg) { + if (!(this_present_msg && that_present_msg)) + return false; + if (!this.msg.equals(that.msg)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_success = true; + list.add(present_success); + if (present_success) + list.add(success); + + boolean present_msg = true && (isSetMsg()); + list.add(present_msg); + if (present_msg) + list.add(msg); + + return list.hashCode(); + } + + @Override + public int compareTo(RemoteApplicationResult other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMsg()).compareTo(other.isSetMsg()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMsg()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.msg, other.msg); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("RemoteApplicationResult("); + boolean first = true; + + sb.append("success:"); + sb.append(this.success); + first = false; + if (!first) sb.append(", "); + sb.append("msg:"); + if (this.msg == null) { + sb.append("null"); + } else { + sb.append(this.msg); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class RemoteApplicationResultStandardSchemeFactory implements SchemeFactory { + public RemoteApplicationResultStandardScheme getScheme() { + return new RemoteApplicationResultStandardScheme(); + } + } + + private static class RemoteApplicationResultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, RemoteApplicationResult struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.success = iprot.readBool(); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // MSG + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.msg = iprot.readString(); + struct.setMsgIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, RemoteApplicationResult struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + oprot.writeBool(struct.success); + oprot.writeFieldEnd(); + if (struct.msg != null) { + oprot.writeFieldBegin(MSG_FIELD_DESC); + oprot.writeString(struct.msg); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class RemoteApplicationResultTupleSchemeFactory implements SchemeFactory { + public RemoteApplicationResultTupleScheme getScheme() { + return new RemoteApplicationResultTupleScheme(); + } + } + + private static class RemoteApplicationResultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, RemoteApplicationResult struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetMsg()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + oprot.writeBool(struct.success); + } + if (struct.isSetMsg()) { + oprot.writeString(struct.msg); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, RemoteApplicationResult struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.success = iprot.readBool(); + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.msg = iprot.readString(); + struct.setMsgIsSet(true); + } + } + } + +} + diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java index 66631d2f2a8..955461951f3 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java @@ -38,7 +38,8 @@ public enum RemoteInterpreterEventType implements org.apache.thrift.TEnum { RESOURCE_GET(7), OUTPUT_APPEND(8), OUTPUT_UPDATE(9), - ANGULAR_REGISTRY_PUSH(10); + ANGULAR_REGISTRY_PUSH(10), + APP_STATUS_UPDATE(11); private final int value; @@ -79,6 +80,8 @@ public static RemoteInterpreterEventType findByValue(int value) { return OUTPUT_UPDATE; case 10: return ANGULAR_REGISTRY_PUSH; + case 11: + return APP_STATUS_UPDATE; default: return null; } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java index 0c1385f5522..e3ddeebc664 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java @@ -96,6 +96,12 @@ public interface Iface { public void angularRegistryPush(String registry) throws org.apache.thrift.TException; + public RemoteApplicationResult loadApplication(String applicationInstanceId, String packageInfo, String noteId, String paragraphId) throws org.apache.thrift.TException; + + public RemoteApplicationResult unloadApplication(String applicationInstanceId) throws org.apache.thrift.TException; + + public RemoteApplicationResult runApplication(String applicationInstanceId) throws org.apache.thrift.TException; + } public interface AsyncIface { @@ -140,6 +146,12 @@ public interface AsyncIface { public void angularRegistryPush(String registry, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + public void loadApplication(String applicationInstanceId, String packageInfo, String noteId, String paragraphId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void unloadApplication(String applicationInstanceId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void runApplication(String applicationInstanceId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + } public static class Client extends org.apache.thrift.TServiceClient implements Iface { @@ -616,6 +628,78 @@ public void recv_angularRegistryPush() throws org.apache.thrift.TException return; } + public RemoteApplicationResult loadApplication(String applicationInstanceId, String packageInfo, String noteId, String paragraphId) throws org.apache.thrift.TException + { + send_loadApplication(applicationInstanceId, packageInfo, noteId, paragraphId); + return recv_loadApplication(); + } + + public void send_loadApplication(String applicationInstanceId, String packageInfo, String noteId, String paragraphId) throws org.apache.thrift.TException + { + loadApplication_args args = new loadApplication_args(); + args.setApplicationInstanceId(applicationInstanceId); + args.setPackageInfo(packageInfo); + args.setNoteId(noteId); + args.setParagraphId(paragraphId); + sendBase("loadApplication", args); + } + + public RemoteApplicationResult recv_loadApplication() throws org.apache.thrift.TException + { + loadApplication_result result = new loadApplication_result(); + receiveBase(result, "loadApplication"); + if (result.isSetSuccess()) { + return result.success; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "loadApplication failed: unknown result"); + } + + public RemoteApplicationResult unloadApplication(String applicationInstanceId) throws org.apache.thrift.TException + { + send_unloadApplication(applicationInstanceId); + return recv_unloadApplication(); + } + + public void send_unloadApplication(String applicationInstanceId) throws org.apache.thrift.TException + { + unloadApplication_args args = new unloadApplication_args(); + args.setApplicationInstanceId(applicationInstanceId); + sendBase("unloadApplication", args); + } + + public RemoteApplicationResult recv_unloadApplication() throws org.apache.thrift.TException + { + unloadApplication_result result = new unloadApplication_result(); + receiveBase(result, "unloadApplication"); + if (result.isSetSuccess()) { + return result.success; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "unloadApplication failed: unknown result"); + } + + public RemoteApplicationResult runApplication(String applicationInstanceId) throws org.apache.thrift.TException + { + send_runApplication(applicationInstanceId); + return recv_runApplication(); + } + + public void send_runApplication(String applicationInstanceId) throws org.apache.thrift.TException + { + runApplication_args args = new runApplication_args(); + args.setApplicationInstanceId(applicationInstanceId); + sendBase("runApplication", args); + } + + public RemoteApplicationResult recv_runApplication() throws org.apache.thrift.TException + { + runApplication_result result = new runApplication_result(); + receiveBase(result, "runApplication"); + if (result.isSetSuccess()) { + return result.success; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "runApplication failed: unknown result"); + } + } public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface { public static class Factory implements org.apache.thrift.async.TAsyncClientFactory { @@ -1355,6 +1439,111 @@ public void getResult() throws org.apache.thrift.TException { } } + public void loadApplication(String applicationInstanceId, String packageInfo, String noteId, String paragraphId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + loadApplication_call method_call = new loadApplication_call(applicationInstanceId, packageInfo, noteId, paragraphId, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class loadApplication_call extends org.apache.thrift.async.TAsyncMethodCall { + private String applicationInstanceId; + private String packageInfo; + private String noteId; + private String paragraphId; + public loadApplication_call(String applicationInstanceId, String packageInfo, String noteId, String paragraphId, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.applicationInstanceId = applicationInstanceId; + this.packageInfo = packageInfo; + this.noteId = noteId; + this.paragraphId = paragraphId; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("loadApplication", org.apache.thrift.protocol.TMessageType.CALL, 0)); + loadApplication_args args = new loadApplication_args(); + args.setApplicationInstanceId(applicationInstanceId); + args.setPackageInfo(packageInfo); + args.setNoteId(noteId); + args.setParagraphId(paragraphId); + args.write(prot); + prot.writeMessageEnd(); + } + + public RemoteApplicationResult getResult() throws org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_loadApplication(); + } + } + + public void unloadApplication(String applicationInstanceId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + unloadApplication_call method_call = new unloadApplication_call(applicationInstanceId, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class unloadApplication_call extends org.apache.thrift.async.TAsyncMethodCall { + private String applicationInstanceId; + public unloadApplication_call(String applicationInstanceId, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.applicationInstanceId = applicationInstanceId; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("unloadApplication", org.apache.thrift.protocol.TMessageType.CALL, 0)); + unloadApplication_args args = new unloadApplication_args(); + args.setApplicationInstanceId(applicationInstanceId); + args.write(prot); + prot.writeMessageEnd(); + } + + public RemoteApplicationResult getResult() throws org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_unloadApplication(); + } + } + + public void runApplication(String applicationInstanceId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + runApplication_call method_call = new runApplication_call(applicationInstanceId, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class runApplication_call extends org.apache.thrift.async.TAsyncMethodCall { + private String applicationInstanceId; + public runApplication_call(String applicationInstanceId, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.applicationInstanceId = applicationInstanceId; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("runApplication", org.apache.thrift.protocol.TMessageType.CALL, 0)); + runApplication_args args = new runApplication_args(); + args.setApplicationInstanceId(applicationInstanceId); + args.write(prot); + prot.writeMessageEnd(); + } + + public RemoteApplicationResult getResult() throws org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_runApplication(); + } + } + } public static class Processor extends org.apache.thrift.TBaseProcessor implements org.apache.thrift.TProcessor { @@ -1388,6 +1577,9 @@ protected Processor(I iface, Map extends org.apache.thrift.ProcessFunction { + public loadApplication() { + super("loadApplication"); + } + + public loadApplication_args getEmptyArgsInstance() { + return new loadApplication_args(); + } + + protected boolean isOneway() { + return false; + } + + public loadApplication_result getResult(I iface, loadApplication_args args) throws org.apache.thrift.TException { + loadApplication_result result = new loadApplication_result(); + result.success = iface.loadApplication(args.applicationInstanceId, args.packageInfo, args.noteId, args.paragraphId); + return result; + } + } + + public static class unloadApplication extends org.apache.thrift.ProcessFunction { + public unloadApplication() { + super("unloadApplication"); + } + + public unloadApplication_args getEmptyArgsInstance() { + return new unloadApplication_args(); + } + + protected boolean isOneway() { + return false; + } + + public unloadApplication_result getResult(I iface, unloadApplication_args args) throws org.apache.thrift.TException { + unloadApplication_result result = new unloadApplication_result(); + result.success = iface.unloadApplication(args.applicationInstanceId); + return result; + } + } + + public static class runApplication extends org.apache.thrift.ProcessFunction { + public runApplication() { + super("runApplication"); + } + + public runApplication_args getEmptyArgsInstance() { + return new runApplication_args(); + } + + protected boolean isOneway() { + return false; + } + + public runApplication_result getResult(I iface, runApplication_args args) throws org.apache.thrift.TException { + runApplication_result result = new runApplication_result(); + result.success = iface.runApplication(args.applicationInstanceId); + return result; + } + } + } public static class AsyncProcessor extends org.apache.thrift.TBaseAsyncProcessor { @@ -1826,6 +2078,9 @@ protected AsyncProcessor(I iface, Map, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("createInterpreter_args"); - - private static final org.apache.thrift.protocol.TField INTP_GROUP_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("intpGroupId", org.apache.thrift.protocol.TType.STRING, (short)1); - private static final org.apache.thrift.protocol.TField NOTE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("noteId", org.apache.thrift.protocol.TType.STRING, (short)2); - private static final org.apache.thrift.protocol.TField CLASS_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("className", org.apache.thrift.protocol.TType.STRING, (short)3); - private static final org.apache.thrift.protocol.TField PROPERTIES_FIELD_DESC = new org.apache.thrift.protocol.TField("properties", org.apache.thrift.protocol.TType.MAP, (short)4); - - private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); - static { - schemes.put(StandardScheme.class, new createInterpreter_argsStandardSchemeFactory()); - schemes.put(TupleScheme.class, new createInterpreter_argsTupleSchemeFactory()); - } - - public String intpGroupId; // required - public String noteId; // required - public String className; // required - public Map properties; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - INTP_GROUP_ID((short)1, "intpGroupId"), - NOTE_ID((short)2, "noteId"), - CLASS_NAME((short)3, "className"), - PROPERTIES((short)4, "properties"); - - private static final Map byName = new HashMap(); - - static { - for (_Fields field : EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } + public static class loadApplication extends org.apache.thrift.AsyncProcessFunction { + public loadApplication() { + super("loadApplication"); } - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - public static _Fields findByThriftId(int fieldId) { - switch(fieldId) { - case 1: // INTP_GROUP_ID - return INTP_GROUP_ID; - case 2: // NOTE_ID - return NOTE_ID; - case 3: // CLASS_NAME - return CLASS_NAME; - case 4: // PROPERTIES - return PROPERTIES; - default: - return null; - } + public loadApplication_args getEmptyArgsInstance() { + return new loadApplication_args(); } - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; + public AsyncMethodCallback getResultHandler(final AsyncFrameBuffer fb, final int seqid) { + final org.apache.thrift.AsyncProcessFunction fcall = this; + return new AsyncMethodCallback() { + public void onComplete(RemoteApplicationResult o) { + loadApplication_result result = new loadApplication_result(); + result.success = o; + try { + fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid); + return; + } catch (Exception e) { + LOGGER.error("Exception writing to internal frame buffer", e); + } + fb.close(); + } + public void onError(Exception e) { + byte msgType = org.apache.thrift.protocol.TMessageType.REPLY; + org.apache.thrift.TBase msg; + loadApplication_result result = new loadApplication_result(); + { + msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; + msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage()); + } + try { + fcall.sendResponse(fb,msg,msgType,seqid); + return; + } catch (Exception ex) { + LOGGER.error("Exception writing to internal frame buffer", ex); + } + fb.close(); + } + }; } - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - public static _Fields findByName(String name) { - return byName.get(name); + protected boolean isOneway() { + return false; } - private final short _thriftId; - private final String _fieldName; - - _Fields(short thriftId, String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; + public void start(I iface, loadApplication_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws TException { + iface.loadApplication(args.applicationInstanceId, args.packageInfo, args.noteId, args.paragraphId,resultHandler); } + } - public short getThriftFieldId() { - return _thriftId; + public static class unloadApplication extends org.apache.thrift.AsyncProcessFunction { + public unloadApplication() { + super("unloadApplication"); } - public String getFieldName() { - return _fieldName; + public unloadApplication_args getEmptyArgsInstance() { + return new unloadApplication_args(); } - } - // isset id assignments - public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - static { - Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + public AsyncMethodCallback getResultHandler(final AsyncFrameBuffer fb, final int seqid) { + final org.apache.thrift.AsyncProcessFunction fcall = this; + return new AsyncMethodCallback() { + public void onComplete(RemoteApplicationResult o) { + unloadApplication_result result = new unloadApplication_result(); + result.success = o; + try { + fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid); + return; + } catch (Exception e) { + LOGGER.error("Exception writing to internal frame buffer", e); + } + fb.close(); + } + public void onError(Exception e) { + byte msgType = org.apache.thrift.protocol.TMessageType.REPLY; + org.apache.thrift.TBase msg; + unloadApplication_result result = new unloadApplication_result(); + { + msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; + msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage()); + } + try { + fcall.sendResponse(fb,msg,msgType,seqid); + return; + } catch (Exception ex) { + LOGGER.error("Exception writing to internal frame buffer", ex); + } + fb.close(); + } + }; + } + + protected boolean isOneway() { + return false; + } + + public void start(I iface, unloadApplication_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws TException { + iface.unloadApplication(args.applicationInstanceId,resultHandler); + } + } + + public static class runApplication extends org.apache.thrift.AsyncProcessFunction { + public runApplication() { + super("runApplication"); + } + + public runApplication_args getEmptyArgsInstance() { + return new runApplication_args(); + } + + public AsyncMethodCallback getResultHandler(final AsyncFrameBuffer fb, final int seqid) { + final org.apache.thrift.AsyncProcessFunction fcall = this; + return new AsyncMethodCallback() { + public void onComplete(RemoteApplicationResult o) { + runApplication_result result = new runApplication_result(); + result.success = o; + try { + fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid); + return; + } catch (Exception e) { + LOGGER.error("Exception writing to internal frame buffer", e); + } + fb.close(); + } + public void onError(Exception e) { + byte msgType = org.apache.thrift.protocol.TMessageType.REPLY; + org.apache.thrift.TBase msg; + runApplication_result result = new runApplication_result(); + { + msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; + msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage()); + } + try { + fcall.sendResponse(fb,msg,msgType,seqid); + return; + } catch (Exception ex) { + LOGGER.error("Exception writing to internal frame buffer", ex); + } + fb.close(); + } + }; + } + + protected boolean isOneway() { + return false; + } + + public void start(I iface, runApplication_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws TException { + iface.runApplication(args.applicationInstanceId,resultHandler); + } + } + + } + + public static class createInterpreter_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("createInterpreter_args"); + + private static final org.apache.thrift.protocol.TField INTP_GROUP_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("intpGroupId", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField NOTE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("noteId", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField CLASS_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("className", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField PROPERTIES_FIELD_DESC = new org.apache.thrift.protocol.TField("properties", org.apache.thrift.protocol.TType.MAP, (short)4); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new createInterpreter_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new createInterpreter_argsTupleSchemeFactory()); + } + + public String intpGroupId; // required + public String noteId; // required + public String className; // required + public Map properties; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + INTP_GROUP_ID((short)1, "intpGroupId"), + NOTE_ID((short)2, "noteId"), + CLASS_NAME((short)3, "className"), + PROPERTIES((short)4, "properties"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // INTP_GROUP_ID + return INTP_GROUP_ID; + case 2: // NOTE_ID + return NOTE_ID; + case 3: // CLASS_NAME + return CLASS_NAME; + case 4: // PROPERTIES + return PROPERTIES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); tmpMap.put(_Fields.INTP_GROUP_ID, new org.apache.thrift.meta_data.FieldMetaData("intpGroupId", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); tmpMap.put(_Fields.NOTE_ID, new org.apache.thrift.meta_data.FieldMetaData("noteId", org.apache.thrift.TFieldRequirementType.DEFAULT, @@ -19097,4 +19505,2500 @@ public void read(org.apache.thrift.protocol.TProtocol prot, angularRegistryPush_ } + public static class loadApplication_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("loadApplication_args"); + + private static final org.apache.thrift.protocol.TField APPLICATION_INSTANCE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("applicationInstanceId", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField PACKAGE_INFO_FIELD_DESC = new org.apache.thrift.protocol.TField("packageInfo", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField NOTE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("noteId", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField PARAGRAPH_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("paragraphId", org.apache.thrift.protocol.TType.STRING, (short)4); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new loadApplication_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new loadApplication_argsTupleSchemeFactory()); + } + + public String applicationInstanceId; // required + public String packageInfo; // required + public String noteId; // required + public String paragraphId; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + APPLICATION_INSTANCE_ID((short)1, "applicationInstanceId"), + PACKAGE_INFO((short)2, "packageInfo"), + NOTE_ID((short)3, "noteId"), + PARAGRAPH_ID((short)4, "paragraphId"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // APPLICATION_INSTANCE_ID + return APPLICATION_INSTANCE_ID; + case 2: // PACKAGE_INFO + return PACKAGE_INFO; + case 3: // NOTE_ID + return NOTE_ID; + case 4: // PARAGRAPH_ID + return PARAGRAPH_ID; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.APPLICATION_INSTANCE_ID, new org.apache.thrift.meta_data.FieldMetaData("applicationInstanceId", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.PACKAGE_INFO, new org.apache.thrift.meta_data.FieldMetaData("packageInfo", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.NOTE_ID, new org.apache.thrift.meta_data.FieldMetaData("noteId", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.PARAGRAPH_ID, new org.apache.thrift.meta_data.FieldMetaData("paragraphId", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(loadApplication_args.class, metaDataMap); + } + + public loadApplication_args() { + } + + public loadApplication_args( + String applicationInstanceId, + String packageInfo, + String noteId, + String paragraphId) + { + this(); + this.applicationInstanceId = applicationInstanceId; + this.packageInfo = packageInfo; + this.noteId = noteId; + this.paragraphId = paragraphId; + } + + /** + * Performs a deep copy on other. + */ + public loadApplication_args(loadApplication_args other) { + if (other.isSetApplicationInstanceId()) { + this.applicationInstanceId = other.applicationInstanceId; + } + if (other.isSetPackageInfo()) { + this.packageInfo = other.packageInfo; + } + if (other.isSetNoteId()) { + this.noteId = other.noteId; + } + if (other.isSetParagraphId()) { + this.paragraphId = other.paragraphId; + } + } + + public loadApplication_args deepCopy() { + return new loadApplication_args(this); + } + + @Override + public void clear() { + this.applicationInstanceId = null; + this.packageInfo = null; + this.noteId = null; + this.paragraphId = null; + } + + public String getApplicationInstanceId() { + return this.applicationInstanceId; + } + + public loadApplication_args setApplicationInstanceId(String applicationInstanceId) { + this.applicationInstanceId = applicationInstanceId; + return this; + } + + public void unsetApplicationInstanceId() { + this.applicationInstanceId = null; + } + + /** Returns true if field applicationInstanceId is set (has been assigned a value) and false otherwise */ + public boolean isSetApplicationInstanceId() { + return this.applicationInstanceId != null; + } + + public void setApplicationInstanceIdIsSet(boolean value) { + if (!value) { + this.applicationInstanceId = null; + } + } + + public String getPackageInfo() { + return this.packageInfo; + } + + public loadApplication_args setPackageInfo(String packageInfo) { + this.packageInfo = packageInfo; + return this; + } + + public void unsetPackageInfo() { + this.packageInfo = null; + } + + /** Returns true if field packageInfo is set (has been assigned a value) and false otherwise */ + public boolean isSetPackageInfo() { + return this.packageInfo != null; + } + + public void setPackageInfoIsSet(boolean value) { + if (!value) { + this.packageInfo = null; + } + } + + public String getNoteId() { + return this.noteId; + } + + public loadApplication_args setNoteId(String noteId) { + this.noteId = noteId; + return this; + } + + public void unsetNoteId() { + this.noteId = null; + } + + /** Returns true if field noteId is set (has been assigned a value) and false otherwise */ + public boolean isSetNoteId() { + return this.noteId != null; + } + + public void setNoteIdIsSet(boolean value) { + if (!value) { + this.noteId = null; + } + } + + public String getParagraphId() { + return this.paragraphId; + } + + public loadApplication_args setParagraphId(String paragraphId) { + this.paragraphId = paragraphId; + return this; + } + + public void unsetParagraphId() { + this.paragraphId = null; + } + + /** Returns true if field paragraphId is set (has been assigned a value) and false otherwise */ + public boolean isSetParagraphId() { + return this.paragraphId != null; + } + + public void setParagraphIdIsSet(boolean value) { + if (!value) { + this.paragraphId = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case APPLICATION_INSTANCE_ID: + if (value == null) { + unsetApplicationInstanceId(); + } else { + setApplicationInstanceId((String)value); + } + break; + + case PACKAGE_INFO: + if (value == null) { + unsetPackageInfo(); + } else { + setPackageInfo((String)value); + } + break; + + case NOTE_ID: + if (value == null) { + unsetNoteId(); + } else { + setNoteId((String)value); + } + break; + + case PARAGRAPH_ID: + if (value == null) { + unsetParagraphId(); + } else { + setParagraphId((String)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case APPLICATION_INSTANCE_ID: + return getApplicationInstanceId(); + + case PACKAGE_INFO: + return getPackageInfo(); + + case NOTE_ID: + return getNoteId(); + + case PARAGRAPH_ID: + return getParagraphId(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case APPLICATION_INSTANCE_ID: + return isSetApplicationInstanceId(); + case PACKAGE_INFO: + return isSetPackageInfo(); + case NOTE_ID: + return isSetNoteId(); + case PARAGRAPH_ID: + return isSetParagraphId(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof loadApplication_args) + return this.equals((loadApplication_args)that); + return false; + } + + public boolean equals(loadApplication_args that) { + if (that == null) + return false; + + boolean this_present_applicationInstanceId = true && this.isSetApplicationInstanceId(); + boolean that_present_applicationInstanceId = true && that.isSetApplicationInstanceId(); + if (this_present_applicationInstanceId || that_present_applicationInstanceId) { + if (!(this_present_applicationInstanceId && that_present_applicationInstanceId)) + return false; + if (!this.applicationInstanceId.equals(that.applicationInstanceId)) + return false; + } + + boolean this_present_packageInfo = true && this.isSetPackageInfo(); + boolean that_present_packageInfo = true && that.isSetPackageInfo(); + if (this_present_packageInfo || that_present_packageInfo) { + if (!(this_present_packageInfo && that_present_packageInfo)) + return false; + if (!this.packageInfo.equals(that.packageInfo)) + return false; + } + + boolean this_present_noteId = true && this.isSetNoteId(); + boolean that_present_noteId = true && that.isSetNoteId(); + if (this_present_noteId || that_present_noteId) { + if (!(this_present_noteId && that_present_noteId)) + return false; + if (!this.noteId.equals(that.noteId)) + return false; + } + + boolean this_present_paragraphId = true && this.isSetParagraphId(); + boolean that_present_paragraphId = true && that.isSetParagraphId(); + if (this_present_paragraphId || that_present_paragraphId) { + if (!(this_present_paragraphId && that_present_paragraphId)) + return false; + if (!this.paragraphId.equals(that.paragraphId)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_applicationInstanceId = true && (isSetApplicationInstanceId()); + list.add(present_applicationInstanceId); + if (present_applicationInstanceId) + list.add(applicationInstanceId); + + boolean present_packageInfo = true && (isSetPackageInfo()); + list.add(present_packageInfo); + if (present_packageInfo) + list.add(packageInfo); + + boolean present_noteId = true && (isSetNoteId()); + list.add(present_noteId); + if (present_noteId) + list.add(noteId); + + boolean present_paragraphId = true && (isSetParagraphId()); + list.add(present_paragraphId); + if (present_paragraphId) + list.add(paragraphId); + + return list.hashCode(); + } + + @Override + public int compareTo(loadApplication_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetApplicationInstanceId()).compareTo(other.isSetApplicationInstanceId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetApplicationInstanceId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.applicationInstanceId, other.applicationInstanceId); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetPackageInfo()).compareTo(other.isSetPackageInfo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetPackageInfo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.packageInfo, other.packageInfo); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetNoteId()).compareTo(other.isSetNoteId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetNoteId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.noteId, other.noteId); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetParagraphId()).compareTo(other.isSetParagraphId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetParagraphId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.paragraphId, other.paragraphId); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("loadApplication_args("); + boolean first = true; + + sb.append("applicationInstanceId:"); + if (this.applicationInstanceId == null) { + sb.append("null"); + } else { + sb.append(this.applicationInstanceId); + } + first = false; + if (!first) sb.append(", "); + sb.append("packageInfo:"); + if (this.packageInfo == null) { + sb.append("null"); + } else { + sb.append(this.packageInfo); + } + first = false; + if (!first) sb.append(", "); + sb.append("noteId:"); + if (this.noteId == null) { + sb.append("null"); + } else { + sb.append(this.noteId); + } + first = false; + if (!first) sb.append(", "); + sb.append("paragraphId:"); + if (this.paragraphId == null) { + sb.append("null"); + } else { + sb.append(this.paragraphId); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class loadApplication_argsStandardSchemeFactory implements SchemeFactory { + public loadApplication_argsStandardScheme getScheme() { + return new loadApplication_argsStandardScheme(); + } + } + + private static class loadApplication_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, loadApplication_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // APPLICATION_INSTANCE_ID + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.applicationInstanceId = iprot.readString(); + struct.setApplicationInstanceIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // PACKAGE_INFO + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.packageInfo = iprot.readString(); + struct.setPackageInfoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // NOTE_ID + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.noteId = iprot.readString(); + struct.setNoteIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // PARAGRAPH_ID + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.paragraphId = iprot.readString(); + struct.setParagraphIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, loadApplication_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.applicationInstanceId != null) { + oprot.writeFieldBegin(APPLICATION_INSTANCE_ID_FIELD_DESC); + oprot.writeString(struct.applicationInstanceId); + oprot.writeFieldEnd(); + } + if (struct.packageInfo != null) { + oprot.writeFieldBegin(PACKAGE_INFO_FIELD_DESC); + oprot.writeString(struct.packageInfo); + oprot.writeFieldEnd(); + } + if (struct.noteId != null) { + oprot.writeFieldBegin(NOTE_ID_FIELD_DESC); + oprot.writeString(struct.noteId); + oprot.writeFieldEnd(); + } + if (struct.paragraphId != null) { + oprot.writeFieldBegin(PARAGRAPH_ID_FIELD_DESC); + oprot.writeString(struct.paragraphId); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class loadApplication_argsTupleSchemeFactory implements SchemeFactory { + public loadApplication_argsTupleScheme getScheme() { + return new loadApplication_argsTupleScheme(); + } + } + + private static class loadApplication_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, loadApplication_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetApplicationInstanceId()) { + optionals.set(0); + } + if (struct.isSetPackageInfo()) { + optionals.set(1); + } + if (struct.isSetNoteId()) { + optionals.set(2); + } + if (struct.isSetParagraphId()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetApplicationInstanceId()) { + oprot.writeString(struct.applicationInstanceId); + } + if (struct.isSetPackageInfo()) { + oprot.writeString(struct.packageInfo); + } + if (struct.isSetNoteId()) { + oprot.writeString(struct.noteId); + } + if (struct.isSetParagraphId()) { + oprot.writeString(struct.paragraphId); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, loadApplication_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(4); + if (incoming.get(0)) { + struct.applicationInstanceId = iprot.readString(); + struct.setApplicationInstanceIdIsSet(true); + } + if (incoming.get(1)) { + struct.packageInfo = iprot.readString(); + struct.setPackageInfoIsSet(true); + } + if (incoming.get(2)) { + struct.noteId = iprot.readString(); + struct.setNoteIdIsSet(true); + } + if (incoming.get(3)) { + struct.paragraphId = iprot.readString(); + struct.setParagraphIdIsSet(true); + } + } + } + + } + + public static class loadApplication_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("loadApplication_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new loadApplication_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new loadApplication_resultTupleSchemeFactory()); + } + + public RemoteApplicationResult success; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, RemoteApplicationResult.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(loadApplication_result.class, metaDataMap); + } + + public loadApplication_result() { + } + + public loadApplication_result( + RemoteApplicationResult success) + { + this(); + this.success = success; + } + + /** + * Performs a deep copy on other. + */ + public loadApplication_result(loadApplication_result other) { + if (other.isSetSuccess()) { + this.success = new RemoteApplicationResult(other.success); + } + } + + public loadApplication_result deepCopy() { + return new loadApplication_result(this); + } + + @Override + public void clear() { + this.success = null; + } + + public RemoteApplicationResult getSuccess() { + return this.success; + } + + public loadApplication_result setSuccess(RemoteApplicationResult success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((RemoteApplicationResult)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof loadApplication_result) + return this.equals((loadApplication_result)that); + return false; + } + + public boolean equals(loadApplication_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_success = true && (isSetSuccess()); + list.add(present_success); + if (present_success) + list.add(success); + + return list.hashCode(); + } + + @Override + public int compareTo(loadApplication_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("loadApplication_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + if (success != null) { + success.validate(); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class loadApplication_resultStandardSchemeFactory implements SchemeFactory { + public loadApplication_resultStandardScheme getScheme() { + return new loadApplication_resultStandardScheme(); + } + } + + private static class loadApplication_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, loadApplication_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.success = new RemoteApplicationResult(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, loadApplication_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + struct.success.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class loadApplication_resultTupleSchemeFactory implements SchemeFactory { + public loadApplication_resultTupleScheme getScheme() { + return new loadApplication_resultTupleScheme(); + } + } + + private static class loadApplication_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, loadApplication_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetSuccess()) { + struct.success.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, loadApplication_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.success = new RemoteApplicationResult(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } + } + } + + } + + public static class unloadApplication_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("unloadApplication_args"); + + private static final org.apache.thrift.protocol.TField APPLICATION_INSTANCE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("applicationInstanceId", org.apache.thrift.protocol.TType.STRING, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new unloadApplication_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new unloadApplication_argsTupleSchemeFactory()); + } + + public String applicationInstanceId; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + APPLICATION_INSTANCE_ID((short)1, "applicationInstanceId"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // APPLICATION_INSTANCE_ID + return APPLICATION_INSTANCE_ID; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.APPLICATION_INSTANCE_ID, new org.apache.thrift.meta_data.FieldMetaData("applicationInstanceId", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(unloadApplication_args.class, metaDataMap); + } + + public unloadApplication_args() { + } + + public unloadApplication_args( + String applicationInstanceId) + { + this(); + this.applicationInstanceId = applicationInstanceId; + } + + /** + * Performs a deep copy on other. + */ + public unloadApplication_args(unloadApplication_args other) { + if (other.isSetApplicationInstanceId()) { + this.applicationInstanceId = other.applicationInstanceId; + } + } + + public unloadApplication_args deepCopy() { + return new unloadApplication_args(this); + } + + @Override + public void clear() { + this.applicationInstanceId = null; + } + + public String getApplicationInstanceId() { + return this.applicationInstanceId; + } + + public unloadApplication_args setApplicationInstanceId(String applicationInstanceId) { + this.applicationInstanceId = applicationInstanceId; + return this; + } + + public void unsetApplicationInstanceId() { + this.applicationInstanceId = null; + } + + /** Returns true if field applicationInstanceId is set (has been assigned a value) and false otherwise */ + public boolean isSetApplicationInstanceId() { + return this.applicationInstanceId != null; + } + + public void setApplicationInstanceIdIsSet(boolean value) { + if (!value) { + this.applicationInstanceId = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case APPLICATION_INSTANCE_ID: + if (value == null) { + unsetApplicationInstanceId(); + } else { + setApplicationInstanceId((String)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case APPLICATION_INSTANCE_ID: + return getApplicationInstanceId(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case APPLICATION_INSTANCE_ID: + return isSetApplicationInstanceId(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof unloadApplication_args) + return this.equals((unloadApplication_args)that); + return false; + } + + public boolean equals(unloadApplication_args that) { + if (that == null) + return false; + + boolean this_present_applicationInstanceId = true && this.isSetApplicationInstanceId(); + boolean that_present_applicationInstanceId = true && that.isSetApplicationInstanceId(); + if (this_present_applicationInstanceId || that_present_applicationInstanceId) { + if (!(this_present_applicationInstanceId && that_present_applicationInstanceId)) + return false; + if (!this.applicationInstanceId.equals(that.applicationInstanceId)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_applicationInstanceId = true && (isSetApplicationInstanceId()); + list.add(present_applicationInstanceId); + if (present_applicationInstanceId) + list.add(applicationInstanceId); + + return list.hashCode(); + } + + @Override + public int compareTo(unloadApplication_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetApplicationInstanceId()).compareTo(other.isSetApplicationInstanceId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetApplicationInstanceId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.applicationInstanceId, other.applicationInstanceId); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("unloadApplication_args("); + boolean first = true; + + sb.append("applicationInstanceId:"); + if (this.applicationInstanceId == null) { + sb.append("null"); + } else { + sb.append(this.applicationInstanceId); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class unloadApplication_argsStandardSchemeFactory implements SchemeFactory { + public unloadApplication_argsStandardScheme getScheme() { + return new unloadApplication_argsStandardScheme(); + } + } + + private static class unloadApplication_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, unloadApplication_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // APPLICATION_INSTANCE_ID + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.applicationInstanceId = iprot.readString(); + struct.setApplicationInstanceIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, unloadApplication_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.applicationInstanceId != null) { + oprot.writeFieldBegin(APPLICATION_INSTANCE_ID_FIELD_DESC); + oprot.writeString(struct.applicationInstanceId); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class unloadApplication_argsTupleSchemeFactory implements SchemeFactory { + public unloadApplication_argsTupleScheme getScheme() { + return new unloadApplication_argsTupleScheme(); + } + } + + private static class unloadApplication_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, unloadApplication_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetApplicationInstanceId()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetApplicationInstanceId()) { + oprot.writeString(struct.applicationInstanceId); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, unloadApplication_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.applicationInstanceId = iprot.readString(); + struct.setApplicationInstanceIdIsSet(true); + } + } + } + + } + + public static class unloadApplication_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("unloadApplication_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new unloadApplication_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new unloadApplication_resultTupleSchemeFactory()); + } + + public RemoteApplicationResult success; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, RemoteApplicationResult.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(unloadApplication_result.class, metaDataMap); + } + + public unloadApplication_result() { + } + + public unloadApplication_result( + RemoteApplicationResult success) + { + this(); + this.success = success; + } + + /** + * Performs a deep copy on other. + */ + public unloadApplication_result(unloadApplication_result other) { + if (other.isSetSuccess()) { + this.success = new RemoteApplicationResult(other.success); + } + } + + public unloadApplication_result deepCopy() { + return new unloadApplication_result(this); + } + + @Override + public void clear() { + this.success = null; + } + + public RemoteApplicationResult getSuccess() { + return this.success; + } + + public unloadApplication_result setSuccess(RemoteApplicationResult success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((RemoteApplicationResult)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof unloadApplication_result) + return this.equals((unloadApplication_result)that); + return false; + } + + public boolean equals(unloadApplication_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_success = true && (isSetSuccess()); + list.add(present_success); + if (present_success) + list.add(success); + + return list.hashCode(); + } + + @Override + public int compareTo(unloadApplication_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("unloadApplication_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + if (success != null) { + success.validate(); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class unloadApplication_resultStandardSchemeFactory implements SchemeFactory { + public unloadApplication_resultStandardScheme getScheme() { + return new unloadApplication_resultStandardScheme(); + } + } + + private static class unloadApplication_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, unloadApplication_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.success = new RemoteApplicationResult(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, unloadApplication_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + struct.success.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class unloadApplication_resultTupleSchemeFactory implements SchemeFactory { + public unloadApplication_resultTupleScheme getScheme() { + return new unloadApplication_resultTupleScheme(); + } + } + + private static class unloadApplication_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, unloadApplication_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetSuccess()) { + struct.success.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, unloadApplication_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.success = new RemoteApplicationResult(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } + } + } + + } + + public static class runApplication_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("runApplication_args"); + + private static final org.apache.thrift.protocol.TField APPLICATION_INSTANCE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("applicationInstanceId", org.apache.thrift.protocol.TType.STRING, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new runApplication_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new runApplication_argsTupleSchemeFactory()); + } + + public String applicationInstanceId; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + APPLICATION_INSTANCE_ID((short)1, "applicationInstanceId"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // APPLICATION_INSTANCE_ID + return APPLICATION_INSTANCE_ID; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.APPLICATION_INSTANCE_ID, new org.apache.thrift.meta_data.FieldMetaData("applicationInstanceId", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(runApplication_args.class, metaDataMap); + } + + public runApplication_args() { + } + + public runApplication_args( + String applicationInstanceId) + { + this(); + this.applicationInstanceId = applicationInstanceId; + } + + /** + * Performs a deep copy on other. + */ + public runApplication_args(runApplication_args other) { + if (other.isSetApplicationInstanceId()) { + this.applicationInstanceId = other.applicationInstanceId; + } + } + + public runApplication_args deepCopy() { + return new runApplication_args(this); + } + + @Override + public void clear() { + this.applicationInstanceId = null; + } + + public String getApplicationInstanceId() { + return this.applicationInstanceId; + } + + public runApplication_args setApplicationInstanceId(String applicationInstanceId) { + this.applicationInstanceId = applicationInstanceId; + return this; + } + + public void unsetApplicationInstanceId() { + this.applicationInstanceId = null; + } + + /** Returns true if field applicationInstanceId is set (has been assigned a value) and false otherwise */ + public boolean isSetApplicationInstanceId() { + return this.applicationInstanceId != null; + } + + public void setApplicationInstanceIdIsSet(boolean value) { + if (!value) { + this.applicationInstanceId = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case APPLICATION_INSTANCE_ID: + if (value == null) { + unsetApplicationInstanceId(); + } else { + setApplicationInstanceId((String)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case APPLICATION_INSTANCE_ID: + return getApplicationInstanceId(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case APPLICATION_INSTANCE_ID: + return isSetApplicationInstanceId(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof runApplication_args) + return this.equals((runApplication_args)that); + return false; + } + + public boolean equals(runApplication_args that) { + if (that == null) + return false; + + boolean this_present_applicationInstanceId = true && this.isSetApplicationInstanceId(); + boolean that_present_applicationInstanceId = true && that.isSetApplicationInstanceId(); + if (this_present_applicationInstanceId || that_present_applicationInstanceId) { + if (!(this_present_applicationInstanceId && that_present_applicationInstanceId)) + return false; + if (!this.applicationInstanceId.equals(that.applicationInstanceId)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_applicationInstanceId = true && (isSetApplicationInstanceId()); + list.add(present_applicationInstanceId); + if (present_applicationInstanceId) + list.add(applicationInstanceId); + + return list.hashCode(); + } + + @Override + public int compareTo(runApplication_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetApplicationInstanceId()).compareTo(other.isSetApplicationInstanceId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetApplicationInstanceId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.applicationInstanceId, other.applicationInstanceId); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("runApplication_args("); + boolean first = true; + + sb.append("applicationInstanceId:"); + if (this.applicationInstanceId == null) { + sb.append("null"); + } else { + sb.append(this.applicationInstanceId); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class runApplication_argsStandardSchemeFactory implements SchemeFactory { + public runApplication_argsStandardScheme getScheme() { + return new runApplication_argsStandardScheme(); + } + } + + private static class runApplication_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, runApplication_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // APPLICATION_INSTANCE_ID + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.applicationInstanceId = iprot.readString(); + struct.setApplicationInstanceIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, runApplication_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.applicationInstanceId != null) { + oprot.writeFieldBegin(APPLICATION_INSTANCE_ID_FIELD_DESC); + oprot.writeString(struct.applicationInstanceId); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class runApplication_argsTupleSchemeFactory implements SchemeFactory { + public runApplication_argsTupleScheme getScheme() { + return new runApplication_argsTupleScheme(); + } + } + + private static class runApplication_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, runApplication_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetApplicationInstanceId()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetApplicationInstanceId()) { + oprot.writeString(struct.applicationInstanceId); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, runApplication_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.applicationInstanceId = iprot.readString(); + struct.setApplicationInstanceIdIsSet(true); + } + } + } + + } + + public static class runApplication_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("runApplication_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new runApplication_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new runApplication_resultTupleSchemeFactory()); + } + + public RemoteApplicationResult success; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, RemoteApplicationResult.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(runApplication_result.class, metaDataMap); + } + + public runApplication_result() { + } + + public runApplication_result( + RemoteApplicationResult success) + { + this(); + this.success = success; + } + + /** + * Performs a deep copy on other. + */ + public runApplication_result(runApplication_result other) { + if (other.isSetSuccess()) { + this.success = new RemoteApplicationResult(other.success); + } + } + + public runApplication_result deepCopy() { + return new runApplication_result(this); + } + + @Override + public void clear() { + this.success = null; + } + + public RemoteApplicationResult getSuccess() { + return this.success; + } + + public runApplication_result setSuccess(RemoteApplicationResult success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((RemoteApplicationResult)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof runApplication_result) + return this.equals((runApplication_result)that); + return false; + } + + public boolean equals(runApplication_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_success = true && (isSetSuccess()); + list.add(present_success); + if (present_success) + list.add(success); + + return list.hashCode(); + } + + @Override + public int compareTo(runApplication_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("runApplication_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + if (success != null) { + success.validate(); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class runApplication_resultStandardSchemeFactory implements SchemeFactory { + public runApplication_resultStandardScheme getScheme() { + return new runApplication_resultStandardScheme(); + } + } + + private static class runApplication_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, runApplication_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.success = new RemoteApplicationResult(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, runApplication_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + struct.success.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class runApplication_resultTupleSchemeFactory implements SchemeFactory { + public runApplication_resultTupleScheme getScheme() { + return new runApplication_resultTupleScheme(); + } + } + + private static class runApplication_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, runApplication_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetSuccess()) { + struct.success.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, runApplication_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.success = new RemoteApplicationResult(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } + } + } + + } + } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java index 1825bfed217..9878d7e766c 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java @@ -19,6 +19,7 @@ import com.google.gson.Gson; import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterManagedProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; import org.slf4j.Logger; @@ -134,3 +135,4 @@ public static void removeResourcesBelongsToParagraph(String noteId, String parag } } } + diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/WellKnownResourceName.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/WellKnownResourceName.java index 2d14fd4bc97..4613c62a30a 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/WellKnownResourceName.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/WellKnownResourceName.java @@ -20,7 +20,8 @@ * Well known resource names in ResourcePool */ public enum WellKnownResourceName { - ParagraphResult("zeppelin.paragraph.result"); // paragraph run result + ZeppelinReplResult("zeppelin.repl.result"), // last object of repl + ZeppelinTableResult("zeppelin.paragraph.result.table"); // paragraph run result String name; WellKnownResourceName(String name) { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java index 33a3ca63c75..28c743740ed 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java @@ -20,6 +20,7 @@ import org.apache.thrift.TException; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterManagedProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; import org.apache.zeppelin.scheduler.Job.Status; @@ -49,8 +50,8 @@ public class RemoteScheduler implements Scheduler { private RemoteInterpreterProcess interpreterProcess; public RemoteScheduler(String name, ExecutorService executor, String noteId, - RemoteInterpreterProcess interpreterProcess, SchedulerListener listener, - int maxConcurrency) { + RemoteInterpreterProcess interpreterProcess, SchedulerListener listener, + int maxConcurrency) { this.name = name; this.executor = executor; this.listener = listener; diff --git a/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift b/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift index 6c3fc36388b..32be4a4a0a8 100644 --- a/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift +++ b/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift @@ -48,7 +48,8 @@ enum RemoteInterpreterEventType { RESOURCE_GET = 7 OUTPUT_APPEND = 8, OUTPUT_UPDATE = 9, - ANGULAR_REGISTRY_PUSH=10 + ANGULAR_REGISTRY_PUSH = 10, + APP_STATUS_UPDATE = 11, } struct RemoteInterpreterEvent { @@ -56,6 +57,11 @@ struct RemoteInterpreterEvent { 2: string data // json serialized data } +struct RemoteApplicationResult { + 1: bool success, + 2: string msg +} + /* * The below variables(name, value) will be connected to getCompletions in paragraph.controller.js * @@ -99,4 +105,8 @@ service RemoteInterpreterService { void angularObjectAdd(1: string name, 2: string noteId, 3: string paragraphId, 4: string object); void angularObjectRemove(1: string name, 2: string noteId, 3: string paragraphId); void angularRegistryPush(1: string registry); + + RemoteApplicationResult loadApplication(1: string applicationInstanceId, 2: string packageInfo, 3: string noteId, 4: string paragraphId); + RemoteApplicationResult unloadApplication(1: string applicationInstanceId); + RemoteApplicationResult runApplication(1: string applicationInstanceId); } diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java new file mode 100644 index 00000000000..06c4a6b8e29 --- /dev/null +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.helium; + +import org.apache.commons.io.FileUtils; +import org.apache.zeppelin.dep.DependencyResolver; +import org.apache.zeppelin.interpreter.InterpreterOutput; +import org.apache.zeppelin.interpreter.InterpreterOutputListener; +import org.apache.zeppelin.resource.LocalResourcePool; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +import static org.junit.Assert.*; + +public class ApplicationLoaderTest { + private File tmpDir; + + @Before + public void setUp() { + tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis()); + tmpDir.mkdirs(); + } + + @After + public void tearDown() throws IOException { + FileUtils.deleteDirectory(tmpDir); + } + + @Test + public void loadUnloadApplication() throws Exception { + // given + LocalResourcePool resourcePool = new LocalResourcePool("pool1"); + DependencyResolver dep = new DependencyResolver(tmpDir.getAbsolutePath()); + ApplicationLoader appLoader = new ApplicationLoader(resourcePool, dep); + + HeliumPackage pkg1 = createPackageInfo(MockApplication1.class.getName(), "artifact1"); + ApplicationContext context1 = createContext("note1", "paragraph1", "app1"); + + // when load application + MockApplication1 app = (MockApplication1) ((ClassLoaderApplication) + appLoader.load(pkg1, context1)).getInnerApplication(); + + // then + assertFalse(app.isUnloaded()); + assertEquals(0, app.getNumRun()); + + // when unload + app.unload(); + + // then + assertTrue(app.isUnloaded()); + assertEquals(0, app.getNumRun()); + } + + public HeliumPackage createPackageInfo(String className, String artifact) { + HeliumPackage app1 = new HeliumPackage( + HeliumPackage.Type.APPLICATION, + "name1", + "desc1", + artifact, + className, + new String[][]{{}}); + return app1; + } + + public ApplicationContext createContext(String noteId, String paragraphId, String appInstanceId) { + ApplicationContext context1 = new ApplicationContext( + noteId, + paragraphId, + appInstanceId, + null, + new InterpreterOutput(new InterpreterOutputListener() { + @Override + public void onAppend(InterpreterOutput out, byte[] line) { + + } + + @Override + public void onUpdate(InterpreterOutput out, byte[] output) { + + } + })); + return context1; + } +} diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/MockApplication1.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/MockApplication1.java new file mode 100644 index 00000000000..df3afeffc10 --- /dev/null +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/MockApplication1.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.resource.ResourceSet; + +/** + * Mock application + */ +public class MockApplication1 extends Application { + boolean unloaded; + int run; + + public MockApplication1(ApplicationContext context) { + super(context); + unloaded = false; + run = 0; + } + + @Override + public void run(ResourceSet args) { + run++; + } + + @Override + public void unload() { + unloaded = true; + } + + public boolean isUnloaded() { + return unloaded; + } + + public int getNumRun() { + return run; + } +} diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java index 7ffa1708710..5def888b637 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java @@ -73,6 +73,7 @@ public void setUp() throws Exception { "fakeRepo", env, 10 * 1000, + null, null ); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java index 4a473f3f368..74649b1e750 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java @@ -70,7 +70,8 @@ private RemoteInterpreter createMockInterpreter() { "fakeRepo", env, 10 * 1000, - this); + this, + null); intpGroup.get("note").add(intp); intp.setInterpreterGroup(intpGroup); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java index f9d7d3942b5..01582821308 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java @@ -41,9 +41,9 @@ public class RemoteInterpreterProcessTest { @Test public void testStartStop() { InterpreterGroup intpGroup = new InterpreterGroup(); - RemoteInterpreterProcess rip = new RemoteInterpreterProcess( + RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess( INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap(), - 10 * 1000, null); + 10 * 1000, null, null); assertFalse(rip.isRunning()); assertEquals(0, rip.referenceCount()); assertEquals(1, rip.reference(intpGroup)); @@ -58,7 +58,7 @@ public void testStartStop() { @Test public void testClientFactory() throws Exception { InterpreterGroup intpGroup = new InterpreterGroup(); - RemoteInterpreterProcess rip = new RemoteInterpreterProcess( + RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess( INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap(), mock(RemoteInterpreterEventPoller.class), 10 * 1000); rip.reference(intpGroup); @@ -96,8 +96,14 @@ public void testStartStopRemoteInterpreter() throws TException, InterruptedExcep InterpreterGroup intpGroup = mock(InterpreterGroup.class); when(intpGroup.getProperty()).thenReturn(properties); when(intpGroup.containsKey(Constants.EXISTING_PROCESS)).thenReturn(true); - RemoteInterpreterProcess rip = new RemoteInterpreterProcess(INTERPRETER_SCRIPT, "nonexists", - "fakeRepo", new HashMap(), 10 * 1000, null); + + RemoteInterpreterProcess rip = new RemoteInterpreterManagedProcess( + INTERPRETER_SCRIPT, + "nonexists", + "fakeRepo", + new HashMap(), + mock(RemoteInterpreterEventPoller.class) + , 10 * 1000); assertFalse(rip.isRunning()); assertEquals(0, rip.referenceCount()); assertEquals(1, rip.reference(intpGroup)); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java index ea0bbf4f21f..af1c4471a05 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java @@ -89,6 +89,7 @@ private RemoteInterpreter createMockInterpreterA(Properties p, String noteId) { "fakeRepo", env, 10 * 1000, + null, null); } @@ -106,6 +107,7 @@ private RemoteInterpreter createMockInterpreterB(Properties p, String noteId) { "fakeRepo", env, 10 * 1000, + null, null); } @@ -204,6 +206,7 @@ public void testRemoteSchedulerSharing() throws TTransportException, IOException "fakeRepo", env, 10 * 1000, + null, null); @@ -219,6 +222,7 @@ public void testRemoteSchedulerSharing() throws TTransportException, IOException "fakeRepo", env, 10 * 1000, + null, null); intpGroup.get("note").add(intpB); @@ -683,7 +687,7 @@ public void should_push_local_angular_repo_to_remote() throws Exception { //Given final Client client = Mockito.mock(Client.class); final RemoteInterpreter intr = new RemoteInterpreter(new Properties(), "noteId", - MockInterpreterA.class.getName(), "runner", "path","localRepo", env, 10 * 1000, null); + MockInterpreterA.class.getName(), "runner", "path","localRepo", env, 10 * 1000, null, null); final AngularObjectRegistry registry = new AngularObjectRegistry("spark", null); registry.add("name", "DuyHai DOAN", "nodeId", "paragraphId"); final InterpreterGroup interpreterGroup = new InterpreterGroup("groupId"); @@ -723,11 +727,12 @@ public void testEnvronmentAndPropertySet() { p, "note", MockInterpreterEnv.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), + new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, 10 * 1000, + null, null); intpGroup.put("note", new LinkedList()); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java index ae5e6f5ff93..02dba20c05d 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java @@ -69,6 +69,7 @@ public void setUp() throws Exception { "fakeRepo", env, 10 * 1000, + null, null ); @@ -86,6 +87,7 @@ public void setUp() throws Exception { "fakeRepo", env, 10 * 1000, + null, null ); @@ -110,11 +112,11 @@ public void setUp() throws Exception { intp1.open(); intp2.open(); - eventPoller1 = new RemoteInterpreterEventPoller(null); + eventPoller1 = new RemoteInterpreterEventPoller(null, null); eventPoller1.setInterpreterGroup(intpGroup1); eventPoller1.setInterpreterProcess(intpGroup1.getRemoteInterpreterProcess()); - eventPoller2 = new RemoteInterpreterEventPoller(null); + eventPoller2 = new RemoteInterpreterEventPoller(null, null); eventPoller2.setInterpreterGroup(intpGroup2); eventPoller2.setInterpreterProcess(intpGroup2.getRemoteInterpreterProcess()); @@ -140,13 +142,12 @@ public void testRemoteDistributedResourcePool() { InterpreterResult ret; intp1.interpret("put key1 value1", context); intp2.interpret("put key2 value2", context); - int numInterpreterResult = 2; ret = intp1.interpret("getAll", context); - assertEquals(numInterpreterResult + 2, gson.fromJson(ret.message(), ResourceSet.class).size()); + assertEquals(2, gson.fromJson(ret.message(), ResourceSet.class).size()); ret = intp2.interpret("getAll", context); - assertEquals(numInterpreterResult + 2, gson.fromJson(ret.message(), ResourceSet.class).size()); + assertEquals(2, gson.fromJson(ret.message(), ResourceSet.class).size()); ret = intp1.interpret("get key1", context); assertEquals("value1", gson.fromJson(ret.message(), String.class)); @@ -218,16 +219,15 @@ public void testResourcePoolUtils() { intp2.interpret("put note2:paragraph1:key1 value1", context); intp2.interpret("put note2:paragraph2:key2 value2", context); - int numInterpreterResult = 2; // then get all resources. - assertEquals(numInterpreterResult + 4, ResourcePoolUtils.getAllResources().size()); + assertEquals(4, ResourcePoolUtils.getAllResources().size()); // when remove all resources from note1 ResourcePoolUtils.removeResourcesBelongsToNote("note1"); // then resources should be removed. - assertEquals(numInterpreterResult + 2, ResourcePoolUtils.getAllResources().size()); + assertEquals(2, ResourcePoolUtils.getAllResources().size()); assertEquals("", gson.fromJson( intp1.interpret("get note1:paragraph1:key1", context).message(), String.class)); @@ -240,7 +240,7 @@ public void testResourcePoolUtils() { ResourcePoolUtils.removeResourcesBelongsToParagraph("note2", "paragraph1"); // then 1 - assertEquals(numInterpreterResult + 1, ResourcePoolUtils.getAllResources().size()); + assertEquals(1, ResourcePoolUtils.getAllResources().size()); assertEquals("value2", gson.fromJson( intp1.interpret("get note2:paragraph2:key2", context).message(), String.class)); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java index 40dcef28ab5..f17d88d50af 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java @@ -80,7 +80,8 @@ public void test() throws Exception { "fakeRepo", env, 10 * 1000, - this); + this, + null); intpGroup.put("note", new LinkedList()); intpGroup.get("note").add(intpA); @@ -168,7 +169,8 @@ public void testAbortOnPending() throws Exception { "fakeRepo", env, 10 * 1000, - this); + this, + null); intpGroup.put("note", new LinkedList()); intpGroup.get("note").add(intpA); diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java new file mode 100644 index 00000000000..062f5b93978 --- /dev/null +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.rest; + +import com.google.gson.Gson; +import org.apache.zeppelin.helium.Helium; +import org.apache.zeppelin.helium.HeliumApplicationFactory; +import org.apache.zeppelin.helium.HeliumPackage; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.Notebook; +import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.server.JsonResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.*; +import javax.ws.rs.core.Response; + +/** + * Helium Rest Api + */ +@Path("/helium") +@Produces("application/json") +public class HeliumRestApi { + Logger logger = LoggerFactory.getLogger(HeliumRestApi.class); + + private Helium helium; + private HeliumApplicationFactory applicationFactory; + private Notebook notebook; + private Gson gson = new Gson(); + + public HeliumRestApi() { + } + + public HeliumRestApi(Helium helium, + HeliumApplicationFactory heliumApplicationFactory, + Notebook notebook) { + this.helium = helium; + this.applicationFactory = heliumApplicationFactory; + this.notebook = notebook; + } + + /** + * Get all packages + * @return + */ + @GET + @Path("all") + public Response getAll() { + return new JsonResponse(Response.Status.OK, "", helium.getAllPackageInfo()).build(); + } + + @GET + @Path("suggest/{noteId}/{paragraphId}") + public Response suggest(@PathParam("noteId") String noteId, + @PathParam("paragraphId") String paragraphId) { + Note note = notebook.getNote(noteId); + if (note == null) { + return new JsonResponse(Response.Status.NOT_FOUND, "Note " + noteId + " not found").build(); + } + + Paragraph paragraph = note.getParagraph(paragraphId); + if (paragraph == null) { + return new JsonResponse(Response.Status.NOT_FOUND, "Paragraph " + paragraphId + " not found") + .build(); + } + + return new JsonResponse(Response.Status.OK, "", helium.suggestApp(paragraph)).build(); + } + + @POST + @Path("load/{noteId}/{paragraphId}") + public Response suggest(@PathParam("noteId") String noteId, + @PathParam("paragraphId") String paragraphId, + String heliumPackage) { + + Note note = notebook.getNote(noteId); + if (note == null) { + return new JsonResponse(Response.Status.NOT_FOUND, "Note " + noteId + " not found").build(); + } + + Paragraph paragraph = note.getParagraph(paragraphId); + if (paragraph == null) { + return new JsonResponse(Response.Status.NOT_FOUND, "Paragraph " + paragraphId + " not found") + .build(); + } + HeliumPackage pkg = gson.fromJson(heliumPackage, HeliumPackage.class); + + String appId = applicationFactory.loadAndRun(pkg, paragraph); + return new JsonResponse(Response.Status.OK, "", appId).build(); + } + +} diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 0ff0dc6ac63..0f7d8a1b9fc 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -17,10 +17,23 @@ package org.apache.zeppelin.server; +import java.io.File; +import java.io.IOException; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ExecutorService; + +import javax.net.ssl.SSLContext; +import javax.servlet.DispatcherType; +import javax.ws.rs.core.Application; + import org.apache.cxf.jaxrs.servlet.CXFNonSpringJaxrsServlet; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.dep.DependencyResolver; +import org.apache.zeppelin.helium.Helium; +import org.apache.zeppelin.helium.HeliumApplicationFactory; import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.notebook.NotebookAuthorization; @@ -46,14 +59,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.DispatcherType; -import javax.ws.rs.core.Application; -import java.io.File; -import java.io.IOException; -import java.util.EnumSet; -import java.util.HashSet; -import java.util.Set; - /** * Main class of Zeppelin. */ @@ -63,6 +68,8 @@ public class ZeppelinServer extends Application { public static Notebook notebook; public static Server jettyWebServer; public static NotebookServer notebookWsServer; + public static Helium helium; + public static HeliumApplicationFactory heliumApplicationFactory; private SchedulerFactory schedulerFactory; private InterpreterFactory replFactory; @@ -77,9 +84,12 @@ public ZeppelinServer() throws Exception { this.depResolver = new DependencyResolver( conf.getString(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO)); + + this.helium = new Helium(conf.getHeliumConfPath(), conf.getHeliumDefaultLocalRegistryPath()); + this.heliumApplicationFactory = new HeliumApplicationFactory(); this.schedulerFactory = new SchedulerFactory(); this.replFactory = new InterpreterFactory(conf, notebookWsServer, - notebookWsServer, depResolver); + notebookWsServer, heliumApplicationFactory, depResolver); this.notebookRepo = new NotebookRepoSync(conf); this.notebookIndex = new LuceneSearch(); this.notebookAuthorization = new NotebookAuthorization(conf); @@ -87,6 +97,13 @@ public ZeppelinServer() throws Exception { notebook = new Notebook(conf, notebookRepo, schedulerFactory, replFactory, notebookWsServer, notebookIndex, notebookAuthorization, credentials); + + // to update notebook from application event from remote process. + heliumApplicationFactory.setNotebook(notebook); + // to update fire websocket event on application event. + heliumApplicationFactory.setApplicationEventListener(notebookWsServer); + + notebook.addNotebookEventListener(heliumApplicationFactory); } public static void main(String[] args) throws InterruptedException { @@ -294,6 +311,9 @@ public Set getSingletons() { NotebookRestApi notebookApi = new NotebookRestApi(notebook, notebookWsServer, notebookIndex); singletons.add(notebookApi); + HeliumRestApi heliumApi = new HeliumRestApi(helium, heliumApplicationFactory, notebook); + singletons.add(heliumApi); + InterpreterRestApi interpreterApi = new InterpreterRestApi(replFactory); singletons.add(interpreterApi); diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 42edb08ccef..1cbd81192b6 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -34,6 +34,8 @@ import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.AngularObjectRegistryListener; +import org.apache.zeppelin.helium.ApplicationEventListener; +import org.apache.zeppelin.helium.HeliumPackage; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; @@ -61,7 +63,7 @@ */ public class NotebookServer extends WebSocketServlet implements NotebookSocketListener, JobListenerFactory, AngularObjectRegistryListener, - RemoteInterpreterProcessListener { + RemoteInterpreterProcessListener, ApplicationEventListener { /** * Job manager service type */ @@ -759,6 +761,7 @@ private void angularObjectUpdated(NotebookSocket conn, HashSet userAndRo if (interpreterGroupId.equals(setting.getInterpreterGroup(note.id()).getId())) { AngularObjectRegistry angularObjectRegistry = setting .getInterpreterGroup(note.id()).getAngularObjectRegistry(); + // first trying to get local registry ao = angularObjectRegistry.get(varName, noteId, paragraphId); if (ao == null) { @@ -1135,7 +1138,6 @@ public void onOutputAppend(String noteId, String paragraphId, String output) { .put("noteId", noteId) .put("paragraphId", paragraphId) .put("data", output); - Paragraph paragraph = notebook().getNote(noteId).getParagraph(paragraphId); broadcast(noteId, msg); } @@ -1151,7 +1153,60 @@ public void onOutputUpdated(String noteId, String paragraphId, String output) { .put("noteId", noteId) .put("paragraphId", paragraphId) .put("data", output); - Paragraph paragraph = notebook().getNote(noteId).getParagraph(paragraphId); + broadcast(noteId, msg); + } + + /** + * When application append output + * @param noteId + * @param paragraphId + * @param appId + * @param output + */ + @Override + public void onOutputAppend(String noteId, String paragraphId, String appId, String output) { + Message msg = new Message(OP.APP_APPEND_OUTPUT) + .put("noteId", noteId) + .put("paragraphId", paragraphId) + .put("appId", appId) + .put("data", output); + broadcast(noteId, msg); + } + + /** + * When application update output + * @param noteId + * @param paragraphId + * @param appId + * @param output + */ + @Override + public void onOutputUpdated(String noteId, String paragraphId, String appId, String output) { + Message msg = new Message(OP.APP_UPDATE_OUTPUT) + .put("noteId", noteId) + .put("paragraphId", paragraphId) + .put("appId", appId) + .put("data", output); + broadcast(noteId, msg); + } + + @Override + public void onLoad(String noteId, String paragraphId, String appId, HeliumPackage pkg) { + Message msg = new Message(OP.APP_LOAD) + .put("noteId", noteId) + .put("paragraphId", paragraphId) + .put("appId", appId) + .put("pkg", pkg); + broadcast(noteId, msg); + } + + @Override + public void onStatusChange(String noteId, String paragraphId, String appId, String status) { + Message msg = new Message(OP.APP_STATUS_CHANGE) + .put("noteId", noteId) + .put("paragraphId", paragraphId) + .put("appId", appId) + .put("status", status); broadcast(noteId, msg); } @@ -1281,19 +1336,17 @@ public void onUpdate(String interpreterGroupId, AngularObject object) { List intpSettings = notebook.getInterpreterFactory() .getInterpreterSettings(note.getId()); - if (intpSettings.isEmpty()) + if (intpSettings.isEmpty()) { continue; - for (InterpreterSetting setting : intpSettings) { - if (setting.getInterpreterGroup(note.id()).getId().equals(interpreterGroupId)) { - broadcast( - note.id(), - new Message(OP.ANGULAR_OBJECT_UPDATE) - .put("angularObject", object) - .put("interpreterGroupId", interpreterGroupId) - .put("noteId", note.id()) - .put("paragraphId", object.getParagraphId())); - } } + + broadcast( + note.id(), + new Message(OP.ANGULAR_OBJECT_UPDATE) + .put("angularObject", object) + .put("interpreterGroupId", interpreterGroupId) + .put("noteId", note.id()) + .put("paragraphId", object.getParagraphId())); } } diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java index 81c7190b186..0ff01353501 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java @@ -165,7 +165,7 @@ public void testSqlSpark() throws Exception { } WebElement paragraph1Result = driver.findElement(By.xpath( - getParagraphXPath(1) + "//div[@class=\"tableDisplay\"]//table")); + getParagraphXPath(1) + "//div[@class=\"tableDisplay\"]/div/div/div/div/div/div[1]")); collector.checkThat("Paragraph from SparkParagraphIT of testSqlSpark result: ", paragraph1Result.getText().toString(), CoreMatchers.equalTo("age\njob\nmarital\neducation\nbalance\n" + "30 unemployed married primary 1,787")); diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java index 7bedd284c59..69006dea473 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java @@ -248,7 +248,7 @@ protected static boolean checkIfServerIsRunning() { request = httpGet("/"); isRunning = request.getStatusCode() == 200; } catch (IOException e) { - LOG.error("Exception in AbstractTestRestApi while checkIfServerIsRunning ", e); + LOG.error("AbstractTestRestApi.checkIfServerIsRunning() fails .. ZeppelinServer is not running"); isRunning = false; } finally { if (request != null) { diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java index d234ffd4b83..3c77b45e71f 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java @@ -24,7 +24,12 @@ import java.util.List; import java.util.Map; +import org.apache.commons.exec.CommandLine; +import org.apache.commons.exec.DefaultExecutor; +import org.apache.commons.exec.ExecuteWatchdog; +import org.apache.commons.exec.PumpStreamHandler; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.Paragraph; @@ -101,6 +106,7 @@ public void sparkRTest() throws IOException { ); note.run(p.getId()); waitForFinish(p); + System.err.println("sparkRTest=" + p.getResult().message()); assertEquals(Status.FINISHED, p.getStatus()); assertEquals("[1] 3", p.getResult().message()); } diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html b/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html index 76135b1d5c3..507c57f984a 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html @@ -12,56 +12,112 @@ limitations under the License. --> -
    -
    - - - - - - -
    - - - - - - - - settings - - + + + + + + + + + + + +
    + +
    + +
    + +
    + + + +
    + + + + settings + + + diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph-results.html b/zeppelin-web/src/app/notebook/paragraph/paragraph-results.html index 612fdbd88f6..fd608c51d31 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph-results.html +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph-results.html @@ -13,6 +13,7 @@ -->
    @@ -60,3 +61,11 @@ ng-bind="paragraph.errorMessage">
    + +
    +
    +
    +
    + + diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 398191c8e42..211ab59cc7f 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -16,7 +16,7 @@ angular.module('zeppelinWebApp') .controller('ParagraphCtrl', function($scope,$rootScope, $route, $window, $element, $routeParams, $location, - $timeout, $compile, websocketMsgSrv, ngToast, SaveAsService) { + $timeout, $compile, $http, websocketMsgSrv, baseUrlSrv, ngToast, SaveAsService) { var ANGULAR_FUNCTION_OBJECT_NAME_PREFIX = '_Z_ANGULAR_FUNC_'; $scope.parentNote = null; $scope.paragraph = null; @@ -116,10 +116,21 @@ angular.module('zeppelinWebApp') } else if ($scope.getResultType() === 'TEXT') { $scope.renderText(); } + + getApplicationStates(); + getSuggestions(); + + var activeApp = _.get($scope.paragraph.config, 'helium.activeApp'); + if (activeApp) { + var app = _.find($scope.apps, {id: activeApp}); + renderApp(app); + } }; - $scope.renderHtml = function() { - var retryRenderer = function() { + + + $scope.renderHtml = function() { + var retryRenderer = function() { if (angular.element('#p' + $scope.paragraph.id + '_html').length) { try { angular.element('#p' + $scope.paragraph.id + '_html').html($scope.paragraph.result.msg); @@ -202,8 +213,23 @@ angular.module('zeppelinWebApp') $scope.$on('angularObjectUpdate', function(event, data) { var noteId = $route.current.pathParams.noteId; - if (!data.noteId || (data.noteId === noteId && (!data.paragraphId || data.paragraphId === $scope.paragraph.id))) { - var scope = paragraphScope; + if (!data.noteId || data.noteId === noteId) { + var scope; + var registry; + + if (!data.paragraphId || data.paragraphId === $scope.paragraph.id) { + scope = paragraphScope; + registry = angularObjectRegistry; + } else { + var app = _.find($scope.apps, { id: data.paragraphId}); + if (app) { + scope = getAppScope(app); + registry = getAppRegistry(app); + } else { + // no matching app in this paragraph + return; + } + } var varName = data.angularObject.name; if (angular.equals(data.angularObject.object, scope[varName])) { @@ -211,32 +237,32 @@ angular.module('zeppelinWebApp') return; } - if (!angularObjectRegistry[varName]) { - angularObjectRegistry[varName] = { + if (!registry[varName]) { + registry[varName] = { interpreterGroupId : data.interpreterGroupId, noteId : data.noteId, paragraphId : data.paragraphId }; } else { - angularObjectRegistry[varName].noteId = angularObjectRegistry[varName].noteId || data.noteId; - angularObjectRegistry[varName].paragraphId = angularObjectRegistry[varName].paragraphId || data.paragraphId; + registry[varName].noteId = registry[varName].noteId || data.noteId; + registry[varName].paragraphId = registry[varName].paragraphId || data.paragraphId; } - angularObjectRegistry[varName].skipEmit = true; + registry[varName].skipEmit = true; - if (!angularObjectRegistry[varName].clearWatcher) { - angularObjectRegistry[varName].clearWatcher = scope.$watch(varName, function(newValue, oldValue) { - console.log('angular object (paragraph) updated %o %o', varName, angularObjectRegistry[varName]); - if (angularObjectRegistry[varName].skipEmit) { - angularObjectRegistry[varName].skipEmit = false; + if (!registry[varName].clearWatcher) { + registry[varName].clearWatcher = scope.$watch(varName, function(newValue, oldValue) { + console.log('angular object (paragraph) updated %o %o', varName, registry[varName]); + if (registry[varName].skipEmit) { + registry[varName].skipEmit = false; return; } websocketMsgSrv.updateAngularObject( - angularObjectRegistry[varName].noteId, - angularObjectRegistry[varName].paragraphId, + registry[varName].noteId, + registry[varName].paragraphId, varName, newValue, - angularObjectRegistry[varName].interpreterGroupId); + registry[varName].interpreterGroupId); }); } console.log('angular object (paragraph) created %o', varName); @@ -258,14 +284,30 @@ angular.module('zeppelinWebApp') $scope.$on('angularObjectRemove', function(event, data) { var noteId = $route.current.pathParams.noteId; - if (!data.noteId || (data.noteId === noteId && (!data.paragraphId || data.paragraphId === $scope.paragraph.id))) { - var scope = paragraphScope; + if (!data.noteId || data.noteId === noteId) { + var scope; + var registry; + + if (!data.paragraphId || data.paragraphId === $scope.paragraph.id) { + scope = paragraphScope; + registry = angularObjectRegistry; + } else { + var app = _.find($scope.apps, { id: data.paragraphId}); + if (app) { + scope = getAppScope(app); + registry = getAppRegistry(app); + } else { + // no matching app in this paragraph + return; + } + } + var varName = data.name; // clear watcher - if (angularObjectRegistry[varName]) { - angularObjectRegistry[varName].clearWatcher(); - angularObjectRegistry[varName] = undefined; + if (registry[varName]) { + registry[varName].clearWatcher(); + registry[varName] = undefined; } // remove scope variable @@ -365,12 +407,17 @@ angular.module('zeppelinWebApp') var newType = $scope.getResultType(data.paragraph); var oldGraphMode = $scope.getGraphMode(); var newGraphMode = $scope.getGraphMode(data.paragraph); + var oldActiveApp = _.get($scope.paragraph.config, 'helium.activeApp'); + var newActiveApp = _.get(data.paragraph.config, 'helium.activeApp'); + var resultRefreshed = (data.paragraph.dateFinished !== $scope.paragraph.dateFinished) || isEmpty(data.paragraph.result) !== isEmpty($scope.paragraph.result) || - data.paragraph.status === 'ERROR'; + data.paragraph.status === 'ERROR' || + (!newActiveApp && oldActiveApp !== newActiveApp); var statusChanged = (data.paragraph.status !== $scope.paragraph.status); + //console.log("updateParagraph oldData %o, newData %o. type %o -> %o, mode %o -> %o", $scope.paragraph, data, oldType, newType, oldGraphMode, newGraphMode); if ($scope.paragraph.text !== data.paragraph.text) { @@ -432,6 +479,14 @@ angular.module('zeppelinWebApp') $scope.renderText(); } + getApplicationStates(); + getSuggestions(); + + if (newActiveApp && newActiveApp !== oldActiveApp) { + var app = _.find($scope.apps, { id : newActiveApp }); + renderApp(app); + } + if (statusChanged || resultRefreshed) { // when last paragraph runs, zeppelin automatically appends new paragraph. // this broadcast will focus to the newly inserted paragraph @@ -1214,6 +1269,9 @@ angular.module('zeppelinWebApp') // graph options newConfig.graph.mode = newMode; + // see switchApp() + _.set(newConfig, 'helium.activeApp', undefined); + commitParagraph($scope.paragraph.title, $scope.paragraph.text, newConfig, newParams); }; @@ -1432,7 +1490,8 @@ angular.module('zeppelinWebApp') }; $scope.isGraphMode = function(graphName) { - if ($scope.getResultType() === 'TABLE' && $scope.getGraphMode()===graphName) { + var activeAppId = _.get($scope.paragraph.config, 'helium.activeApp'); + if ($scope.getResultType() === 'TABLE' && $scope.getGraphMode()===graphName && !activeAppId) { return true; } else { return false; @@ -2180,4 +2239,188 @@ angular.module('zeppelinWebApp') } SaveAsService.SaveAs(dsv, 'data', extension); }; + + // Helium --------------------------------------------- + + // app states + $scope.apps = []; + + // suggested apps + $scope.suggestion = {}; + + $scope.switchApp = function(appId) { + var app = _.find($scope.apps, { id : appId }); + var config = $scope.paragraph.config; + var settings = $scope.paragraph.settings; + + var newConfig = angular.copy(config); + var newParams = angular.copy(settings.params); + + // 'helium.activeApp' can be cleared by setGraphMode() + _.set(newConfig, 'helium.activeApp', appId); + + commitConfig(newConfig, newParams); + }; + + $scope.loadApp = function(heliumPackage) { + var noteId = $route.current.pathParams.noteId; + $http.post(baseUrlSrv.getRestApiBase() + '/helium/load/' + noteId + '/' + $scope.paragraph.id, + heliumPackage) + .success(function(data, status, headers, config) { + console.log('Load app %o', data); + }) + .error(function(err, status, headers, config) { + console.log('Error %o', err); + }); + }; + + var commitConfig = function(config, params) { + var paragraph = $scope.paragraph; + commitParagraph(paragraph.title, paragraph.text, config, params); + }; + + var getApplicationStates = function() { + var appStates = []; + var paragraph = $scope.paragraph; + + // Display ApplicationState + if (paragraph.apps) { + _.forEach(paragraph.apps, function (app) { + appStates.push({ + id: app.id, + pkg: app.pkg, + status: app.status, + output: app.output + }); + }); + } + + // update or remove app states no longer exists + _.forEach($scope.apps, function(currentAppState, idx) { + var newAppState = _.find(appStates, { id : currentAppState.id }); + if (newAppState) { + angular.extend($scope.apps[idx], newAppState); + } else { + $scope.apps.splice(idx, 1); + } + }); + + // add new app states + _.forEach(appStates, function(app, idx) { + if ($scope.apps.length <= idx || $scope.apps[idx].id !== app.id) { + $scope.apps.splice(idx, 0, app); + } + }); + }; + + var getSuggestions = function() { + // Get suggested apps + var noteId = $route.current.pathParams.noteId; + $http.get(baseUrlSrv.getRestApiBase() + '/helium/suggest/' + noteId + '/' + $scope.paragraph.id) + .success(function(data, status, headers, config) { + console.log('Suggested apps %o', data); + $scope.suggestion = data.body; + }) + .error(function(err, status, headers, config) { + console.log('Error %o', err); + }); + }; + + var getAppScope = function(appState) { + if (!appState.scope) { + appState.scope = $rootScope.$new(true, $rootScope); + } + + return appState.scope; + }; + + var getAppRegistry = function(appState) { + if (!appState.registry) { + appState.registry = {}; + } + + return appState.registry; + }; + + var renderApp = function(appState) { + var retryRenderer = function() { + var targetEl = angular.element(document.getElementById('p' + appState.id)); + console.log('retry renderApp %o', targetEl); + if (targetEl.length) { + try { + console.log('renderApp %o', appState); + targetEl.html(appState.output); + $compile(targetEl.contents())(getAppScope(appState)); + } catch(err) { + console.log('App rendering error %o', err); + } + } else { + $timeout(retryRenderer, 1000); + } + }; + $timeout(retryRenderer); + }; + + $scope.$on('appendAppOutput', function(event, data) { + if ($scope.paragraph.id === data.paragraphId) { + var app = _.find($scope.apps, { id : data.appId }); + if (app) { + app.output += data.data; + + var paragraphAppState = _.find($scope.paragraph.apps, { id : data.appId }); + paragraphAppState.output = app.output; + + var targetEl = angular.element(document.getElementById('p' + app.id)); + targetEl.html(app.output); + $compile(targetEl.contents())(getAppScope(app)); + console.log('append app output %o', $scope.apps); + } + } + }); + + $scope.$on('updateAppOutput', function(event, data) { + if ($scope.paragraph.id === data.paragraphId) { + var app = _.find($scope.apps, { id : data.appId }); + if (app) { + app.output = data.data; + + var paragraphAppState = _.find($scope.paragraph.apps, { id : data.appId }); + paragraphAppState.output = app.output; + + var targetEl = angular.element(document.getElementById('p' + app.id)); + targetEl.html(app.output); + $compile(targetEl.contents())(getAppScope(app)); + console.log('append app output'); + } + } + }); + + $scope.$on('appLoad', function(event, data) { + if ($scope.paragraph.id === data.paragraphId) { + var app = _.find($scope.apps, {id: data.appId}); + if (!app) { + app = { + id: data.appId, + pkg: data.pkg, + status: 'UNLOADED', + output: '' + }; + + $scope.apps.push(app); + $scope.paragraph.apps.push(app); + $scope.switchApp(app.id); + } + } + }); + + $scope.$on('appStatusChange', function(event, data) { + if ($scope.paragraph.id === data.paragraphId) { + var app = _.find($scope.apps, {id: data.appId}); + if (app) { + app.status = data.status; + var paragraphAppState = _.find($scope.paragraph.apps, { id : data.appId }); + paragraphAppState.status = app.status; + } + } + }); }); diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.css b/zeppelin-web/src/app/notebook/paragraph/paragraph.css index cea3ebde4c7..d8b464eeed1 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.css +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.css @@ -484,6 +484,11 @@ table.table-striped { right: 15px; } +.appSuggestion { + width: 200px; + padding: 5px 10px 5px 10px; +} + /* DSV download toggle button */ .caretBtn { padding-right: 4px !important; diff --git a/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js b/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js index e07fb165af9..19afdc09edc 100644 --- a/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js +++ b/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js @@ -96,6 +96,14 @@ angular.module('zeppelinWebApp').factory('websocketEvents', function($rootScope, $rootScope.$broadcast('angularObjectUpdate', data); } else if (op === 'ANGULAR_OBJECT_REMOVE') { $rootScope.$broadcast('angularObjectRemove', data); + } else if (op === 'APP_APPEND_OUTPUT') { + $rootScope.$broadcast('appendAppOutput', data); + } else if (op === 'APP_UPDATE_OUTPUT') { + $rootScope.$broadcast('updateAppOutput', data); + } else if (op === 'APP_LOAD') { + $rootScope.$broadcast('appLoad', data); + } else if (op === 'APP_STATUS_CHANGE') { + $rootScope.$broadcast('appStatusChange', data); } }); diff --git a/zeppelin-web/test/spec/controllers/paragraph.js b/zeppelin-web/test/spec/controllers/paragraph.js index 7cdf74876f1..77fc495cc6d 100644 --- a/zeppelin-web/test/spec/controllers/paragraph.js +++ b/zeppelin-web/test/spec/controllers/paragraph.js @@ -10,6 +10,13 @@ describe('Controller: ParagraphCtrl', function() { var paragraphMock = { config: {} }; + var route = { + current : { + pathParams : { + noteId : 'noteId' + } + } + }; beforeEach(inject(function($controller, $rootScope) { scope = $rootScope.$new(); @@ -18,8 +25,10 @@ describe('Controller: ParagraphCtrl', function() { ParagraphCtrl = $controller('ParagraphCtrl', { $scope: scope, websocketMsgSrv: websocketMsgSrvMock, - $element: {} + $element: {}, + $route: route }); + scope.init(paragraphMock); })); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 57de4a106be..667db2ed5e5 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -362,6 +362,14 @@ public String getInterpreterSettingPath() { return getRelativeDir(String.format("%s/interpreter.json", getConfDir())); } + public String getHeliumConfPath() { + return getRelativeDir(String.format("%s/helium.json", getConfDir())); + } + + public String getHeliumDefaultLocalRegistryPath() { + return getRelativeDir(ConfVars.ZEPPELIN_HELIUM_LOCALREGISTRY_DEFAULT); + } + public String getNotebookAuthorizationPath() { return getRelativeDir(String.format("%s/notebook-authorization.json", getConfDir())); } @@ -540,6 +548,7 @@ public static enum ConfVars { ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING("zeppelin.notebook.autoInterpreterBinding", true), ZEPPELIN_CONF_DIR("zeppelin.conf.dir", "conf"), ZEPPELIN_DEP_LOCALREPO("zeppelin.dep.localrepo", "local-repo"), + ZEPPELIN_HELIUM_LOCALREGISTRY_DEFAULT("zeppelin.helium.localregistry.default", "helium"), // Allows a way to specify a ',' separated list of allowed origins for rest and websockets // i.e. http://localhost:8080 ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"), diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java new file mode 100644 index 00000000000..a07f5f0fde5 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.commons.io.FileUtils; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.resource.DistributedResourcePool; +import org.apache.zeppelin.resource.ResourcePool; +import org.apache.zeppelin.resource.ResourcePoolUtils; +import org.apache.zeppelin.resource.ResourceSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * Manages helium packages + */ +public class Helium { + Logger logger = LoggerFactory.getLogger(Helium.class); + private List registry = new LinkedList(); + + private final HeliumConf heliumConf; + private final String heliumConfPath; + private final String defaultLocalRegistryPath; + private final Gson gson; + + public Helium(String heliumConfPath, String defaultLocalRegistryPath) throws IOException { + this.heliumConfPath = heliumConfPath; + this.defaultLocalRegistryPath = defaultLocalRegistryPath; + + GsonBuilder builder = new GsonBuilder(); + builder.setPrettyPrinting(); + builder.registerTypeAdapter( + HeliumRegistry.class, new HeliumRegistrySerializer()); + gson = builder.create(); + + heliumConf = loadConf(heliumConfPath); + } + + /** + * Add HeliumRegistry + * + * @param registry + */ + public void addRegistry(HeliumRegistry registry) { + synchronized (this.registry) { + this.registry.add(registry); + } + } + + public List getAllRegistry() { + synchronized (this.registry) { + List list = new LinkedList(); + for (HeliumRegistry r : registry) { + list.add(r); + } + return list; + } + } + + private synchronized HeliumConf loadConf(String path) throws IOException { + File heliumConfFile = new File(path); + if (!heliumConfFile.isFile()) { + logger.warn("{} does not exists", path); + HeliumConf conf = new HeliumConf(); + LinkedList defaultRegistry = new LinkedList(); + defaultRegistry.add(new HeliumLocalRegistry("local", defaultLocalRegistryPath)); + conf.setRegistry(defaultRegistry); + this.registry = conf.getRegistry(); + return conf; + } else { + String jsonString = FileUtils.readFileToString(heliumConfFile); + HeliumConf conf = gson.fromJson(jsonString, HeliumConf.class); + this.registry = conf.getRegistry(); + return conf; + } + } + + public synchronized void save() throws IOException { + String jsonString; + synchronized (registry) { + heliumConf.setRegistry(registry); + jsonString = gson.toJson(heliumConf); + } + + File heliumConfFile = new File(heliumConfPath); + if (!heliumConfFile.exists()) { + heliumConfFile.createNewFile(); + } + + FileUtils.writeStringToFile(heliumConfFile, jsonString); + } + + public List getAllPackageInfo() { + List list = new LinkedList(); + synchronized (registry) { + for (HeliumRegistry r : registry) { + try { + for (HeliumPackage pkg : r.getAll()) { + list.add(new HeliumPackageSearchResult(r.name(), pkg)); + } + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + } + } + return list; + } + + public HeliumPackageSuggestion suggestApp(Paragraph paragraph) { + HeliumPackageSuggestion suggestion = new HeliumPackageSuggestion(); + + Interpreter intp = paragraph.getCurrentRepl(); + if (intp == null) { + return suggestion; + } + + ResourcePool resourcePool = intp.getInterpreterGroup().getResourcePool(); + ResourceSet allResources; + + if (resourcePool != null) { + if (resourcePool instanceof DistributedResourcePool) { + allResources = ((DistributedResourcePool) resourcePool).getAll(true); + } else { + allResources = resourcePool.getAll(); + } + } else { + allResources = ResourcePoolUtils.getAllResources(); + } + + for (HeliumPackageSearchResult pkg : getAllPackageInfo()) { + ResourceSet resources = ApplicationLoader.findRequiredResourceSet( + pkg.getPkg().getResources(), + paragraph.getNote().getId(), + paragraph.getId(), + allResources); + if (resources == null) { + continue; + } else { + suggestion.addAvailablePackage(pkg); + } + } + + suggestion.sort(); + return suggestion; + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java new file mode 100644 index 00000000000..6759f979d13 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java @@ -0,0 +1,480 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import com.google.gson.Gson; +import org.apache.thrift.TException; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; +import org.apache.zeppelin.interpreter.thrift.RemoteApplicationResult; +import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; +import org.apache.zeppelin.notebook.*; +import org.apache.zeppelin.scheduler.ExecutorFactory; +import org.apache.zeppelin.scheduler.Job; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.concurrent.ExecutorService; + +/** + * HeliumApplicationFactory + */ +public class HeliumApplicationFactory implements ApplicationEventListener, NotebookEventListener { + private final Logger logger = LoggerFactory.getLogger(HeliumApplicationFactory.class); + private final ExecutorService executor; + private final Gson gson = new Gson(); + private Notebook notebook; + private ApplicationEventListener applicationEventListener; + + public HeliumApplicationFactory() { + executor = ExecutorFactory.singleton().createOrGet( + HeliumApplicationFactory.class.getName(), 10); + } + + private boolean isRemote(InterpreterGroup group) { + return group.getAngularObjectRegistry() instanceof RemoteAngularObjectRegistry; + } + + + /** + * Load pkg and run task + */ + public String loadAndRun(HeliumPackage pkg, Paragraph paragraph) { + ApplicationState appState = paragraph.createOrGetApplicationState(pkg); + onLoad(paragraph.getNote().getId(), paragraph.getId(), appState.getId(), + appState.getHeliumPackage()); + executor.submit(new LoadApplication(appState, pkg, paragraph)); + return appState.getId(); + } + + /** + * Load application and run in the remote process + */ + private class LoadApplication implements Runnable { + private final HeliumPackage pkg; + private final Paragraph paragraph; + private final ApplicationState appState; + + public LoadApplication(ApplicationState appState, HeliumPackage pkg, Paragraph paragraph) { + this.appState = appState; + this.pkg = pkg; + this.paragraph = paragraph; + } + + @Override + public void run() { + try { + // get interpreter process + Interpreter intp = paragraph.getRepl(paragraph.getRequiredReplName()); + InterpreterGroup intpGroup = intp.getInterpreterGroup(); + RemoteInterpreterProcess intpProcess = intpGroup.getRemoteInterpreterProcess(); + if (intpProcess == null) { + throw new ApplicationException("Target interpreter process is not running"); + } + + // load application + load(intpProcess, appState); + + // run application + RunApplication runTask = new RunApplication(paragraph, appState.getId()); + runTask.run(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + + if (appState != null) { + appStatusChange(paragraph, appState.getId(), ApplicationState.Status.ERROR); + appState.setOutput(e.getMessage()); + } + } + } + + private void load(RemoteInterpreterProcess intpProcess, ApplicationState appState) + throws Exception { + + RemoteInterpreterService.Client client = null; + + synchronized (appState) { + if (appState.getStatus() == ApplicationState.Status.LOADED) { + // already loaded + return; + } + + try { + appStatusChange(paragraph, appState.getId(), ApplicationState.Status.LOADING); + String pkgInfo = gson.toJson(pkg); + String appId = appState.getId(); + + client = intpProcess.getClient(); + RemoteApplicationResult ret = client.loadApplication( + appId, + pkgInfo, + paragraph.getNote().getId(), + paragraph.getId()); + + if (ret.isSuccess()) { + appStatusChange(paragraph, appState.getId(), ApplicationState.Status.LOADED); + } else { + throw new ApplicationException(ret.getMsg()); + } + } catch (TException e) { + intpProcess.releaseBrokenClient(client); + throw e; + } finally { + if (client != null) { + intpProcess.releaseClient(client); + } + } + } + } + } + + /** + * Get ApplicationState + * @param paragraph + * @param appId + * @return + */ + public ApplicationState get(Paragraph paragraph, String appId) { + return paragraph.getApplicationState(appId); + } + + /** + * Unload application + * It does not remove ApplicationState + * + * @param paragraph + * @param appId + */ + public void unload(Paragraph paragraph, String appId) { + executor.execute(new UnloadApplication(paragraph, appId)); + } + + /** + * Unload application task + */ + private class UnloadApplication implements Runnable { + private final Paragraph paragraph; + private final String appId; + + public UnloadApplication(Paragraph paragraph, String appId) { + this.paragraph = paragraph; + this.appId = appId; + } + + @Override + public void run() { + ApplicationState appState = null; + try { + appState = paragraph.getApplicationState(appId); + + if (appState == null) { + logger.warn("Can not find {} to unload from {}", appId, paragraph.getId()); + return; + } + if (appState.getStatus() == ApplicationState.Status.UNLOADED) { + // not loaded + return; + } + unload(appState); + } catch (Exception e) { + logger.error(e.getMessage(), e); + if (appState != null) { + appStatusChange(paragraph, appId, ApplicationState.Status.ERROR); + appState.setOutput(e.getMessage()); + } + } + } + + private void unload(ApplicationState appsToUnload) throws ApplicationException { + synchronized (appsToUnload) { + if (appsToUnload.getStatus() != ApplicationState.Status.LOADED) { + throw new ApplicationException( + "Can't unload application status " + appsToUnload.getStatus()); + } + appStatusChange(paragraph, appsToUnload.getId(), ApplicationState.Status.UNLOADING); + Interpreter intp = paragraph.getCurrentRepl(); + if (intp == null) { + throw new ApplicationException("No interpreter found"); + } + + RemoteInterpreterProcess intpProcess = + intp.getInterpreterGroup().getRemoteInterpreterProcess(); + if (intpProcess == null) { + throw new ApplicationException("Target interpreter process is not running"); + } + + RemoteInterpreterService.Client client; + try { + client = intpProcess.getClient(); + } catch (Exception e) { + throw new ApplicationException(e); + } + + try { + RemoteApplicationResult ret = client.unloadApplication(appsToUnload.getId()); + + if (ret.isSuccess()) { + appStatusChange(paragraph, appsToUnload.getId(), ApplicationState.Status.UNLOADED); + } else { + throw new ApplicationException(ret.getMsg()); + } + } catch (TException e) { + intpProcess.releaseBrokenClient(client); + throw new ApplicationException(e); + } finally { + intpProcess.releaseClient(client); + } + } + } + } + + /** + * Run application + * It does not remove ApplicationState + * + * @param paragraph + * @param appId + */ + public void run(Paragraph paragraph, String appId) { + executor.execute(new RunApplication(paragraph, appId)); + } + + /** + * Run application task + */ + private class RunApplication implements Runnable { + private final Paragraph paragraph; + private final String appId; + + public RunApplication(Paragraph paragraph, String appId) { + this.paragraph = paragraph; + this.appId = appId; + } + + @Override + public void run() { + ApplicationState appState = null; + try { + appState = paragraph.getApplicationState(appId); + + if (appState == null) { + logger.warn("Can not find {} to unload from {}", appId, paragraph.getId()); + return; + } + + run(appState); + } catch (Exception e) { + logger.error(e.getMessage(), e); + if (appState != null) { + appStatusChange(paragraph, appId, ApplicationState.Status.UNLOADED); + appState.setOutput(e.getMessage()); + } + } + } + + private void run(ApplicationState app) throws ApplicationException { + synchronized (app) { + if (app.getStatus() != ApplicationState.Status.LOADED) { + throw new ApplicationException( + "Can't run application status " + app.getStatus()); + } + + Interpreter intp = paragraph.getCurrentRepl(); + if (intp == null) { + throw new ApplicationException("No interpreter found"); + } + + RemoteInterpreterProcess intpProcess = + intp.getInterpreterGroup().getRemoteInterpreterProcess(); + if (intpProcess == null) { + throw new ApplicationException("Target interpreter process is not running"); + } + RemoteInterpreterService.Client client = null; + try { + client = intpProcess.getClient(); + } catch (Exception e) { + throw new ApplicationException(e); + } + + try { + RemoteApplicationResult ret = client.runApplication(app.getId()); + + if (ret.isSuccess()) { + // success + } else { + throw new ApplicationException(ret.getMsg()); + } + } catch (TException e) { + intpProcess.releaseBrokenClient(client); + client = null; + throw new ApplicationException(e); + } finally { + if (client != null) { + intpProcess.releaseClient(client); + } + } + } + } + } + + @Override + public void onOutputAppend(String noteId, String paragraphId, String appId, String output) { + ApplicationState appToUpdate = getAppState(noteId, paragraphId, appId); + + if (appToUpdate != null) { + appToUpdate.appendOutput(output); + } else { + logger.error("Can't find app {}", appId); + } + + if (applicationEventListener != null) { + applicationEventListener.onOutputAppend(noteId, paragraphId, appId, output); + } + } + + @Override + public void onOutputUpdated(String noteId, String paragraphId, String appId, String output) { + ApplicationState appToUpdate = getAppState(noteId, paragraphId, appId); + + if (appToUpdate != null) { + appToUpdate.setOutput(output); + } else { + logger.error("Can't find app {}", appId); + } + + if (applicationEventListener != null) { + applicationEventListener.onOutputUpdated(noteId, paragraphId, appId, output); + } + } + + @Override + public void onLoad(String noteId, String paragraphId, String appId, HeliumPackage pkg) { + if (applicationEventListener != null) { + applicationEventListener.onLoad(noteId, paragraphId, appId, pkg); + } + } + + @Override + public void onStatusChange(String noteId, String paragraphId, String appId, String status) { + ApplicationState appToUpdate = getAppState(noteId, paragraphId, appId); + if (appToUpdate != null) { + appToUpdate.setStatus(ApplicationState.Status.valueOf(status)); + } + + if (applicationEventListener != null) { + applicationEventListener.onStatusChange(noteId, paragraphId, appId, status); + } + } + + private void appStatusChange(Paragraph paragraph, + String appId, + ApplicationState.Status status) { + ApplicationState app = paragraph.getApplicationState(appId); + app.setStatus(status); + onStatusChange(paragraph.getNote().getId(), paragraph.getId(), appId, status.toString()); + } + + private ApplicationState getAppState(String noteId, String paragraphId, String appId) { + if (notebook == null) { + return null; + } + + Note note = notebook.getNote(noteId); + if (note == null) { + logger.error("Can't get note {}", noteId); + return null; + } + Paragraph paragraph = note.getParagraph(paragraphId); + if (paragraph == null) { + logger.error("Can't get paragraph {}", paragraphId); + return null; + } + + ApplicationState appFound = paragraph.getApplicationState(appId); + + return appFound; + } + + public Notebook getNotebook() { + return notebook; + } + + public void setNotebook(Notebook notebook) { + this.notebook = notebook; + } + + public ApplicationEventListener getApplicationEventListener() { + return applicationEventListener; + } + + public void setApplicationEventListener(ApplicationEventListener applicationEventListener) { + this.applicationEventListener = applicationEventListener; + } + + @Override + public void onNoteRemove(Note note) { + } + + @Override + public void onNoteCreate(Note note) { + + } + + @Override + public void onUnbindInterpreter(Note note, InterpreterSetting setting) { + for (Paragraph p : note.getParagraphs()) { + Interpreter currentInterpreter = p.getCurrentRepl(); + List infos = setting.getInterpreterInfos(); + for (InterpreterSetting.InterpreterInfo info : infos) { + if (info.getClassName().equals(currentInterpreter.getClassName())) { + onParagraphRemove(p); + break; + } + } + } + } + + @Override + public void onParagraphRemove(Paragraph paragraph) { + List appStates = paragraph.getAllApplicationStates(); + for (ApplicationState app : appStates) { + UnloadApplication unloadJob = new UnloadApplication(paragraph, app.getId()); + unloadJob.run(); + } + } + + @Override + public void onParagraphCreate(Paragraph p) { + + } + + @Override + public void onParagraphStatusChange(Paragraph p, Job.Status status) { + if (status == Job.Status.FINISHED) { + // refresh application + List appStates = p.getAllApplicationStates(); + + for (ApplicationState app : appStates) { + loadAndRun(app.getHeliumPackage(), p); + } + } + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumConf.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumConf.java new file mode 100644 index 00000000000..2a93caa0b57 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumConf.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import java.util.LinkedList; +import java.util.List; + +/** + * Helium config. This object will be persisted to conf/heliumc.conf + */ +public class HeliumConf { + List registry = new LinkedList(); + + public List getRegistry() { + return registry; + } + + public void setRegistry(List registry) { + this.registry = registry; + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java new file mode 100644 index 00000000000..ef2883531e8 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import com.google.gson.Gson; +import com.google.gson.stream.JsonReader; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.net.URI; +import java.util.LinkedList; +import java.util.List; + +/** + * Simple Helium registry on local filesystem + */ +public class HeliumLocalRegistry extends HeliumRegistry { + Logger logger = LoggerFactory.getLogger(HeliumLocalRegistry.class); + + private final Gson gson; + + public HeliumLocalRegistry(String name, String uri) { + super(name, uri); + gson = new Gson(); + + } + + + @Override + public synchronized List getAll() throws IOException { + List result = new LinkedList(); + + File file = new File(uri()); + File [] files = file.listFiles(); + if (files == null) { + return result; + } + + for (File f : files) { + if (f.getName().startsWith(".")) { + continue; + } + + HeliumPackage pkgInfo = readPackageInfo(f); + if (pkgInfo != null) { + result.add(pkgInfo); + } + } + return result; + } + + private HeliumPackage readPackageInfo(File f) { + try { + JsonReader reader = new JsonReader(new StringReader(FileUtils.readFileToString(f))); + reader.setLenient(true); + + return gson.fromJson(reader, HeliumPackage.class); + } catch (IOException e) { + logger.error(e.getMessage(), e); + return null; + } + } + +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumPackageSearchResult.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumPackageSearchResult.java new file mode 100644 index 00000000000..57a9d4512f4 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumPackageSearchResult.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +/** + * search result + */ +public class HeliumPackageSearchResult { + private final String registry; + private final HeliumPackage pkg; + + /** + * Create search result item + * @param registry registry name + * @param pkg package information + */ + public HeliumPackageSearchResult(String registry, HeliumPackage pkg) { + this.registry = registry; + this.pkg = pkg; + } + + public String getRegistry() { + return registry; + } + + public HeliumPackage getPkg() { + return pkg; + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumPackageSuggestion.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumPackageSuggestion.java new file mode 100644 index 00000000000..45c16403bd4 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumPackageSuggestion.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedList; +import java.util.List; + +/** + * Suggested apps + */ +public class HeliumPackageSuggestion { + private final List available = + new LinkedList(); + + /* + * possible future improvement + * provides n - 'favorite' list, based on occurrence of apps in notebook + */ + + public HeliumPackageSuggestion() { + + } + + public void addAvailablePackage(HeliumPackageSearchResult r) { + available.add(r); + + } + + public void sort() { + Collections.sort(available, new Comparator() { + @Override + public int compare(HeliumPackageSearchResult o1, HeliumPackageSearchResult o2) { + return o1.getPkg().getName().compareTo(o2.getPkg().getName()); + } + }); + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumRegistry.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumRegistry.java new file mode 100644 index 00000000000..125ad9298c2 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumRegistry.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import java.io.IOException; +import java.net.URI; +import java.util.List; + +/** + * Helium package registry + */ +public abstract class HeliumRegistry { + private final String name; + private final String uri; + + public HeliumRegistry(String name, String uri) { + this.name = name; + this.uri = uri; + } + public String name() { + return name; + } + public String uri() { + return uri; + } + public abstract List getAll() throws IOException; +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumRegistrySerializer.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumRegistrySerializer.java new file mode 100644 index 00000000000..3abcb9ffe1a --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumRegistrySerializer.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import com.google.gson.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Type; +import java.net.URI; +import java.net.URISyntaxException; + +/** + * HeliumRegistrySerializer (and deserializer) for gson + */ +public class HeliumRegistrySerializer + implements JsonSerializer, JsonDeserializer { + Logger logger = LoggerFactory.getLogger(HeliumRegistrySerializer.class); + + @Override + public HeliumRegistry deserialize(JsonElement json, + Type type, + JsonDeserializationContext jsonDeserializationContext) + throws JsonParseException { + JsonObject jsonObject = json.getAsJsonObject(); + String className = jsonObject.get("class").getAsString(); + String uri = jsonObject.get("uri").getAsString(); + String name = jsonObject.get("name").getAsString(); + + try { + logger.info("Restore helium registry {} {} {}", name, className, uri); + Class cls = + (Class) getClass().getClassLoader().loadClass(className); + Constructor constructor = cls.getConstructor(String.class, String.class); + HeliumRegistry registry = constructor.newInstance(name, uri); + return registry; + } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | + InstantiationException | InvocationTargetException e) { + logger.error(e.getMessage(), e); + return null; + } + } + + @Override + public JsonElement serialize(HeliumRegistry heliumRegistry, + Type type, + JsonSerializationContext jsonSerializationContext) { + JsonObject json = new JsonObject(); + json.addProperty("class", heliumRegistry.getClass().getName()); + json.addProperty("uri", heliumRegistry.uri()); + json.addProperty("name", heliumRegistry.name()); + return json; + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 5595c148c29..47a4325a72e 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -30,7 +30,10 @@ import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.AngularObjectRegistryListener; +import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter; +import org.apache.zeppelin.interpreter.dev.DevInterpreter; +import org.apache.zeppelin.interpreter.dev.ZeppelinDevServer; import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; @@ -82,22 +85,29 @@ public class InterpreterFactory implements InterpreterGroupFactory { private AngularObjectRegistryListener angularObjectRegistryListener; private final RemoteInterpreterProcessListener remoteInterpreterProcessListener; + private final ApplicationEventListener appEventListener; private DependencyResolver depResolver; + private Map env = new HashMap(); + + private Interpreter devInterpreter; + public InterpreterFactory(ZeppelinConfiguration conf, AngularObjectRegistryListener angularObjectRegistryListener, RemoteInterpreterProcessListener remoteInterpreterProcessListener, + ApplicationEventListener appEventListener, DependencyResolver depResolver) throws InterpreterException, IOException, RepositoryException { this(conf, new InterpreterOption(true), angularObjectRegistryListener, - remoteInterpreterProcessListener, depResolver); + remoteInterpreterProcessListener, appEventListener, depResolver); } public InterpreterFactory(ZeppelinConfiguration conf, InterpreterOption defaultOption, AngularObjectRegistryListener angularObjectRegistryListener, RemoteInterpreterProcessListener remoteInterpreterProcessListener, + ApplicationEventListener appEventListener, DependencyResolver depResolver) throws InterpreterException, IOException, RepositoryException { this.conf = conf; @@ -106,6 +116,7 @@ public InterpreterFactory(ZeppelinConfiguration conf, InterpreterOption defaultO this.depResolver = depResolver; this.interpreterRepositories = depResolver.getRepos(); this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; + this.appEventListener = appEventListener; String replsConf = conf.getString(ConfVars.ZEPPELIN_INTERPRETERS); interpreterClassList = replsConf.split(","); String groupOrder = conf.getString(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER); @@ -547,8 +558,18 @@ public void createInterpretersForNote(InterpreterSetting interpreterSetting, Str Interpreter intp; if (option.isRemote()) { - intp = createRemoteRepl(info.getPath(), key, info.getClassName(), properties, - interpreterSetting.id()); + if (option.isConnectExistingProcess()) { + intp = connectToRemoteRepl( + noteId, + info.getClassName(), + option.getHost(), option.getPort(), properties); + } else { + intp = createRemoteRepl(info.getPath(), + key, + info.getClassName(), + properties, + interpreterSetting.id()); + } } else { intp = createRepl(info.getPath(), info.getClassName(), properties); } @@ -850,6 +871,26 @@ private Interpreter createRepl(String dirName, String className, } } + private Interpreter connectToRemoteRepl(String noteId, + String className, + String host, + int port, + Properties property) { + int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT); + int maxPoolSize = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE); + LazyOpenInterpreter intp = new LazyOpenInterpreter( + new RemoteInterpreter( + property, + noteId, + className, + host, + port, + connectTimeout, + maxPoolSize, + remoteInterpreterProcessListener, + appEventListener)); + return intp; + } private Interpreter createRemoteRepl(String interpreterPath, String noteId, String className, Properties property, String interpreterSettingId) { @@ -859,10 +900,14 @@ private Interpreter createRemoteRepl(String interpreterPath, String noteId, Stri updatePropertiesFromRegisteredInterpreter(property, className); - LazyOpenInterpreter intp = new LazyOpenInterpreter(new RemoteInterpreter(property, noteId, - className, conf.getInterpreterRemoteRunnerPath(), interpreterPath, localRepoPath, - connectTimeout, maxPoolSize, remoteInterpreterProcessListener)); - return intp; + + RemoteInterpreter remoteInterpreter = new RemoteInterpreter( + property, noteId, className, conf.getInterpreterRemoteRunnerPath(), + interpreterPath, localRepoPath, connectTimeout, + maxPoolSize, remoteInterpreterProcessListener, appEventListener); + remoteInterpreter.setEnv(env); + + return new LazyOpenInterpreter(remoteInterpreter); } private Properties updatePropertiesFromRegisteredInterpreter(Properties properties, @@ -1037,6 +1082,11 @@ public Interpreter getInterpreter(String noteId, String replName) { } } + // dev interpreter + if (DevInterpreter.isInterpreterName(replName)) { + return getDevInterpreter(); + } + return null; } @@ -1073,4 +1123,34 @@ public void removeRepository(String id) throws IOException { depResolver.delRepo(id); saveToFile(); } + + public Map getEnv() { + return env; + } + + public void setEnv(Map env) { + this.env = env; + } + + + public Interpreter getDevInterpreter() { + if (devInterpreter == null) { + InterpreterOption option = new InterpreterOption(); + option.setRemote(true); + + InterpreterGroup interpreterGroup = createInterpreterGroup("dev", option); + + devInterpreter = connectToRemoteRepl("dev", DevInterpreter.class.getName(), + "localhost", + ZeppelinDevServer.DEFAULT_TEST_INTERPRETER_PORT, + new Properties()); + + LinkedList intpList = new LinkedList(); + intpList.add(devInterpreter); + interpreterGroup.put("dev", intpList); + + devInterpreter.setInterpreterGroup(interpreterGroup); + } + return devInterpreter; + } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java index f9e43abfcae..7aac7816c5e 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java @@ -22,16 +22,13 @@ */ public class InterpreterOption { boolean remote; + String host = null; + int port = -1; boolean perNoteSession; boolean perNoteProcess; boolean isExistingProcess; - String host; - String port; - - - public boolean isExistingProcess() { return isExistingProcess; @@ -41,18 +38,10 @@ public void setExistingProcess(boolean isExistingProcess) { this.isExistingProcess = isExistingProcess; } - public String getPort() { - return port; - } - - public void setPort(String port) { + public void setPort(int port) { this.port = port; } - public String getHost() { - return host; - } - public void setHost(String host) { this.host = host; } @@ -82,6 +71,18 @@ public void setPerNoteSession(boolean perNoteSession) { this.perNoteSession = perNoteSession; } + public boolean isConnectExistingProcess() { + return (host != null && port != -1); + } + + public String getHost() { + return host; + } + + public int getPort() { + return port; + } + public boolean isPerNoteProcess() { return perNoteProcess; } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java new file mode 100644 index 00000000000..1505db9ada3 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.notebook; + +import org.apache.zeppelin.helium.HeliumPackage; +import org.apache.zeppelin.interpreter.InterpreterGroup; + +/** + * Current state of application + */ +public class ApplicationState { + + /** + * Status of Application + */ + public static enum Status { + LOADING, + LOADED, + UNLOADING, + UNLOADED, + ERROR + }; + + Status status = Status.UNLOADED; + + String id; // unique id for this instance. Similar to note id or paragraph id + HeliumPackage pkg; + String output; + + public ApplicationState(String id, HeliumPackage pkg) { + this.id = id; + this.pkg = pkg; + } + + /** + * After ApplicationState is restored from NotebookRepo, + * such as after Zeppelin daemon starts or Notebook import, + * Application status need to be reset. + */ + public void resetStatus() { + if (status != Status.ERROR) { + status = Status.UNLOADED; + } + } + + + @Override + public boolean equals(Object o) { + String compareName; + if (o instanceof ApplicationState) { + return pkg.equals(((ApplicationState) o).getHeliumPackage()); + } else if (o instanceof HeliumPackage) { + return pkg.equals((HeliumPackage) o); + } else { + return false; + } + } + + @Override + public int hashCode() { + return pkg.hashCode(); + } + + public String getId() { + return id; + } + + public void setStatus(Status status) { + this.status = status; + } + + public Status getStatus() { + return status; + } + + public String getOutput() { + return output; + } + + public void setOutput(String output) { + this.output = output; + } + + public synchronized void appendOutput(String output) { + if (this.output == null) { + this.output = output; + } else { + this.output += output; + } + } + + public HeliumPackage getHeliumPackage() { + return pkg; + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index e57ed9bf545..0a8a45d05e2 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -30,6 +30,7 @@ import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.Input; +import org.apache.zeppelin.helium.HeliumApplicationFactory; import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; @@ -51,7 +52,7 @@ /** * Binded interpreters for a note */ -public class Note implements Serializable, JobListener { +public class Note implements Serializable, ParagraphJobListener { static Logger logger = LoggerFactory.getLogger(Note.class); private static final long serialVersionUID = 7920699076577612429L; @@ -78,6 +79,7 @@ public class Note implements Serializable, JobListener { private transient NotebookRepo repo; private transient SearchService index; private transient ScheduledFuture delayedPersist; + private transient NoteEventListener noteEventListener; private transient Credentials credentials; /** @@ -98,11 +100,13 @@ public class Note implements Serializable, JobListener { public Note() {} public Note(NotebookRepo repo, InterpreterFactory factory, - JobListenerFactory jlFactory, SearchService noteIndex, Credentials credentials) { + JobListenerFactory jlFactory, SearchService noteIndex, Credentials credentials, + NoteEventListener noteEventListener) { this.repo = repo; this.factory = factory; this.jobListenerFactory = jlFactory; this.index = noteIndex; + this.noteEventListener = noteEventListener; this.credentials = credentials; generateId(); } @@ -202,6 +206,9 @@ public Paragraph addParagraph() { synchronized (paragraphs) { paragraphs.add(p); } + if (noteEventListener != null) { + noteEventListener.onParagraphCreate(p); + } return p; } @@ -239,6 +246,9 @@ public void addCloneParagraph(Paragraph srcParagraph) { synchronized (paragraphs) { paragraphs.add(newParagraph); } + if (noteEventListener != null) { + noteEventListener.onParagraphCreate(newParagraph); + } } /** @@ -252,6 +262,9 @@ public Paragraph insertParagraph(int index) { synchronized (paragraphs) { paragraphs.add(index, p); } + if (noteEventListener != null) { + noteEventListener.onParagraphCreate(p); + } return p; } @@ -287,12 +300,14 @@ public Paragraph removeParagraph(String paragraphId) { if (p.getId().equals(paragraphId)) { index.deleteIndexDoc(this, p); i.remove(); + + if (noteEventListener != null) { + noteEventListener.onParagraphRemove(p); + } return p; } } } - - return null; } @@ -431,9 +446,11 @@ public void runAll() { AuthenticationInfo authenticationInfo = new AuthenticationInfo(); authenticationInfo.setUser(cronExecutingUser); p.setAuthenticationInfo(authenticationInfo); + p.setInterpreterFactory(factory); p.setListener(jobListenerFactory.getParagraphJobListener(this)); Interpreter intp = factory.getInterpreter(getId(), p.getRequiredReplName()); + intp.getScheduler().submit(p); } } @@ -450,6 +467,7 @@ public void run(String paragraphId) { p.setListener(jobListenerFactory.getParagraphJobListener(this)); String requiredReplName = p.getRequiredReplName(); Interpreter intp = factory.getInterpreter(getId(), requiredReplName); + if (intp == null) { // TODO(jongyoul): Make "%jdbc" configurable from JdbcInterpreter if (conf.getUseJdbcAlias() && null != (intp = factory.getInterpreter(getId(), "jdbc"))) { @@ -526,8 +544,25 @@ private void removeAllAngularObjectInParagraph(String paragraphId) { if (registry instanceof RemoteAngularObjectRegistry) { // remove paragraph scope object ((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id, paragraphId); + + // remove app scope object + List appStates = getParagraph(paragraphId).getAllApplicationStates(); + if (appStates != null) { + for (ApplicationState app : appStates) { + ((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess( + id, app.getId()); + } + } } else { registry.removeAll(id, paragraphId); + + // remove app scope object + List appStates = getParagraph(paragraphId).getAllApplicationStates(); + if (appStates != null) { + for (ApplicationState app : appStates) { + registry.removeAll(id, app.getId()); + } + } } } } @@ -624,12 +659,67 @@ public String getLastInterpreterName() { @Override public void beforeStatusChange(Job job, Status before, Status after) { + if (jobListenerFactory != null) { + ParagraphJobListener listener = jobListenerFactory.getParagraphJobListener(this); + if (listener != null) { + listener.beforeStatusChange(job, before, after); + } + } } @Override public void afterStatusChange(Job job, Status before, Status after) { + if (jobListenerFactory != null) { + ParagraphJobListener listener = jobListenerFactory.getParagraphJobListener(this); + if (listener != null) { + listener.afterStatusChange(job, before, after); + } + } + + if (noteEventListener != null) { + noteEventListener.onParagraphStatusChange((Paragraph) job, after); + } } @Override - public void onProgressUpdate(Job job, int progress) {} + public void onProgressUpdate(Job job, int progress) { + if (jobListenerFactory != null) { + ParagraphJobListener listener = jobListenerFactory.getParagraphJobListener(this); + if (listener != null) { + listener.onProgressUpdate(job, progress); + } + } + } + + + @Override + public void onOutputAppend(Paragraph paragraph, InterpreterOutput out, String output) { + if (jobListenerFactory != null) { + ParagraphJobListener listener = jobListenerFactory.getParagraphJobListener(this); + if (listener != null) { + listener.onOutputAppend(paragraph, out, output); + } + } + } + + @Override + public void onOutputUpdate(Paragraph paragraph, InterpreterOutput out, String output) { + if (jobListenerFactory != null) { + ParagraphJobListener listener = jobListenerFactory.getParagraphJobListener(this); + if (listener != null) { + listener.onOutputUpdate(paragraph, out, output); + } + } + } + + + + public NoteEventListener getNoteEventListener() { + return noteEventListener; + } + + public void setNoteEventListener(NoteEventListener noteEventListener) { + this.noteEventListener = noteEventListener; + } + } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteEventListener.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteEventListener.java new file mode 100644 index 00000000000..5f98f7017dc --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteEventListener.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.notebook; + +import org.apache.zeppelin.scheduler.Job; + +/** + * NoteEventListener + */ +public interface NoteEventListener { + public void onParagraphRemove(Paragraph p); + public void onParagraphCreate(Paragraph p); + public void onParagraphStatusChange(Paragraph p, Job.Status status); +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index 3243ba774ab..ab2ce5dd061 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -41,6 +41,7 @@ import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.apache.zeppelin.notebook.repo.NotebookRepoSync; import org.apache.zeppelin.resource.ResourcePoolUtils; +import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.SchedulerFactory; import org.apache.zeppelin.search.SearchService; import org.apache.zeppelin.user.AuthenticationInfo; @@ -64,7 +65,7 @@ /** * Collection of Notes. */ -public class Notebook { +public class Notebook implements NoteEventListener { static Logger logger = LoggerFactory.getLogger(Notebook.class); @SuppressWarnings("unused") @Deprecated //TODO(bzz): remove unused @@ -80,6 +81,8 @@ public class Notebook { private NotebookRepo notebookRepo; private SearchService notebookIndex; private NotebookAuthorization notebookAuthorization; + private final List notebookEventListeners = + Collections.synchronizedList(new LinkedList()); private Credentials credentials; /** @@ -151,7 +154,13 @@ public Note createNote(AuthenticationInfo subject) throws IOException { */ public Note createNote(List interpreterIds, AuthenticationInfo subject) throws IOException { - Note note = new Note(notebookRepo, replFactory, jobListenerFactory, notebookIndex, credentials); + Note note = new Note( + notebookRepo, + replFactory, + jobListenerFactory, + notebookIndex, + credentials, + this); synchronized (notes) { notes.put(note.id(), note); } @@ -162,6 +171,7 @@ public Note createNote(List interpreterIds, AuthenticationInfo subject) notebookIndex.addIndexDoc(note); note.persist(subject); + fireNoteCreateEvent(note); return note; } @@ -216,7 +226,7 @@ public Note importNote(String sourceJson, String noteName, AuthenticationInfo su logger.error(e.toString(), e); throw e; } - + return newNote; } @@ -256,6 +266,13 @@ public void bindInterpretersToNote(String id, List interpreterSettingIds) throws IOException { Note note = getNote(id); if (note != null) { + List currentBindings = replFactory.getInterpreterSettings(id); + for (InterpreterSetting setting : currentBindings) { + if (!interpreterSettingIds.contains(setting.id())) { + fireUnbindInterpreter(note, setting); + } + } + replFactory.setInterpreters(note.getId(), interpreterSettingIds); // comment out while note.getNoteReplLoader().setInterpreters(...) do the same // replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds); @@ -303,6 +320,15 @@ public void removeNote(String id, AuthenticationInfo subject) { // remove paragraph scope object for (Paragraph p : note.getParagraphs()) { ((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id, p.getId()); + + // remove app scope object + List appStates = p.getAllApplicationStates(); + if (appStates != null) { + for (ApplicationState app : appStates) { + ((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess( + id, app.getId()); + } + } } // remove notebook scope object ((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id, null); @@ -310,6 +336,14 @@ public void removeNote(String id, AuthenticationInfo subject) { // remove paragraph scope object for (Paragraph p : note.getParagraphs()) { registry.removeAll(id, p.getId()); + + // remove app scope object + List appStates = p.getAllApplicationStates(); + if (appStates != null) { + for (ApplicationState app : appStates) { + registry.removeAll(id, app.getId()); + } + } } // remove notebook scope object registry.removeAll(id, null); @@ -318,6 +352,8 @@ public void removeNote(String id, AuthenticationInfo subject) { ResourcePoolUtils.removeResourcesBelongsToNote(id); + fireNoteRemoveEvent(note); + try { note.unpersist(subject); } catch (IOException e) { @@ -379,6 +415,8 @@ private Note loadNoteFromRepo(String id, AuthenticationInfo subject) { } } + note.setNoteEventListener(this); + synchronized (notes) { notes.put(note.id(), note); refreshCron(note.id()); @@ -402,6 +440,7 @@ private Note loadNoteFromRepo(String id, AuthenticationInfo subject) { } } } + return note; } @@ -737,4 +776,46 @@ public void close() { this.notebookIndex.close(); } + public void addNotebookEventListener(NotebookEventListener listener) { + notebookEventListeners.add(listener); + } + + private void fireNoteCreateEvent(Note note) { + for (NotebookEventListener listener : notebookEventListeners) { + listener.onNoteCreate(note); + } + } + + private void fireNoteRemoveEvent(Note note) { + for (NotebookEventListener listener : notebookEventListeners) { + listener.onNoteRemove(note); + } + } + + private void fireUnbindInterpreter(Note note, InterpreterSetting setting) { + for (NotebookEventListener listener : notebookEventListeners) { + listener.onUnbindInterpreter(note, setting); + } + } + + @Override + public void onParagraphRemove(Paragraph p) { + for (NotebookEventListener listener : notebookEventListeners) { + listener.onParagraphRemove(p); + } + } + + @Override + public void onParagraphCreate(Paragraph p) { + for (NotebookEventListener listener : notebookEventListeners) { + listener.onParagraphCreate(p); + } + } + + @Override + public void onParagraphStatusChange(Paragraph p, Job.Status status) { + for (NotebookEventListener listener : notebookEventListeners) { + listener.onParagraphStatusChange(p, status); + } + } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookEventListener.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookEventListener.java new file mode 100644 index 00000000000..904eba0c574 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookEventListener.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.notebook; + +import org.apache.zeppelin.interpreter.InterpreterSetting; + +/** + * Notebook event + */ +public interface NotebookEventListener extends NoteEventListener { + public void onNoteRemove(Note note); + public void onNoteCreate(Note note); + + public void onUnbindInterpreter(Note note, InterpreterSetting setting); +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index df4765d8cee..d1a782420e7 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -19,6 +19,7 @@ import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.helium.HeliumPackage; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.user.Credentials; @@ -61,6 +62,11 @@ public class Paragraph extends Job implements Serializable, Cloneable { private Map config; // paragraph configs like isOpen, colWidth, etc public final GUI settings; // form and parameter settings + /** + * Applicaiton states in this paragraph + */ + private final List apps = new LinkedList(); + @VisibleForTesting Paragraph() { super(generateId(), null); @@ -230,7 +236,7 @@ public int progress() { String replName = getRequiredReplName(); Interpreter repl = getRepl(replName); if (repl != null) { - return repl.getProgress(getInterpreterContext()); + return repl.getProgress(getInterpreterContext(null)); } else { return 0; } @@ -283,7 +289,6 @@ protected Object jobRun() throws Throwable { context.out.flush(); InterpreterResult.Type outputType = context.out.getType(); byte[] interpreterOutput = context.out.toByteArray(); - context.out.clear(); if (interpreterOutput != null && interpreterOutput.length > 0) { message = new String(interpreterOutput); @@ -323,12 +328,44 @@ protected boolean jobAbort() { if (job != null) { job.setStatus(Status.ABORT); } else { - repl.cancel(getInterpreterContext()); + repl.cancel(getInterpreterContext(null)); } return true; } private InterpreterContext getInterpreterContext() { + final Paragraph self = this; + + return getInterpreterContext(new InterpreterOutput(new InterpreterOutputListener() { + @Override + public void onAppend(InterpreterOutput out, byte[] line) { + updateParagraphResult(out); + ((ParagraphJobListener) getListener()).onOutputAppend(self, out, new String(line)); + } + + @Override + public void onUpdate(InterpreterOutput out, byte[] output) { + updateParagraphResult(out); + ((ParagraphJobListener) getListener()).onOutputUpdate(self, out, + new String(output)); + } + + private void updateParagraphResult(InterpreterOutput out) { + // update paragraph result + Throwable t = null; + String message = null; + try { + message = new String(out.toByteArray()); + } catch (IOException e) { + logger().error(e.getMessage(), e); + t = e; + } + setReturn(new InterpreterResult(Code.SUCCESS, out.getType(), message), t); + } + })); + } + + private InterpreterContext getInterpreterContext(InterpreterOutput output) { AngularObjectRegistry registry = null; ResourcePool resourcePool = null; @@ -363,33 +400,7 @@ private InterpreterContext getInterpreterContext() { registry, resourcePool, runners, - new InterpreterOutput(new InterpreterOutputListener() { - @Override - public void onAppend(InterpreterOutput out, byte[] line) { - updateParagraphResult(out); - ((ParagraphJobListener) getListener()).onOutputAppend(self, out, new String(line)); - } - - @Override - public void onUpdate(InterpreterOutput out, byte[] output) { - updateParagraphResult(out); - ((ParagraphJobListener) getListener()).onOutputUpdate(self, out, - new String(output)); - } - - private void updateParagraphResult(InterpreterOutput out) { - // update paragraph result - Throwable t = null; - String message = null; - try { - message = new String(out.toByteArray()); - } catch (IOException e) { - logger().error(e.getMessage(), e); - t = e; - } - setReturn(new InterpreterResult(Code.SUCCESS, out.getType(), message), t); - } - })); + output); return interpreterContext; } @@ -433,6 +444,44 @@ public Object clone() throws CloneNotSupportedException { return paraClone; } + private String getApplicationId(HeliumPackage pkg) { + return "app_" + getNote().getId() + "-" + getId() + pkg.getName().replaceAll("\\.", "_"); + } + + public ApplicationState createOrGetApplicationState(HeliumPackage pkg) { + synchronized (apps) { + for (ApplicationState as : apps) { + if (as.equals(pkg)) { + return as; + } + } + + String appId = getApplicationId(pkg); + ApplicationState appState = new ApplicationState(appId, pkg); + apps.add(appState); + return appState; + } + } + + + public ApplicationState getApplicationState(String appId) { + synchronized (apps) { + for (ApplicationState as : apps) { + if (as.getId().equals(appId)) { + return as; + } + } + } + + return null; + } + + public List getAllApplicationStates() { + synchronized (apps) { + return new LinkedList(apps); + } + } + String extractVariablesFromAngularRegistry(String scriptBody, Map inputs, AngularObjectRegistry angularRegistry) { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java index 152e0875a15..a74e6c75c8f 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java @@ -36,6 +36,7 @@ import org.apache.commons.vfs2.VFS; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.notebook.ApplicationState; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.NoteInfo; import org.apache.zeppelin.notebook.Paragraph; @@ -172,6 +173,15 @@ private Note getNote(FileObject noteDir) throws IOException { if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) { p.setStatus(Status.ABORT); } + + List appStates = p.getAllApplicationStates(); + if (appStates != null) { + for (ApplicationState app : appStates) { + if (app.getStatus() != ApplicationState.Status.ERROR) { + app.setStatus(ApplicationState.Status.UNLOADED); + } + } + } } return note; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java index 320709e4ff0..08b32359226 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java @@ -111,9 +111,15 @@ public static enum OP { CONFIGURATIONS_INFO, // [s-c] all key/value pairs of configurations // @param settings serialized Map object - CHECKPOINT_NOTEBOOK, // [c-s] checkpoint notebook to storage repository - // @param noteId - // @param checkpointName + CHECKPOINT_NOTEBOOK, // [c-s] checkpoint notebook to storage repository + // @param noteId + // @param checkpointName + + APP_APPEND_OUTPUT, // [s-c] append output + APP_UPDATE_OUTPUT, // [s-c] update (replace) output + APP_LOAD, // [s-c] on app load + APP_STATUS_CHANGE, // [s-c] on app status change + LIST_NOTEBOOK_JOBS, // [c-s] get notebook job management infomations LIST_UPDATE_NOTEBOOK_JOBS // [c-s] get job management informations for until unixtime // @param unixTime diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java new file mode 100644 index 00000000000..602f38425ed --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java @@ -0,0 +1,323 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import org.apache.commons.io.FileUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.dep.DependencyResolver; +import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.interpreter.mock.MockInterpreter1; +import org.apache.zeppelin.interpreter.mock.MockInterpreter2; +import org.apache.zeppelin.notebook.*; +import org.apache.zeppelin.notebook.repo.VFSNotebookRepo; +import org.apache.zeppelin.scheduler.ExecutorFactory; +import org.apache.zeppelin.scheduler.Job; +import org.apache.zeppelin.scheduler.SchedulerFactory; +import org.apache.zeppelin.search.SearchService; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +public class HeliumApplicationFactoryTest implements JobListenerFactory { + private File tmpDir; + private File notebookDir; + private ZeppelinConfiguration conf; + private SchedulerFactory schedulerFactory; + private DependencyResolver depResolver; + private InterpreterFactory factory; + private VFSNotebookRepo notebookRepo; + private Notebook notebook; + private HeliumApplicationFactory heliumAppFactory; + + @Before + public void setUp() throws Exception { + tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); + tmpDir.mkdirs(); + File confDir = new File(tmpDir, "conf"); + confDir.mkdirs(); + notebookDir = new File(tmpDir + "/notebook"); + notebookDir.mkdirs(); + + File home = new File(getClass().getClassLoader().getResource("note").getFile()) // zeppelin/zeppelin-zengine/target/test-classes/note + .getParentFile() // zeppelin/zeppelin-zengine/target/test-classes + .getParentFile() // zeppelin/zeppelin-zengine/target + .getParentFile() // zeppelin/zeppelin-zengine + .getParentFile(); // zeppelin + + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), home.getAbsolutePath()); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(), tmpDir.getAbsolutePath() + "/conf"); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath()); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1,org.apache.zeppelin.interpreter.mock.MockInterpreter2"); + + conf = new ZeppelinConfiguration(); + + this.schedulerFactory = new SchedulerFactory(); + + MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1"); + MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); + + + heliumAppFactory = new HeliumApplicationFactory(); + depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); + factory = new InterpreterFactory(conf, + new InterpreterOption(true), null, null, heliumAppFactory, depResolver); + HashMap env = new HashMap(); + env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath()); + factory.setEnv(env); + + SearchService search = mock(SearchService.class); + notebookRepo = new VFSNotebookRepo(conf); + NotebookAuthorization notebookAuthorization = new NotebookAuthorization(conf); + notebook = new Notebook( + conf, + notebookRepo, + schedulerFactory, + factory, + this, + search, + notebookAuthorization, + null); + + heliumAppFactory.setNotebook(notebook); + + notebook.addNotebookEventListener(heliumAppFactory); + } + + @After + public void tearDown() throws Exception { + List settings = factory.get(); + for (InterpreterSetting setting : settings) { + for (InterpreterGroup intpGroup : setting.getAllInterpreterGroups()) { + intpGroup.close(); + } + } + + FileUtils.deleteDirectory(tmpDir); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(), + ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getStringValue()); + } + + + @Test + public void testLoadRunUnloadApplication() + throws IOException, ApplicationException, InterruptedException { + // given + HeliumPackage pkg1 = new HeliumPackage(HeliumPackage.Type.APPLICATION, + "name1", + "desc1", + "", + HeliumTestApplication.class.getName(), + new String[][]{}); + + Note note1 = notebook.createNote(null); + factory.setInterpreters(note1.getId(),factory.getDefaultInterpreterSettingList()); + + Paragraph p1 = note1.addParagraph(); + + // make sure interpreter process running + p1.setText("%mock1 job"); + note1.run(p1.getId()); + while(p1.isTerminated()==false || p1.getResult()==null) Thread.yield(); + + assertEquals("repl1: job", p1.getResult().message()); + + // when + assertEquals(0, p1.getAllApplicationStates().size()); + String appId = heliumAppFactory.loadAndRun(pkg1, p1); + assertEquals(1, p1.getAllApplicationStates().size()); + ApplicationState app = p1.getApplicationState(appId); + Thread.sleep(500); // wait for enough time + + // then + assertEquals("Hello world 1", app.getOutput()); + + // when + heliumAppFactory.run(p1, appId); + Thread.sleep(500); // wait for enough time + + // then + assertEquals("Hello world 2", app.getOutput()); + + // clean + heliumAppFactory.unload(p1, appId); + notebook.removeNote(note1.getId(), null); + } + + @Test + public void testUnloadOnParagraphRemove() throws IOException { + // given + HeliumPackage pkg1 = new HeliumPackage(HeliumPackage.Type.APPLICATION, + "name1", + "desc1", + "", + HeliumTestApplication.class.getName(), + new String[][]{}); + + Note note1 = notebook.createNote(null); + factory.setInterpreters(note1.id(), factory.getDefaultInterpreterSettingList()); + + Paragraph p1 = note1.addParagraph(); + + // make sure interpreter process running + p1.setText("%mock1 job"); + note1.run(p1.getId()); + while(p1.isTerminated()==false || p1.getResult()==null) Thread.yield(); + + assertEquals(0, p1.getAllApplicationStates().size()); + String appId = heliumAppFactory.loadAndRun(pkg1, p1); + ApplicationState app = p1.getApplicationState(appId); + while (app.getStatus() != ApplicationState.Status.LOADED) { + Thread.yield(); + } + + // when remove paragraph + note1.removeParagraph(p1.getId()); + + // then + assertEquals(ApplicationState.Status.UNLOADED, app.getStatus()); + + // clean + notebook.removeNote(note1.getId(), null); + } + + + @Test + public void testUnloadOnInterpreterUnbind() throws IOException { + // given + HeliumPackage pkg1 = new HeliumPackage(HeliumPackage.Type.APPLICATION, + "name1", + "desc1", + "", + HeliumTestApplication.class.getName(), + new String[][]{}); + + Note note1 = notebook.createNote(null); + notebook.bindInterpretersToNote(note1.id(), factory.getDefaultInterpreterSettingList()); + + Paragraph p1 = note1.addParagraph(); + + // make sure interpreter process running + p1.setText("%mock1 job"); + note1.run(p1.getId()); + while(p1.isTerminated()==false || p1.getResult()==null) Thread.yield(); + + assertEquals(0, p1.getAllApplicationStates().size()); + String appId = heliumAppFactory.loadAndRun(pkg1, p1); + ApplicationState app = p1.getApplicationState(appId); + while (app.getStatus() != ApplicationState.Status.LOADED) { + Thread.yield(); + } + + // when unbind interpreter + notebook.bindInterpretersToNote(note1.id(), new LinkedList()); + + // then + assertEquals(ApplicationState.Status.UNLOADED, app.getStatus()); + + // clean + notebook.removeNote(note1.getId(), null); + } + + + @Test + public void testUnloadOnInterpreterRestart() throws IOException { + // given + HeliumPackage pkg1 = new HeliumPackage(HeliumPackage.Type.APPLICATION, + "name1", + "desc1", + "", + HeliumTestApplication.class.getName(), + new String[][]{}); + + Note note1 = notebook.createNote(null); + notebook.bindInterpretersToNote(note1.id(), factory.getDefaultInterpreterSettingList()); + String mock1IntpSettingId = null; + for (InterpreterSetting setting : notebook.getBindedInterpreterSettings(note1.id())) { + if (setting.getName().equals("mock1")) { + mock1IntpSettingId = setting.id(); + break; + } + } + + Paragraph p1 = note1.addParagraph(); + + // make sure interpreter process running + p1.setText("%mock1 job"); + note1.run(p1.getId()); + while(p1.isTerminated()==false || p1.getResult()==null) Thread.yield(); + assertEquals(0, p1.getAllApplicationStates().size()); + String appId = heliumAppFactory.loadAndRun(pkg1, p1); + ApplicationState app = p1.getApplicationState(appId); + while (app.getStatus() != ApplicationState.Status.LOADED) { + Thread.yield(); + } + // wait until application is executed + while (!"Hello world 1".equals(app.getOutput())) { + Thread.yield(); + } + // when restart interpreter + factory.restart(mock1IntpSettingId); + while (app.getStatus() == ApplicationState.Status.LOADED) { + Thread.yield(); + } + // then + assertEquals(ApplicationState.Status.UNLOADED, app.getStatus()); + + // clean + notebook.removeNote(note1.getId(), null); + } + + @Override + public ParagraphJobListener getParagraphJobListener(Note note) { + return new ParagraphJobListener() { + @Override + public void onOutputAppend(Paragraph paragraph, InterpreterOutput out, String output) { + } + + @Override + public void onOutputUpdate(Paragraph paragraph, InterpreterOutput out, String output) { + + } + + @Override + public void onProgressUpdate(Job job, int progress) { + + } + + @Override + public void beforeStatusChange(Job job, Job.Status before, Job.Status after) { + + } + + @Override + public void afterStatusChange(Job job, Job.Status before, Job.Status after) { + + } + }; + } +} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumLocalRegistryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumLocalRegistryTest.java new file mode 100644 index 00000000000..e90cbc30a73 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumLocalRegistryTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import com.google.gson.Gson; +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + +public class HeliumLocalRegistryTest { + private File tmpDir; + + @Before + public void setUp() throws Exception { + tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis()); + tmpDir.mkdirs(); + } + + @After + public void tearDown() throws IOException { + FileUtils.deleteDirectory(tmpDir); + } + + @Test + public void testGetAllPackage() throws IOException { + // given + File r1Path = new File(tmpDir, "r1"); + HeliumLocalRegistry r1 = new HeliumLocalRegistry("r1", r1Path.getAbsolutePath()); + assertEquals(0, r1.getAll().size()); + + // when + Gson gson = new Gson(); + HeliumPackage pkg1 = new HeliumPackage(HeliumPackage.Type.APPLICATION, + "app1", + "desc1", + "artifact1", + "classname1", + new String[][]{}); + FileUtils.writeStringToFile(new File(r1Path, "pkg1.json"), gson.toJson(pkg1)); + + // then + assertEquals(1, r1.getAll().size()); + } +} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java new file mode 100644 index 00000000000..1ed31c5bb95 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class HeliumTest { + private File tmpDir; + private File localRegistryPath; + + @Before + public void setUp() throws Exception { + tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis()); + tmpDir.mkdirs(); + localRegistryPath = new File(tmpDir, "helium"); + localRegistryPath.mkdirs(); + } + + @After + public void tearDown() throws IOException { + FileUtils.deleteDirectory(tmpDir); + } + + @Test + public void testSaveLoadConf() throws IOException, URISyntaxException { + // given + File heliumConf = new File(tmpDir, "helium.conf"); + Helium helium = new Helium(heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath()); + assertFalse(heliumConf.exists()); + HeliumTestRegistry registry1 = new HeliumTestRegistry("r1", "r1"); + helium.addRegistry(registry1); + assertEquals(2, helium.getAllRegistry().size()); + assertEquals(0, helium.getAllPackageInfo().size()); + + // when + helium.save(); + + // then + assertTrue(heliumConf.exists()); + + // then + Helium heliumRestored = new Helium(heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath()); + assertEquals(2, heliumRestored.getAllRegistry().size()); + } + + @Test + public void testRestoreRegistryInstances() throws IOException, URISyntaxException { + File heliumConf = new File(tmpDir, "helium.conf"); + Helium helium = new Helium(heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath()); + HeliumTestRegistry registry1 = new HeliumTestRegistry("r1", "r1"); + HeliumTestRegistry registry2 = new HeliumTestRegistry("r2", "r2"); + helium.addRegistry(registry1); + helium.addRegistry(registry2); + + // when + registry1.add(new HeliumPackage( + HeliumPackage.Type.APPLICATION, + "name1", + "desc1", + "artifact1", + "className1", + new String[][]{})); + + registry2.add(new HeliumPackage( + HeliumPackage.Type.APPLICATION, + "name2", + "desc2", + "artifact2", + "className2", + new String[][]{})); + + // then + assertEquals(2, helium.getAllPackageInfo().size()); + } +} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestApplication.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestApplication.java new file mode 100644 index 00000000000..8b1d14ff17b --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestApplication.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import org.apache.zeppelin.resource.ResourceSet; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; + +public class HeliumTestApplication extends Application { + AtomicInteger numRun = new AtomicInteger(0); + public HeliumTestApplication(ApplicationContext context) { + super(context); + } + + @Override + public void run(ResourceSet args) throws ApplicationException { + try { + context().out.clear(); + context().out.write("Hello world " + numRun.incrementAndGet()); + context().out.flush(); + } catch (IOException e) { + throw new ApplicationException(e); + } + } + + @Override + public void unload() throws ApplicationException { + + } +} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestRegistry.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestRegistry.java new file mode 100644 index 00000000000..d99f73f2817 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestRegistry.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import java.io.IOException; +import java.net.URI; +import java.util.LinkedList; +import java.util.List; + +public class HeliumTestRegistry extends HeliumRegistry { + List infos = new LinkedList(); + + public HeliumTestRegistry(String name, String uri) { + super(name, uri); + } + + @Override + public List getAll() throws IOException { + return infos; + } + + public void add(HeliumPackage info) { + infos.add(info); + } +} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java index 3d9ee6ff5cb..9df494f8519 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Properties; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang.NullArgumentException; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; @@ -57,35 +58,28 @@ public void setUp() throws Exception { System.setProperty(ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1,org.apache.zeppelin.interpreter.mock.MockInterpreter2"); conf = new ZeppelinConfiguration(); depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver); context = new InterpreterContext("note", "id", "title", "text", null, null, null, null, null, null, null); - } @After public void tearDown() throws Exception { - delete(tmpDir); + FileUtils.deleteDirectory(tmpDir); } - private void delete(File file){ - if(file.isFile()) file.delete(); - else if(file.isDirectory()){ - File [] files = file.listFiles(); - if(files!=null && files.length>0){ - for(File f : files){ - delete(f); - } + @Test + public void testBasic() { + List all = factory.get(); + InterpreterSetting mock1Setting = null; + for (InterpreterSetting setting : all) { + if (setting.getName().equals("mock1")) { + mock1Setting = setting; + break; } - file.delete(); } - } - @Test - public void testBasic() { - List all = factory.getDefaultInterpreterSettingList(); - InterpreterSetting setting = factory.get(all.get(0)); - InterpreterGroup interpreterGroup = setting.getInterpreterGroup("sharedProcess"); - factory.createInterpretersForNote(setting, "sharedProcess", "session"); + InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("sharedProcess"); + factory.createInterpretersForNote(mock1Setting, "sharedProcess", "session"); // get interpreter assertNotNull("get Interpreter", interpreterGroup.get("session").get(0)); @@ -94,15 +88,15 @@ public void testBasic() { assertNull(factory.get("unknown")); // restart interpreter - factory.restart(all.get(0)); - assertNull(setting.getInterpreterGroup("sharedProcess").get("session")); + factory.restart(mock1Setting.id()); + assertNull(mock1Setting.getInterpreterGroup("sharedProcess").get("session")); } @Test public void testFactoryDefaultList() throws IOException, RepositoryException { // get default settings List all = factory.getDefaultInterpreterSettingList(); - assertEquals(2, all.size()); + assertTrue(factory.getRegisteredInterpreterList().size() >= all.size()); } @Test @@ -125,15 +119,15 @@ public void testExceptions() throws InterpreterException, IOException, Repositor @Test public void testSaveLoad() throws IOException, RepositoryException { // interpreter settings - assertEquals(2, factory.get().size()); + int numInterpreters = factory.get().size(); // check if file saved assertTrue(new File(conf.getInterpreterSettingPath()).exists()); factory.add("newsetting", "mock1", new LinkedList(), new InterpreterOption(false), new Properties()); - assertEquals(3, factory.get().size()); + assertEquals(numInterpreters + 1, factory.get().size()); InterpreterFactory factory2 = new InterpreterFactory(conf, null, null, null, depResolver); - assertEquals(3, factory2.get().size()); + assertEquals(numInterpreters + 1, factory2.get().size()); } } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java index 794ab6ccba2..d690b670caf 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java @@ -1,19 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ package org.apache.zeppelin.interpreter.mock; @@ -30,7 +30,7 @@ import org.apache.zeppelin.scheduler.SchedulerFactory; public class MockInterpreter1 extends Interpreter{ - Map vars = new HashMap(); +Map vars = new HashMap(); public MockInterpreter1(Properties property) { super(property); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java index 7d9071c07cd..245089963ea 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java @@ -60,7 +60,7 @@ public void setUp() throws Exception { MockInterpreter2.register("mock2", "group2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver); } @After diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java index 2ca817fb2e0..9a737e69347 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java @@ -57,6 +57,9 @@ public class NoteTest { @Mock Scheduler scheduler; + @Mock + NoteEventListener noteEventListener; + @Mock InterpreterFactory interpreterFactory; @@ -66,7 +69,8 @@ public void runNormalTest() { when(interpreter.getScheduler()).thenReturn(scheduler); String pText = "%spark sc.version"; - Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials); + Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); + Paragraph p = note.addParagraph(); p.setText(pText); note.run(p.getId()); @@ -85,7 +89,8 @@ public void runJdbcTest() { when(interpreter.getScheduler()).thenReturn(scheduler); String pText = "%mysql show databases"; - Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials); + + Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); Paragraph p = note.addParagraph(); p.setText(pText); note.run(p.getId()); @@ -103,7 +108,7 @@ public void putDefaultReplNameIfInterpreterSettingAbsent() { when(interpreterFactory.getDefaultInterpreterSetting(anyString())) .thenReturn(null); - Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials); + Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); note.putDefaultReplName(); assertEquals(StringUtils.EMPTY, note.getLastReplName()); @@ -117,7 +122,7 @@ public void putDefaultReplNameIfInterpreterSettingPresent() { when(interpreterFactory.getDefaultInterpreterSetting(anyString())) .thenReturn(interpreterSetting); - Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials); + Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); note.putDefaultReplName(); assertEquals("spark", note.getLastReplName()); @@ -131,7 +136,7 @@ public void addParagraphWithLastReplName() { when(interpreterFactory.getDefaultInterpreterSetting(anyString())) .thenReturn(interpreterSetting); - Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials); + Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); note.putDefaultReplName(); //set lastReplName Paragraph p = note.addParagraph(); @@ -146,7 +151,7 @@ public void insertParagraphWithLastReplName() { when(interpreterFactory.getDefaultInterpreterSetting(anyString())) .thenReturn(interpreterSetting); - Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials); + Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); note.putDefaultReplName(); //set lastReplName Paragraph p = note.insertParagraph(note.getParagraphs().size()); @@ -157,7 +162,7 @@ public void insertParagraphWithLastReplName() { @Test public void setLastReplName() { String paragraphId = "HelloWorld"; - Note note = Mockito.spy(new Note(repo, interpreterFactory, jobListenerFactory, index, credentials)); + Note note = Mockito.spy(new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener)); Paragraph mockParagraph = Mockito.mock(Paragraph.class); when(note.getParagraph(paragraphId)).thenReturn(mockParagraph); when(mockParagraph.getRequiredReplName()).thenReturn("spark"); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 13771b70718..479f5675f0d 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -23,6 +23,7 @@ import java.io.File; import java.io.IOException; import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; import com.google.common.collect.Sets; import org.apache.commons.io.FileUtils; @@ -85,7 +86,7 @@ public void setUp() throws Exception { MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver); SearchService search = mock(SearchService.class); notebookRepo = new VFSNotebookRepo(conf); @@ -699,6 +700,66 @@ public void testPerSessionInterpreterCloseOnUnbindInterpreterSetting() throws IO notebook.removeNote(note1.getId(), null); } + @Test + public void testNotebookEventListener() throws IOException { + final AtomicInteger onNoteRemove = new AtomicInteger(0); + final AtomicInteger onNoteCreate = new AtomicInteger(0); + final AtomicInteger onParagraphRemove = new AtomicInteger(0); + final AtomicInteger onParagraphCreate = new AtomicInteger(0); + final AtomicInteger unbindInterpreter = new AtomicInteger(0); + + notebook.addNotebookEventListener(new NotebookEventListener() { + @Override + public void onNoteRemove(Note note) { + onNoteRemove.incrementAndGet(); + } + + @Override + public void onNoteCreate(Note note) { + onNoteCreate.incrementAndGet(); + } + + @Override + public void onUnbindInterpreter(Note note, InterpreterSetting setting) { + unbindInterpreter.incrementAndGet(); + } + + @Override + public void onParagraphRemove(Paragraph p) { + onParagraphRemove.incrementAndGet(); + } + + @Override + public void onParagraphCreate(Paragraph p) { + onParagraphCreate.incrementAndGet(); + } + + @Override + public void onParagraphStatusChange(Paragraph p, Status status) { + } + }); + + Note note1 = notebook.createNote(null); + assertEquals(1, onNoteCreate.get()); + + Paragraph p1 = note1.addParagraph(); + assertEquals(1, onParagraphCreate.get()); + + note1.addCloneParagraph(p1); + assertEquals(2, onParagraphCreate.get()); + + note1.removeParagraph(p1.getId()); + assertEquals(1, onParagraphRemove.get()); + + List settings = notebook.getBindedInterpreterSettingsIds(note1.id()); + notebook.bindInterpretersToNote(note1.id(), new LinkedList()); + assertEquals(settings.size(), unbindInterpreter.get()); + + notebook.removeNote(note1.getId(), null); + assertEquals(1, onNoteRemove.get()); + assertEquals(1, onParagraphRemove.get()); + } + @Test public void testNormalizeNoteName() throws IOException { // create a notes diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java index fe7e3fc2de2..0c67d79789b 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java @@ -94,7 +94,7 @@ public void setUp() throws Exception { MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver); search = mock(SearchService.class); notebookRepoSync = new NotebookRepoSync(conf); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java index 9fb0360f603..7eaca8c74eb 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java @@ -76,7 +76,7 @@ public void setUp() throws Exception { this.schedulerFactory = new SchedulerFactory(); depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver); SearchService search = mock(SearchService.class); notebookRepo = new VFSNotebookRepo(conf); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java index b7fa2df6c87..81c4b81329b 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java @@ -284,7 +284,7 @@ private Paragraph addParagraphWithTextAndTitle(Note note, String text, String ti } private Note newNote(String name) { - Note note = new Note(notebookRepoMock, interpreterFactory, null, notebookIndex, null); + Note note = new Note(notebookRepoMock, interpreterFactory, null, notebookIndex, null, null); note.setName(name); return note; }