) tezDAGStatsMapField.get(stat);
+ for (TezDAGStats dagStats : tezDAGStatsMap.values()) {
+ LOGGER.debug("Tez JobId:" + dagStats.getJobId());
+ jobIds.add(dagStats.getJobId());
+ }
+ return jobIds;
+ } catch (Exception e) {
+ LOGGER.error("Can not extract jobIds from TezPigScriptStats", e);
+ throw new RuntimeException("Can not extract jobIds from TezPigScriptStats", e);
+ }
+ }
+}
diff --git a/pig/src/main/resources/interpreter-setting.json b/pig/src/main/resources/interpreter-setting.json
new file mode 100644
index 00000000000..27918ede1bf
--- /dev/null
+++ b/pig/src/main/resources/interpreter-setting.json
@@ -0,0 +1,46 @@
+[
+ {
+ "group": "pig",
+ "name": "script",
+ "className": "org.apache.zeppelin.pig.PigInterpreter",
+ "properties": {
+ "zeppelin.pig.execType": {
+ "envName": null,
+ "propertyName": "zeppelin.pig.execType",
+ "defaultValue": "mapreduce",
+ "description": "local | mapreduce | tez"
+ },
+ "zeppelin.pig.includeJobStats": {
+ "envName": null,
+ "propertyName": "zeppelin.pig.includeJobStats",
+ "defaultValue": "false",
+ "description": "flag to include job stats in output"
+ }
+ },
+ "editor": {
+ "language": "pig"
+ }
+ },
+ {
+ "group": "pig",
+ "name": "query",
+ "className": "org.apache.zeppelin.pig.PigQueryInterpreter",
+ "properties": {
+ "zeppelin.pig.execType": {
+ "envName": null,
+ "propertyName": "zeppelin.pig.execType",
+ "defaultValue": "mapreduce",
+ "description": "local | mapreduce | tez"
+ },
+ "zeppelin.pig.maxResult": {
+ "envName": null,
+ "propertyName": "zeppelin.pig.maxResult",
+ "defaultValue": "1000",
+ "description": "max row number for %pig.query"
+ }
+ },
+ "editor": {
+ "language": "pig"
+ }
+ }
+]
diff --git a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java
new file mode 100644
index 00000000000..3d062d61579
--- /dev/null
+++ b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.zeppelin.pig;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResult.Code;
+import org.apache.zeppelin.interpreter.InterpreterResult.Type;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class PigInterpreterTest {
+
+ private PigInterpreter pigInterpreter;
+ private InterpreterContext context;
+
+ @Before
+ public void setUp() {
+ Properties properties = new Properties();
+ properties.put("zeppelin.pig.execType", "local");
+ pigInterpreter = new PigInterpreter(properties);
+ pigInterpreter.open();
+ context = new InterpreterContext(null, "paragraph_id", null, null, null, null, null, null, null,
+ null, null);
+ }
+
+ @After
+ public void tearDown() {
+ pigInterpreter.close();
+ }
+
+ @Test
+ public void testBasics() throws IOException {
+ String content = "1\tandy\n"
+ + "2\tpeter\n";
+ File tmpFile = File.createTempFile("zeppelin", "test");
+ FileWriter writer = new FileWriter(tmpFile);
+ IOUtils.write(content, writer);
+ writer.close();
+
+ // simple pig script using dump
+ String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';"
+ + "dump a;";
+ InterpreterResult result = pigInterpreter.interpret(pigscript, context);
+ assertEquals(Type.TEXT, result.type());
+ assertEquals(Code.SUCCESS, result.code());
+ assertTrue(result.message().contains("(1,andy)\n(2,peter)"));
+
+ // describe
+ pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (id: int, name: bytearray);"
+ + "describe a;";
+ result = pigInterpreter.interpret(pigscript, context);
+ assertEquals(Type.TEXT, result.type());
+ assertEquals(Code.SUCCESS, result.code());
+ assertTrue(result.message().contains("a: {id: int,name: bytearray}"));
+
+ // syntax error (compilation error)
+ pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';"
+ + "describe a;";
+ result = pigInterpreter.interpret(pigscript, context);
+ assertEquals(Type.TEXT, result.type());
+ assertEquals(Code.ERROR, result.code());
+ assertTrue(result.message().contains("Syntax error, unexpected symbol at or near 'a'"));
+
+ // execution error
+ pigscript = "a = load 'invalid_path';"
+ + "dump a;";
+ result = pigInterpreter.interpret(pigscript, context);
+ assertEquals(Type.TEXT, result.type());
+ assertEquals(Code.ERROR, result.code());
+ assertTrue(result.message().contains("Input path does not exist"));
+ }
+
+
+ @Test
+ public void testIncludeJobStats() throws IOException {
+ Properties properties = new Properties();
+ properties.put("zeppelin.pig.execType", "local");
+ properties.put("zeppelin.pig.includeJobStats", "true");
+ pigInterpreter = new PigInterpreter(properties);
+ pigInterpreter.open();
+
+ String content = "1\tandy\n"
+ + "2\tpeter\n";
+ File tmpFile = File.createTempFile("zeppelin", "test");
+ FileWriter writer = new FileWriter(tmpFile);
+ IOUtils.write(content, writer);
+ writer.close();
+
+ // simple pig script using dump
+ String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';"
+ + "dump a;";
+ InterpreterResult result = pigInterpreter.interpret(pigscript, context);
+ assertEquals(Type.TEXT, result.type());
+ assertEquals(Code.SUCCESS, result.code());
+ assertTrue(result.message().contains("Counters:"));
+ assertTrue(result.message().contains("(1,andy)\n(2,peter)"));
+
+ // describe
+ pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (id: int, name: bytearray);"
+ + "describe a;";
+ result = pigInterpreter.interpret(pigscript, context);
+ assertEquals(Type.TEXT, result.type());
+ assertEquals(Code.SUCCESS, result.code());
+ // no job is launched, so no jobStats
+ assertTrue(!result.message().contains("Counters:"));
+ assertTrue(result.message().contains("a: {id: int,name: bytearray}"));
+
+ // syntax error (compilation error)
+ pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';"
+ + "describe a;";
+ result = pigInterpreter.interpret(pigscript, context);
+ assertEquals(Type.TEXT, result.type());
+ assertEquals(Code.ERROR, result.code());
+ // no job is launched, so no jobStats
+ assertTrue(!result.message().contains("Counters:"));
+ assertTrue(result.message().contains("Syntax error, unexpected symbol at or near 'a'"));
+
+ // execution error
+ pigscript = "a = load 'invalid_path';"
+ + "dump a;";
+ result = pigInterpreter.interpret(pigscript, context);
+ assertEquals(Type.TEXT, result.type());
+ assertEquals(Code.ERROR, result.code());
+ assertTrue(result.message().contains("Counters:"));
+ assertTrue(result.message().contains("Input path does not exist"));
+ }
+}
diff --git a/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java b/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java
new file mode 100644
index 00000000000..00ece440542
--- /dev/null
+++ b/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.pig;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ *
+ */
+public class PigQueryInterpreterTest {
+
+ private PigInterpreter pigInterpreter;
+ private PigQueryInterpreter pigQueryInterpreter;
+ private InterpreterContext context;
+
+ @Before
+ public void setUp() {
+ Properties properties = new Properties();
+ properties.put("zeppelin.pig.execType", "local");
+ properties.put("zeppelin.pig.maxResult", "20");
+
+ pigInterpreter = new PigInterpreter(properties);
+ pigQueryInterpreter = new PigQueryInterpreter(properties);
+ List interpreters = new ArrayList();
+ interpreters.add(pigInterpreter);
+ interpreters.add(pigQueryInterpreter);
+ InterpreterGroup group = new InterpreterGroup();
+ group.put("note_id", interpreters);
+ pigInterpreter.setInterpreterGroup(group);
+ pigQueryInterpreter.setInterpreterGroup(group);
+ pigInterpreter.open();
+ pigQueryInterpreter.open();
+
+ context = new InterpreterContext(null, "paragraph_id", null, null, null, null, null, null, null,
+ null, null);
+ }
+
+ @After
+ public void tearDown() {
+ pigInterpreter.close();
+ pigQueryInterpreter.close();
+ }
+
+ @Test
+ public void testBasics() throws IOException {
+ String content = "andy\tmale\t10\n"
+ + "peter\tmale\t20\n"
+ + "amy\tfemale\t14\n";
+ File tmpFile = File.createTempFile("zeppelin", "test");
+ FileWriter writer = new FileWriter(tmpFile);
+ IOUtils.write(content, writer);
+ writer.close();
+
+ // run script in PigInterpreter
+ String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (name, gender, age);\n"
+ + "a2 = load 'invalid_path' as (name, gender, age);\n"
+ + "dump a;";
+ InterpreterResult result = pigInterpreter.interpret(pigscript, context);
+ assertEquals(InterpreterResult.Type.TEXT, result.type());
+ assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+ assertTrue(result.message().contains("(andy,male,10)\n(peter,male,20)\n(amy,female,14)"));
+
+ // run single line query in PigQueryInterpreter
+ String query = "foreach a generate name, age;";
+ result = pigQueryInterpreter.interpret(query, context);
+ assertEquals(InterpreterResult.Type.TABLE, result.type());
+ assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+ assertEquals("name\tage\nandy\t10\npeter\t20\namy\t14\n", result.message());
+
+ // run multiple line query in PigQueryInterpreter
+ query = "b = group a by gender;\nforeach b generate group as gender, COUNT($1) as count;";
+ result = pigQueryInterpreter.interpret(query, context);
+ assertEquals(InterpreterResult.Type.TABLE, result.type());
+ assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+ assertEquals("gender\tcount\nmale\t2\nfemale\t1\n", result.message());
+
+ // syntax error in PigQueryInterpereter
+ query = "b = group a by invalid_column;\nforeach b generate group as gender, COUNT($1) as count;";
+ result = pigQueryInterpreter.interpret(query, context);
+ assertEquals(InterpreterResult.Type.TEXT, result.type());
+ assertEquals(InterpreterResult.Code.ERROR, result.code());
+ assertTrue(result.message().contains("Projected field [invalid_column] does not exist in schema"));
+
+ // execution error in PigQueryInterpreter
+ query = "foreach a2 generate name, age;";
+ result = pigQueryInterpreter.interpret(query, context);
+ assertEquals(InterpreterResult.Type.TEXT, result.type());
+ assertEquals(InterpreterResult.Code.ERROR, result.code());
+ assertTrue(result.message().contains("Input path does not exist"));
+ }
+
+ @Test
+ public void testMaxResult() throws IOException {
+ StringBuilder content = new StringBuilder();
+ for (int i=0;i<30;++i) {
+ content.append(i + "\tname_" + i + "\n");
+ }
+ File tmpFile = File.createTempFile("zeppelin", "test");
+ FileWriter writer = new FileWriter(tmpFile);
+ IOUtils.write(content, writer);
+ writer.close();
+
+ // run script in PigInterpreter
+ String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (id, name);";
+ InterpreterResult result = pigInterpreter.interpret(pigscript, context);
+ assertEquals(InterpreterResult.Type.TEXT, result.type());
+ assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+ // empty output
+ assertTrue(result.message().isEmpty());
+
+ // run single line query in PigQueryInterpreter
+ String query = "foreach a generate id;";
+ result = pigQueryInterpreter.interpret(query, context);
+ assertEquals(InterpreterResult.Type.TABLE, result.type());
+ assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+ assertTrue(result.message().contains("id\n0\n1\n2"));
+ assertTrue(result.message().contains("Results are limited by 20"));
+ }
+}
diff --git a/pig/src/test/resources/log4j.properties b/pig/src/test/resources/log4j.properties
new file mode 100644
index 00000000000..8daee59d60d
--- /dev/null
+++ b/pig/src/test/resources/log4j.properties
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+log4j.rootLogger = INFO, stdout
+
+log4j.appender.stdout = org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n
diff --git a/pom.xml b/pom.xml
index d0f43885514..558ce0624a1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -62,6 +62,7 @@
shell
livy
hbase
+ pig
postgresql
jdbc
file
@@ -579,6 +580,13 @@
+
+ beam
+
+ beam
+
+
+
examples
@@ -746,6 +754,7 @@
.spark-dist/**
**/interpreter-setting.json
**/constants.json
+ scripts/**
docs/assets/themes/zeppelin/bootstrap/**
diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java
index 0561d86f867..568b75a9680 100644
--- a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java
+++ b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java
@@ -68,7 +68,7 @@ public PythonInterpreter(Properties property) {
@Override
public void open() {
- LOG.info("Starting Python interpreter .....");
+ LOG.info("Starting Python interpreter ---->");
LOG.info("Python path is set to:" + property.getProperty(ZEPPELIN_PYTHON));
maxResult = Integer.valueOf(getProperty(MAX_RESULT));
@@ -111,7 +111,7 @@ public void open() {
@Override
public void close() {
- LOG.info("closing Python interpreter .....");
+ LOG.info("closing Python interpreter <----");
try {
if (process != null) {
process.close();
@@ -134,11 +134,9 @@ public InterpreterResult interpret(String cmd, InterpreterContext contextInterpr
InterpreterResult result;
if (pythonErrorIn(output)) {
- result = new InterpreterResult(Code.ERROR, output);
+ result = new InterpreterResult(Code.ERROR, output.replaceAll("\\.\\.\\.", ""));
} else {
- // TODO(zjffdu), we should not do string replacement operation in the result, as it is
- // possible that the output contains the kind of pattern itself, e.g. print("...")
- result = new InterpreterResult(Code.SUCCESS, output.replaceAll("\\.\\.\\.", ""));
+ result = new InterpreterResult(Code.SUCCESS, output);
}
return result;
}
diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java b/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java
index 0ab14613101..190b3da8808 100644
--- a/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java
+++ b/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java
@@ -91,11 +91,6 @@ public String sendAndGetResult(String cmd) throws IOException {
String line = null;
while (!(line = reader.readLine()).contains(STATEMENT_END)) {
logger.debug("Read line from python shell : " + line);
- if (line.equals("...")) {
- logger.warn("Syntax error ! ");
- output.append("Syntax error ! ");
- break;
- }
output.append(line + "\n");
}
return output.toString();
diff --git a/python/src/main/resources/interpreter-setting.json b/python/src/main/resources/interpreter-setting.json
index 4b12cad3009..d3df586defd 100644
--- a/python/src/main/resources/interpreter-setting.json
+++ b/python/src/main/resources/interpreter-setting.json
@@ -16,6 +16,9 @@
"defaultValue": "1000",
"description": "Max number of dataframe rows to display."
}
+ },
+ "editor": {
+ "language": "python"
}
},
{
diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterWithPythonInstalledTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterWithPythonInstalledTest.java
index 38b46e71ca8..383533b00c9 100644
--- a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterWithPythonInstalledTest.java
+++ b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterWithPythonInstalledTest.java
@@ -56,6 +56,8 @@ public void badPythonSyntaxFails() {
//System.out.println("\nInterpreter response: \n" + ret.message());
assertEquals(InterpreterResult.Code.ERROR, ret.code());
assertTrue(ret.message().length() > 0);
+
+ realPython.close();
}
@Test
@@ -73,6 +75,36 @@ public void goodPythonSyntaxRuns() {
//System.out.println("\nInterpreter response: \n" + ret.message());
assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
assertTrue(ret.message().length() > 0);
+
+ realPython.close();
+ }
+
+ @Test
+ public void testZeppelin1555() {
+ //given
+ PythonInterpreter realPython = new PythonInterpreter(
+ PythonInterpreterTest.getPythonTestProperties());
+ realPython.open();
+
+ //when
+ InterpreterResult ret1 = realPython.interpret("print \"...\"", null);
+
+ //then
+ //System.out.println("\nInterpreter response: \n" + ret.message());
+ assertEquals(InterpreterResult.Code.SUCCESS, ret1.code());
+ assertEquals("...\n", ret1.message());
+
+
+ InterpreterResult ret2 = realPython.interpret("for i in range(5):", null);
+ //then
+ //System.out.println("\nInterpreterResultterpreter response: \n" + ret2.message());
+ assertEquals(InterpreterResult.Code.ERROR, ret2.code());
+ assertEquals(" File \"\", line 2\n" +
+ " \n" +
+ " ^\n" +
+ "IndentationError: expected an indented block\n", ret2.message());
+
+ realPython.close();
}
}
diff --git a/scripts/docker/spark-cluster-managers/cdh/hdfs_conf/core-site.xml b/scripts/docker/spark-cluster-managers/cdh/hdfs_conf/core-site.xml
new file mode 100644
index 00000000000..6cdbc7fbb51
--- /dev/null
+++ b/scripts/docker/spark-cluster-managers/cdh/hdfs_conf/core-site.xml
@@ -0,0 +1,6 @@
+
+
+ fs.defaultFS
+ hdfs://0.0.0.0:8020
+
+
diff --git a/scripts/docker/spark-cluster-managers/cdh/hdfs_conf/hdfs-site.xml b/scripts/docker/spark-cluster-managers/cdh/hdfs_conf/hdfs-site.xml
new file mode 100644
index 00000000000..ce031cfe5e0
--- /dev/null
+++ b/scripts/docker/spark-cluster-managers/cdh/hdfs_conf/hdfs-site.xml
@@ -0,0 +1,64 @@
+
+
+ dfs.replication
+ 1
+
+
+
+
+ dfs.data.dir
+ /data/hdfs
+ true
+
+
+
+ dfs.permissions
+ false
+
+
+
+
+ dfs.client.use.datanode.hostname
+ true
+ Whether clients should use datanode hostnames when
+ connecting to datanodes.
+
+
+
+
+ dfs.datanode.use.datanode.hostname
+ true
+ Whether datanodes should use datanode hostnames when
+ connecting to other datanodes for data transfer.
+
+
+
+
+ dfs.datanode.address
+ 0.0.0.0:50010
+
+ The address where the datanode server will listen to.
+ If the port is 0 then the server will start on a free port.
+
+
+
+
+ dfs.datanode.http.address
+ 0.0.0.0:50075
+
+ The datanode http server address and port.
+ If the port is 0 then the server will start on a free port.
+
+
+
+
+ dfs.datanode.ipc.address
+ 0.0.0.0:50020
+
+ The datanode ipc server address and port.
+ If the port is 0 then the server will start on a free port.
+
+
+
+
+
diff --git a/scripts/docker/spark-cluster-managers/cdh/hdfs_conf/mapred-site.xml b/scripts/docker/spark-cluster-managers/cdh/hdfs_conf/mapred-site.xml
new file mode 100644
index 00000000000..6dc557dca67
--- /dev/null
+++ b/scripts/docker/spark-cluster-managers/cdh/hdfs_conf/mapred-site.xml
@@ -0,0 +1,6 @@
+
+
+ mapreduce.framework.name
+ yarn
+
+
diff --git a/scripts/docker/spark-cluster-managers/cdh/hdfs_conf/yarn-site.xml b/scripts/docker/spark-cluster-managers/cdh/hdfs_conf/yarn-site.xml
new file mode 100644
index 00000000000..4fce42f9869
--- /dev/null
+++ b/scripts/docker/spark-cluster-managers/cdh/hdfs_conf/yarn-site.xml
@@ -0,0 +1,26 @@
+
+
+ yarn.resourcemanager.scheduler.address
+ 0.0.0.0:8030
+
+
+ yarn.resourcemanager.address
+ 0.0.0.0:8032
+
+
+ yarn.resourcemanager.webapp.address
+ 0.0.0.0:8088
+
+
+ yarn.resourcemanager.resource-tracker.address
+ 0.0.0.0:8031
+
+
+ yarn.resourcemanager.admin.address
+ 0.0.0.0:8033
+
+
+ yarn.application.classpath
+ /usr/local/hadoop/etc/hadoop, /usr/local/hadoop/share/hadoop/common/*, /usr/local/hadoop/share/hadoop/common/lib/*, /usr/local/hadoop/share/hadoop/hdfs/*, /usr/local/hadoop/share/hadoop/hdfs/lib/*, /usr/local/hadoop/share/hadoop/mapreduce/*, /usr/local/hadoop/share/hadoop/mapreduce/lib/*, /usr/local/hadoop/share/hadoop/yarn/*, /usr/local/hadoop/share/hadoop/yarn/lib/*, /usr/local/hadoop/share/spark/*
+
+
diff --git a/scripts/vagrant/zeppelin-dev/README.md b/scripts/vagrant/zeppelin-dev/README.md
index 9e5c9c9cebe..fd428d69208 100644
--- a/scripts/vagrant/zeppelin-dev/README.md
+++ b/scripts/vagrant/zeppelin-dev/README.md
@@ -140,7 +140,7 @@ import matplotlib.pyplot as plt
import numpy as np
import StringIO
-# clear out any previous plots on this notebook
+# clear out any previous plots on this note
plt.clf()
def show(p):
diff --git a/shell/src/main/resources/interpreter-setting.json b/shell/src/main/resources/interpreter-setting.json
index 78621df684e..d1996fab7e2 100644
--- a/shell/src/main/resources/interpreter-setting.json
+++ b/shell/src/main/resources/interpreter-setting.json
@@ -28,6 +28,9 @@
"defaultValue": "",
"description": "Kerberos principal"
}
+ },
+ "editor": {
+ "language": "sh"
}
}
-]
\ No newline at end of file
+]
diff --git a/spark-dependencies/pom.xml b/spark-dependencies/pom.xml
index f320680072b..04b6983fb03 100644
--- a/spark-dependencies/pom.xml
+++ b/spark-dependencies/pom.xml
@@ -525,9 +525,9 @@
true
- 2.0.0
+ 2.0.1
2.5.0
- 0.10.1
+ 0.10.3
2.11.8
diff --git a/spark/pom.xml b/spark/pom.xml
index 66d93c42ee6..efb7452647c 100644
--- a/spark/pom.xml
+++ b/spark/pom.xml
@@ -37,7 +37,7 @@
1.8.2
1.10.19
1.6.4
- 2.0.0
+ 2.0.1
@@ -519,9 +519,9 @@
true
- 2.0.0
+ 2.0.1
2.5.0
- 0.10.1
+ 0.10.3
2.11.8
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
index 9a54912a35c..0812c761038 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
@@ -49,6 +49,7 @@
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterHookRegistry;
import org.apache.zeppelin.interpreter.InterpreterProperty;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
@@ -101,6 +102,7 @@ public class SparkInterpreter extends Interpreter {
private SparkConf conf;
private static SparkContext sc;
private static SQLContext sqlc;
+ private static InterpreterHookRegistry hooks;
private static SparkEnv env;
private static Object sparkSession; // spark 2.x
private static JobProgressListener sparkListener;
@@ -118,7 +120,7 @@ public class SparkInterpreter extends Interpreter {
private Map binder;
private SparkVersion sparkVersion;
- private File outputDir; // class outputdir for scala 2.11
+ private static File outputDir; // class outputdir for scala 2.11
private Object classServer; // classserver for scala 2.11
@@ -329,6 +331,7 @@ public Object createSparkSession() {
}
setupConfForPySpark(conf);
+ setupConfForSparkR(conf);
Class SparkSession = Utils.findClass("org.apache.spark.sql.SparkSession");
Object builder = Utils.invokeStaticMethod(SparkSession, "builder");
Utils.invokeMethod(builder, "config", new Class[]{ SparkConf.class }, new Object[]{ conf });
@@ -383,6 +386,7 @@ public SparkContext createSparkContext_1() {
}
String classServerUri = null;
+ String replClassOutputDirectory = null;
try { // in case of spark 1.1x, spark 1.2x
Method classServer = intp.getClass().getMethod("classServer");
@@ -406,6 +410,16 @@ public SparkContext createSparkContext_1() {
}
}
+ if (classServerUri == null) {
+ try { // for RcpEnv
+ Method getClassOutputDirectory = intp.getClass().getMethod("getClassOutputDirectory");
+ File classOutputDirectory = (File) getClassOutputDirectory.invoke(intp);
+ replClassOutputDirectory = classOutputDirectory.getAbsolutePath();
+ } catch (NoSuchMethodException | SecurityException | IllegalAccessException
+ | IllegalArgumentException | InvocationTargetException e) {
+ // continue
+ }
+ }
if (Utils.isScala2_11()) {
classServer = createHttpServer(outputDir);
@@ -420,6 +434,10 @@ public SparkContext createSparkContext_1() {
conf.set("spark.repl.class.uri", classServerUri);
}
+ if (replClassOutputDirectory != null) {
+ conf.set("spark.repl.class.outputDir", replClassOutputDirectory);
+ }
+
if (jars.length > 0) {
conf.setJars(jars);
}
@@ -443,6 +461,7 @@ public SparkContext createSparkContext_1() {
}
}
setupConfForPySpark(conf);
+ setupConfForSparkR(conf);
SparkContext sparkContext = new SparkContext(conf);
return sparkContext;
}
@@ -462,7 +481,7 @@ private void setupConfForPySpark(SparkConf conf) {
//Only one of py4j-0.9-src.zip and py4j-0.8.2.1-src.zip should exist
String[] pythonLibs = new String[]{"pyspark.zip", "py4j-0.9-src.zip", "py4j-0.8.2.1-src.zip",
- "py4j-0.10.1-src.zip"};
+ "py4j-0.10.1-src.zip", "py4j-0.10.3-src.zip"};
ArrayList pythonLibUris = new ArrayList<>();
for (String lib : pythonLibs) {
File libFile = new File(pysparkPath, lib);
@@ -494,6 +513,35 @@ private void setupConfForPySpark(SparkConf conf) {
}
}
+ private void setupConfForSparkR(SparkConf conf) {
+ String sparkRBasePath = new InterpreterProperty("SPARK_HOME", null, null, null).getValue();
+ File sparkRPath;
+ if (null == sparkRBasePath) {
+ sparkRBasePath =
+ new InterpreterProperty("ZEPPELIN_HOME", "zeppelin.home", "../", null).getValue();
+ sparkRPath = new File(sparkRBasePath,
+ "interpreter" + File.separator + "spark" + File.separator + "R");
+ } else {
+ sparkRPath = new File(sparkRBasePath, "R" + File.separator + "lib");
+ }
+
+ sparkRPath = new File(sparkRPath, "sparkr.zip");
+ if (sparkRPath.exists() && sparkRPath.isFile()) {
+ String archives = null;
+ if (conf.contains("spark.yarn.dist.archives")) {
+ archives = conf.get("spark.yarn.dist.archives");
+ }
+ if (archives != null) {
+ archives = archives + "," + sparkRPath + "#sparkr";
+ } else {
+ archives = sparkRPath + "#sparkr";
+ }
+ conf.set("spark.yarn.dist.archives", archives);
+ } else {
+ logger.warn("sparkr.zip is not found, sparkr may not work.");
+ }
+ }
+
static final String toString(Object o) {
return (o instanceof String) ? (String) o : "";
}
@@ -553,6 +601,24 @@ public void open() {
argList.add(arg);
}
+ DepInterpreter depInterpreter = getDepInterpreter();
+ String depInterpreterClasspath = "";
+ if (depInterpreter != null) {
+ SparkDependencyContext depc = depInterpreter.getDependencyContext();
+ if (depc != null) {
+ List files = depc.getFiles();
+ if (files != null) {
+ for (File f : files) {
+ if (depInterpreterClasspath.length() > 0) {
+ depInterpreterClasspath += File.pathSeparator;
+ }
+ depInterpreterClasspath += f.getAbsolutePath();
+ }
+ }
+ }
+ }
+
+
if (Utils.isScala2_10()) {
scala.collection.immutable.List list =
JavaConversions.asScalaBuffer(argList).toList();
@@ -572,15 +638,30 @@ public void open() {
sparkReplClassDir = System.getProperty("java.io.tmpdir");
}
- outputDir = createTempDir(sparkReplClassDir);
-
+ synchronized (sharedInterpreterLock) {
+ if (outputDir == null) {
+ outputDir = createTempDir(sparkReplClassDir);
+ }
+ }
argList.add("-Yrepl-class-based");
argList.add("-Yrepl-outdir");
argList.add(outputDir.getAbsolutePath());
+
+ String classpath = "";
if (conf.contains("spark.jars")) {
- String jars = StringUtils.join(conf.get("spark.jars").split(","), File.separator);
+ classpath = StringUtils.join(conf.get("spark.jars").split(","), File.separator);
+ }
+
+ if (!depInterpreterClasspath.isEmpty()) {
+ if (!classpath.isEmpty()) {
+ classpath += File.separator;
+ }
+ classpath += depInterpreterClasspath;
+ }
+
+ if (!classpath.isEmpty()) {
argList.add("-classpath");
- argList.add(jars);
+ argList.add(classpath);
}
scala.collection.immutable.List list =
@@ -592,6 +673,7 @@ public void open() {
// set classpath for scala compiler
PathSetting pathSettings = settings.classpath();
String classpath = "";
+
List paths = currentClassPath();
for (File f : paths) {
if (classpath.length() > 0) {
@@ -610,21 +692,10 @@ public void open() {
}
// add dependency from DepInterpreter
- DepInterpreter depInterpreter = getDepInterpreter();
- if (depInterpreter != null) {
- SparkDependencyContext depc = depInterpreter.getDependencyContext();
- if (depc != null) {
- List files = depc.getFiles();
- if (files != null) {
- for (File f : files) {
- if (classpath.length() > 0) {
- classpath += File.pathSeparator;
- }
- classpath += f.getAbsolutePath();
- }
- }
- }
+ if (classpath.length() > 0) {
+ classpath += File.pathSeparator;
}
+ classpath += depInterpreterClasspath;
// add dependency from local repo
String localRepo = getProperty("zeppelin.interpreter.localRepo");
@@ -744,8 +815,10 @@ public void open() {
sqlc = getSQLContext();
dep = getDependencyResolver();
+
+ hooks = getInterpreterGroup().getInterpreterHookRegistry();
- z = new ZeppelinContext(sc, sqlc, null, dep,
+ z = new ZeppelinContext(sc, sqlc, null, dep, hooks,
Integer.parseInt(getProperty("zeppelin.spark.maxResult")));
interpret("@transient val _binder = new java.util.HashMap[String, Object]()");
@@ -1276,7 +1349,12 @@ public void close() {
logger.info("Close interpreter");
if (numReferenceOfSparkContext.decrementAndGet() == 0) {
- sc.stop();
+ if (sparkSession != null) {
+ Utils.invokeMethod(sparkSession, "stop");
+ } else if (sc != null){
+ sc.stop();
+ }
+ sparkSession = null;
sc = null;
if (classServer != null) {
Utils.invokeMethod(classServer, "stop");
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java
index 7bccbac7d52..92b50d0a3b8 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java
@@ -24,14 +24,18 @@
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
import org.apache.spark.SparkContext;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.catalyst.expressions.Attribute;
import org.apache.zeppelin.annotation.ZeppelinApi;
+import org.apache.zeppelin.annotation.Experimental;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectWatcher;
@@ -40,6 +44,7 @@
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterHookRegistry;
import org.apache.zeppelin.spark.dep.SparkDependencyResolver;
import org.apache.zeppelin.resource.Resource;
import org.apache.zeppelin.resource.ResourcePool;
@@ -52,19 +57,53 @@
* Spark context for zeppelin.
*/
public class ZeppelinContext {
+ // Map interpreter class name (to be used by hook registry) from
+ // given replName in parapgraph
+ private static final Map interpreterClassMap;
+ static {
+ interpreterClassMap = new HashMap();
+ interpreterClassMap.put("spark", "org.apache.zeppelin.spark.SparkInterpreter");
+ interpreterClassMap.put("sql", "org.apache.zeppelin.spark.SparkSqlInterpreter");
+ interpreterClassMap.put("dep", "org.apache.zeppelin.spark.DepInterpreter");
+ interpreterClassMap.put("pyspark", "org.apache.zeppelin.spark.PySparkInterpreter");
+ }
+
private SparkDependencyResolver dep;
private InterpreterContext interpreterContext;
private int maxResult;
-
+ private List supportedClasses;
+ private InterpreterHookRegistry hooks;
+
public ZeppelinContext(SparkContext sc, SQLContext sql,
InterpreterContext interpreterContext,
SparkDependencyResolver dep,
+ InterpreterHookRegistry hooks,
int maxResult) {
this.sc = sc;
this.sqlContext = sql;
this.interpreterContext = interpreterContext;
this.dep = dep;
+ this.hooks = hooks;
this.maxResult = maxResult;
+ this.supportedClasses = new ArrayList<>();
+ try {
+ supportedClasses.add(this.getClass().forName("org.apache.spark.sql.Dataset"));
+ } catch (ClassNotFoundException e) {
+ }
+
+ try {
+ supportedClasses.add(this.getClass().forName("org.apache.spark.sql.DataFrame"));
+ } catch (ClassNotFoundException e) {
+ }
+
+ try {
+ supportedClasses.add(this.getClass().forName("org.apache.spark.sql.SchemaRDD"));
+ } catch (ClassNotFoundException e) {
+ }
+
+ if (supportedClasses.isEmpty()) {
+ throw new InterpreterException("Can not road Dataset/DataFrame/SchemaRDD class");
+ }
}
public SparkContext sc;
@@ -161,33 +200,8 @@ public void show(Object o) {
@ZeppelinApi
public void show(Object o, int maxResult) {
- Class cls = null;
- try {
- cls = this.getClass().forName("org.apache.spark.sql.Dataset");
- } catch (ClassNotFoundException e) {
- }
-
- if (cls == null) {
- try {
- cls = this.getClass().forName("org.apache.spark.sql.DataFrame");
- } catch (ClassNotFoundException e) {
- }
- }
-
- if (cls == null) {
- try {
- cls = this.getClass().forName("org.apache.spark.sql.SchemaRDD");
- } catch (ClassNotFoundException e) {
- }
- }
-
- if (cls == null) {
- throw new InterpreterException("Can not road Dataset/DataFrame/SchemaRDD class");
- }
-
-
try {
- if (cls.isInstance(o)) {
+ if (supportedClasses.contains(o.getClass())) {
interpreterContext.out.write(showDF(sc, interpreterContext, o, maxResult));
} else {
interpreterContext.out.write(o.toString());
@@ -210,6 +224,12 @@ public static String showDF(SparkContext sc,
sc.setJobGroup(jobGroup, "Zeppelin", false);
try {
+ // convert it to DataFrame if it is Dataset, as we will iterate all the records
+ // and assume it is type Row.
+ if (df.getClass().getCanonicalName().equals("org.apache.spark.sql.Dataset")) {
+ Method convertToDFMethod = df.getClass().getMethod("toDF");
+ df = convertToDFMethod.invoke(df);
+ }
take = df.getClass().getMethod("take", int.class);
rows = (Object[]) take.invoke(df, maxResult + 1);
} catch (NoSuchMethodException | SecurityException | IllegalAccessException
@@ -695,6 +715,90 @@ private void angularUnbind(String name, String noteId) {
registry.remove(name, noteId, null);
}
+ /**
+ * Get the interpreter class name from name entered in paragraph
+ * @param replName if replName is a valid className, return that instead.
+ */
+ public String getClassNameFromReplName(String replName) {
+ for (String name : interpreterClassMap.values()) {
+ if (replName.equals(name)) {
+ return replName;
+ }
+ }
+
+ if (replName.contains("spark.")) {
+ replName = replName.replace("spark.", "");
+ }
+ return interpreterClassMap.get(replName);
+ }
+
+ /**
+ * General function to register hook event
+ * @param event The type of event to hook to (pre_exec, post_exec)
+ * @param cmd The code to be executed by the interpreter on given event
+ * @param replName Name of the interpreter
+ */
+ @Experimental
+ public void registerHook(String event, String cmd, String replName) {
+ String noteId = interpreterContext.getNoteId();
+ String className = getClassNameFromReplName(replName);
+ hooks.register(noteId, className, event, cmd);
+ }
+
+ /**
+ * registerHook() wrapper for current repl
+ * @param event The type of event to hook to (pre_exec, post_exec)
+ * @param cmd The code to be executed by the interpreter on given event
+ */
+ @Experimental
+ public void registerHook(String event, String cmd) {
+ String className = interpreterContext.getClassName();
+ registerHook(event, cmd, className);
+ }
+
+ /**
+ * Get the hook code
+ * @param event The type of event to hook to (pre_exec, post_exec)
+ * @param replName Name of the interpreter
+ */
+ @Experimental
+ public String getHook(String event, String replName) {
+ String noteId = interpreterContext.getNoteId();
+ String className = getClassNameFromReplName(replName);
+ return hooks.get(noteId, className, event);
+ }
+
+ /**
+ * getHook() wrapper for current repl
+ * @param event The type of event to hook to (pre_exec, post_exec)
+ */
+ @Experimental
+ public String getHook(String event) {
+ String className = interpreterContext.getClassName();
+ return getHook(event, className);
+ }
+
+ /**
+ * Unbind code from given hook event
+ * @param event The type of event to hook to (pre_exec, post_exec)
+ * @param replName Name of the interpreter
+ */
+ @Experimental
+ public void unregisterHook(String event, String replName) {
+ String noteId = interpreterContext.getNoteId();
+ String className = getClassNameFromReplName(replName);
+ hooks.unregister(noteId, className, event);
+ }
+
+ /**
+ * unregisterHook() wrapper for current repl
+ * @param event The type of event to hook to (pre_exec, post_exec)
+ */
+ @Experimental
+ public void unregisterHook(String event) {
+ String className = interpreterContext.getClassName();
+ unregisterHook(event, className);
+ }
/**
* Add object into resource pool
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java
index 26488337b11..961793db17e 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java
@@ -141,6 +141,9 @@ public void open() throws IOException {
cmd.addArgument(Integer.toString(port));
cmd.addArgument(libPath);
cmd.addArgument(Integer.toString(sparkVersion.toNumber()));
+
+ // dump out the R command to facilitate manually running it, e.g. for fault diagnosis purposes
+ logger.debug(cmd.toString());
executor = new DefaultExecutor();
outputStream = new SparkOutputStream(logger);
diff --git a/spark/src/main/resources/interpreter-setting.json b/spark/src/main/resources/interpreter-setting.json
index d87a6c7ff75..55d25bd66e0 100644
--- a/spark/src/main/resources/interpreter-setting.json
+++ b/spark/src/main/resources/interpreter-setting.json
@@ -54,6 +54,9 @@
"defaultValue": "local[*]",
"description": "Spark master uri. ex) spark://masterhost:7077"
}
+ },
+ "editor": {
+ "language": "scala"
}
},
{
@@ -85,6 +88,9 @@
"defaultValue": "true",
"description": "Import implicits, UDF collection, and sql if set true. true by default."
}
+ },
+ "editor": {
+ "language": "sql"
}
},
{
@@ -104,6 +110,9 @@
"defaultValue": "spark-packages,http://dl.bintray.com/spark-packages/maven,false;",
"description": "A list of 'id,remote-repository-URL,is-snapshot;' for each remote repository."
}
+ },
+ "editor": {
+ "language": "scala"
}
},
{
@@ -117,6 +126,9 @@
"defaultValue": "python",
"description": "Python command to run pyspark with"
}
+ },
+ "editor": {
+ "language": "python"
}
}
]
diff --git a/spark/src/main/resources/python/zeppelin_pyspark.py b/spark/src/main/resources/python/zeppelin_pyspark.py
index 3e6535fa4f9..a53f503b83c 100644
--- a/spark/src/main/resources/python/zeppelin_pyspark.py
+++ b/spark/src/main/resources/python/zeppelin_pyspark.py
@@ -80,16 +80,16 @@ def put(self, key, value):
def get(self, key):
return self.__getitem__(key)
- def input(self, name, defaultValue = ""):
+ def input(self, name, defaultValue=""):
return self.z.input(name, defaultValue)
- def select(self, name, options, defaultValue = ""):
+ def select(self, name, options, defaultValue=""):
# auto_convert to ArrayList doesn't match the method signature on JVM side
tuples = list(map(lambda items: self.__tupleToScalaTuple2(items), options))
iterables = gateway.jvm.scala.collection.JavaConversions.collectionAsScalaIterable(tuples)
return self.z.select(name, defaultValue, iterables)
- def checkbox(self, name, options, defaultChecked = None):
+ def checkbox(self, name, options, defaultChecked=None):
if defaultChecked is None:
defaultChecked = list(map(lambda items: items[0], options))
optionTuples = list(map(lambda items: self.__tupleToScalaTuple2(items), options))
@@ -99,6 +99,23 @@ def checkbox(self, name, options, defaultChecked = None):
checkedIterables = self.z.checkbox(name, defaultCheckedIterables, optionIterables)
return gateway.jvm.scala.collection.JavaConversions.asJavaCollection(checkedIterables)
+ def registerHook(self, event, cmd, replName=None):
+ if replName is None:
+ self.z.registerHook(event, cmd)
+ else:
+ self.z.registerHook(event, cmd, replName)
+
+ def unregisterHook(self, event, replName=None):
+ if replName is None:
+ self.z.unregisterHook(event)
+ else:
+ self.z.unregisterHook(event, replName)
+
+ def getHook(self, event, replName=None):
+ if replName is None:
+ return self.z.getHook(event)
+ return self.z.getHook(event, replName)
+
def __tupleToScalaTuple2(self, tuple):
if (len(tuple) == 2):
return gateway.jvm.scala.Tuple2(tuple[0], tuple[1])
@@ -218,11 +235,12 @@ def getCompletion(self, text_value):
jconf = intp.getSparkConf()
conf = SparkConf(_jvm = gateway.jvm, _jconf = jconf)
sc = SparkContext(jsc=jsc, gateway=gateway, conf=conf)
-sqlc = SQLContext(sc, intp.getSQLContext())
-sqlContext = sqlc
-
if sparkVersion.isSpark2():
spark = SparkSession(sc, intp.getSparkSession())
+ sqlc = spark._wrapped
+else:
+ sqlc = SQLContext(sparkContext=sc, sqlContext=intp.getSQLContext())
+sqlContext = sqlc
completion = PySparkCompletion(intp)
z = PyZeppelinContext(intp.getZeppelinContext())
diff --git a/spark/src/main/sparkr-resources/interpreter-setting.json b/spark/src/main/sparkr-resources/interpreter-setting.json
index f884fe48702..338c86147af 100644
--- a/spark/src/main/sparkr-resources/interpreter-setting.json
+++ b/spark/src/main/sparkr-resources/interpreter-setting.json
@@ -54,6 +54,9 @@
"defaultValue": "local[*]",
"description": "Spark master uri. ex) spark://masterhost:7077"
}
+ },
+ "editor": {
+ "language": "scala"
}
},
{
@@ -85,6 +88,9 @@
"defaultValue": "true",
"description": "Import implicits, UDF collection, and sql if set true. true by default."
}
+ },
+ "editor": {
+ "language": "sql"
}
},
{
@@ -104,6 +110,9 @@
"defaultValue": "spark-packages,http://dl.bintray.com/spark-packages/maven,false;",
"description": "A list of 'id,remote-repository-URL,is-snapshot;' for each remote repository."
}
+ },
+ "editor": {
+ "language": "scala"
}
},
{
@@ -117,6 +126,9 @@
"defaultValue": "python",
"description": "Python command to run pyspark with"
}
+ },
+ "editor": {
+ "language": "python"
}
},
{
@@ -148,6 +160,9 @@
"defaultValue": "out.format = 'html', comment = NA, echo = FALSE, results = 'asis', message = F, warning = F",
"description": ""
}
+ },
+ "editor": {
+ "language": "r"
}
}
]
diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
index 1c7979fc428..ff26e6ad8df 100644
--- a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
+++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
@@ -178,20 +178,29 @@ public void testListener() {
@Test
public void testCreateDataFrame() {
- repl.interpret("case class Person(name:String, age:Int)\n", context);
- repl.interpret("val people = sc.parallelize(Seq(Person(\"moon\", 33), Person(\"jobs\", 51), Person(\"gates\", 51), Person(\"park\", 34)))\n", context);
- repl.interpret("people.toDF.count", context);
- assertEquals(new Long(4), context.getResourcePool().get(
- context.getNoteId(),
- context.getParagraphId(),
- WellKnownResourceName.ZeppelinReplResult.toString()).get());
+ if (getSparkVersionNumber() >= 13) {
+ repl.interpret("case class Person(name:String, age:Int)\n", context);
+ repl.interpret("val people = sc.parallelize(Seq(Person(\"moon\", 33), Person(\"jobs\", 51), Person(\"gates\", 51), Person(\"park\", 34)))\n", context);
+ repl.interpret("people.toDF.count", context);
+ assertEquals(new Long(4), context.getResourcePool().get(
+ context.getNoteId(),
+ context.getParagraphId(),
+ WellKnownResourceName.ZeppelinReplResult.toString()).get());
+ }
}
@Test
public void testZShow() {
+ String code = "";
repl.interpret("case class Person(name:String, age:Int)\n", context);
repl.interpret("val people = sc.parallelize(Seq(Person(\"moon\", 33), Person(\"jobs\", 51), Person(\"gates\", 51), Person(\"park\", 34)))\n", context);
- assertEquals(Code.SUCCESS, repl.interpret("z.show(people.toDF)", context).code());
+ if (getSparkVersionNumber() < 13) {
+ repl.interpret("people.registerTempTable(\"people\")", context);
+ code = "z.show(sqlc.sql(\"select * from people\"))";
+ } else {
+ code = "z.show(people.toDF)";
+ }
+ assertEquals(Code.SUCCESS, repl.interpret(code, context).code());
}
@Test
@@ -203,14 +212,15 @@ public void testSparkSql(){
if (getSparkVersionNumber() <= 11) { // spark 1.2 or later does not allow create multiple SparkContext in the same jvm by default.
// create new interpreter
- Properties p = new Properties();
- SparkInterpreter repl2 = new SparkInterpreter(p);
+ SparkInterpreter repl2 = new SparkInterpreter(getSparkTestProperties());
+ repl2.setInterpreterGroup(intpGroup);
+ intpGroup.get("note").add(repl2);
repl2.open();
- repl.interpret("case class Man(name:String, age:Int)", context);
- repl.interpret("val man = sc.parallelize(Seq(Man(\"moon\", 33), Man(\"jobs\", 51), Man(\"gates\", 51), Man(\"park\", 34)))", context);
- assertEquals(Code.SUCCESS, repl.interpret("man.take(3)", context).code());
- repl2.getSparkContext().stop();
+ repl2.interpret("case class Man(name:String, age:Int)", context);
+ repl2.interpret("val man = sc.parallelize(Seq(Man(\"moon\", 33), Man(\"jobs\", 51), Man(\"gates\", 51), Man(\"park\", 34)))", context);
+ assertEquals(Code.SUCCESS, repl2.interpret("man.take(3)", context).code());
+ repl2.close();
}
}
@@ -253,33 +263,37 @@ public void shareSingleSparkContext() throws InterruptedException {
@Test
public void testEnableImplicitImport() {
- // Set option of importing implicits to "true", and initialize new Spark repl
- Properties p = getSparkTestProperties();
- p.setProperty("zeppelin.spark.importImplicit", "true");
- SparkInterpreter repl2 = new SparkInterpreter(p);
- repl2.setInterpreterGroup(intpGroup);
- intpGroup.get("note").add(repl2);
+ if (getSparkVersionNumber() >= 13) {
+ // Set option of importing implicits to "true", and initialize new Spark repl
+ Properties p = getSparkTestProperties();
+ p.setProperty("zeppelin.spark.importImplicit", "true");
+ SparkInterpreter repl2 = new SparkInterpreter(p);
+ repl2.setInterpreterGroup(intpGroup);
+ intpGroup.get("note").add(repl2);
- repl2.open();
- String ddl = "val df = Seq((1, true), (2, false)).toDF(\"num\", \"bool\")";
- assertEquals(Code.SUCCESS, repl2.interpret(ddl, context).code());
- repl2.close();
+ repl2.open();
+ String ddl = "val df = Seq((1, true), (2, false)).toDF(\"num\", \"bool\")";
+ assertEquals(Code.SUCCESS, repl2.interpret(ddl, context).code());
+ repl2.close();
+ }
}
@Test
public void testDisableImplicitImport() {
- // Set option of importing implicits to "false", and initialize new Spark repl
- // this test should return error status when creating DataFrame from sequence
- Properties p = getSparkTestProperties();
- p.setProperty("zeppelin.spark.importImplicit", "false");
- SparkInterpreter repl2 = new SparkInterpreter(p);
- repl2.setInterpreterGroup(intpGroup);
- intpGroup.get("note").add(repl2);
+ if (getSparkVersionNumber() >= 13) {
+ // Set option of importing implicits to "false", and initialize new Spark repl
+ // this test should return error status when creating DataFrame from sequence
+ Properties p = getSparkTestProperties();
+ p.setProperty("zeppelin.spark.importImplicit", "false");
+ SparkInterpreter repl2 = new SparkInterpreter(p);
+ repl2.setInterpreterGroup(intpGroup);
+ intpGroup.get("note").add(repl2);
- repl2.open();
- String ddl = "val df = Seq((1, true), (2, false)).toDF(\"num\", \"bool\")";
- assertEquals(Code.ERROR, repl2.interpret(ddl, context).code());
- repl2.close();
+ repl2.open();
+ String ddl = "val df = Seq((1, true), (2, false)).toDF(\"num\", \"bool\")";
+ assertEquals(Code.ERROR, repl2.interpret(ddl, context).code());
+ repl2.close();
+ }
}
@Test
diff --git a/testing/startSparkCluster.sh b/testing/startSparkCluster.sh
deleted file mode 100755
index dc7613de302..00000000000
--- a/testing/startSparkCluster.sh
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-if [[ "$#" -ne 2 ]]; then
- echo "usage) $0 [spark version] [hadoop version]"
- echo " eg) $0 1.3.1 2.6"
- exit 1
-fi
-
-SPARK_VERSION="${1}"
-HADOOP_VERSION="${2}"
-
-echo "${SPARK_VERSION}" | grep "^1.[123].[0-9]" > /dev/null
-if [[ "$?" -eq 0 ]]; then
- echo "${SPARK_VERSION}" | grep "^1.[12].[0-9]" > /dev/null
- if [[ "$?" -eq 0 ]]; then
- SPARK_VER_RANGE="<=1.2"
- else
- SPARK_VER_RANGE="<=1.3"
- fi
-else
- SPARK_VER_RANGE=">1.3"
-fi
-
-set -xe
-
-FWDIR="$(dirname "${BASH_SOURCE-$0}")"
-ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
-
-SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}"
-export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}"
-echo "SPARK_HOME is ${SPARK_HOME}"
-
-# create PID dir. test case detect pid file so they can select active spark home dir for test
-export SPARK_PID_DIR="${SPARK_HOME}/run"
-mkdir -p "${SPARK_PID_DIR}"
-
-# start
-export SPARK_MASTER_PORT=7071
-export SPARK_MASTER_WEBUI_PORT=7072
-export SPARK_WORKER_WEBUI_PORT=8082
-${SPARK_HOME}/sbin/start-master.sh
-
-if [ "${SPARK_VER_RANGE}" == "<=1.3" ]||[ "${SPARK_VER_RANGE}" == "<=1.2" ]; then
- # spark 1.3 or prior
- ${SPARK_HOME}/sbin/start-slave.sh 1 `hostname`:${SPARK_MASTER_PORT}
-else
- ${SPARK_HOME}/sbin/start-slave.sh spark://`hostname`:7071
-fi
-
-set +xe
diff --git a/zeppelin-display/src/main/scala/org/apache/zeppelin/display/angular/notebookscope/AngularElem.scala b/zeppelin-display/src/main/scala/org/apache/zeppelin/display/angular/notebookscope/AngularElem.scala
index 9ba88ffd916..53dac43fcd8 100644
--- a/zeppelin-display/src/main/scala/org/apache/zeppelin/display/angular/notebookscope/AngularElem.scala
+++ b/zeppelin-display/src/main/scala/org/apache/zeppelin/display/angular/notebookscope/AngularElem.scala
@@ -71,7 +71,7 @@ object AngularElem {
}
/**
- * Disassociate (remove) all angular object in this notebook
+ * Disassociate (remove) all angular object in this note
*/
def disassociate() = {
val ic = InterpreterContext.get
diff --git a/zeppelin-display/src/main/scala/org/apache/zeppelin/display/angular/paragraphscope/AngularElem.scala b/zeppelin-display/src/main/scala/org/apache/zeppelin/display/angular/paragraphscope/AngularElem.scala
index 50bd0ed4094..bc0501755ff 100644
--- a/zeppelin-display/src/main/scala/org/apache/zeppelin/display/angular/paragraphscope/AngularElem.scala
+++ b/zeppelin-display/src/main/scala/org/apache/zeppelin/display/angular/paragraphscope/AngularElem.scala
@@ -73,7 +73,7 @@ object AngularElem {
}
/**
- * Disassociate (remove) all angular object in this notebook
+ * Disassociate (remove) all angular object in this note
*/
def disassociate() = {
val ic = InterpreterContext.get
diff --git a/zeppelin-distribution/src/bin_license/LICENSE b/zeppelin-distribution/src/bin_license/LICENSE
index b33ad64b108..99853d77c62 100644
--- a/zeppelin-distribution/src/bin_license/LICENSE
+++ b/zeppelin-distribution/src/bin_license/LICENSE
@@ -42,10 +42,12 @@ The following components are provided under Apache License.
(Apache 2.0) Apache Kylin (http://kylin.apache.org/)
(Apache 2.0) Apache Lens (http://lens.apache.org/)
(Apache 2.0) Apache Flink (http://flink.apache.org/)
+ (Apache 2.0) Apache Beam (http://beam.apache.org/)
(Apache 2.0) Apache Thrift (http://thrift.apache.org/)
(Apache 2.0) Apache Lucene (https://lucene.apache.org/)
(Apache 2.0) Apache Zookeeper (org.apache.zookeeper:zookeeper:jar:3.4.5 - http://zookeeper.apache.org/)
(Apache 2.0) Chill (com.twitter:chill-java:jar:0.8.0 - https://github.com/twitter/chill/)
+ (Apache 2.0) QDox (com.thoughtworks.qdox:qdox:jar:2.0-M3 - https://github.com/paul-hammant/qdox/)
(Apache 2.0) Codehaus Plexus (org.codehaus.plexus:plexus:jar:1.5.6 - https://codehaus-plexus.github.io/)
(Apache 2.0) findbugs jsr305 (com.google.code.findbugs:jsr305:jar:1.3.9 - http://findbugs.sourceforge.net/)
(Apache 2.0) Google Guava (com.google.guava:guava:15.0 - https://code.google.com/p/guava-libraries/)
@@ -114,7 +116,53 @@ The following components are provided under Apache License.
(Apache 2.0) Utility classes for Jetty (org.mortbay.jetty:jetty-util:6.1.26 - http://javadox.com/org.mortbay.jetty/jetty/6.1.26/overview-tree.html)
(Apache 2.0) Servlet API (org.mortbay.jetty:servlet-api:2.5-20081211 - https://en.wikipedia.org/wiki/Jetty_(web_server))
(Apache 2.0) Google HTTP Client Library for Java (com.google.http-client:google-http-client-jackson2:1.21.0 - https://github.com/google/google-http-java-client/tree/dev/google-http-client-jackson2)
- (Apache 2.0) angular-esri-map (https://github.com/Esri/angular-esri-map)
+ (Apache 2.0) pegdown (org.pegdown:pegdown:1.6.0 - https://github.com/sirthias/pegdown)
+ (Apache 2.0) parboiled-java (org.parboiled:parboiled-java:1.1.7 - https://github.com/sirthias/parboiled)
+ (Apache 2.0) parboiled-core (org.parboiled:parboiled-core:1.1.7 - https://github.com/sirthias/parboiled)
+ (Apache 2.0) ZkClient (com.101tec:zkclient:0.7 - https://github.com/sgroschupf/zkclient)
+ (Apache 2.0) jackson-module-scala (com.fasterxml.jackson.module:jackson-module-scala_2.10:2.4.4 - http://wiki.fasterxml.com/JacksonModuleScala)
+ (Apache 2.0) BigQuery API v2-rev295-1.22.0 (com.google.apis:google-api-services-bigquery:v2-rev295-1.22.0 - http://nexus.sonatype.org/oss-repository-hosting.html/google-api-services-bigquery)
+ (Apache 2.0) Google Cloud Debugger API v2-rev8-1.22.0 (com.google.apis:google-api-services-clouddebugger:v2-rev8-1.22.0 - http://nexus.sonatype.org/oss-repository-hosting.html/google-api-services-clouddebugger)
+ (Apache 2.0) Google Dataflow API v1b3-rev30-1.22.0 (com.google.apis:google-api-services-dataflow:v1b3-rev30-1.22.0 - http://nexus.sonatype.org/oss-repository-hosting.html/google-api-services-dataflow)
+ (Apache 2.0) Google Cloud Pub/Sub API v1-rev10-1.22.0 (com.google.apis:google-api-services-pubsub:v1-rev10-1.22.0 - http://nexus.sonatype.org/oss-repository-hosting.html/google-api-services-pubsub)
+ (Apache 2.0) Cloud Storage JSON API v1-rev71-1.22.0 (com.google.apis:google-api-services-storage:v1-rev71-1.22.0 - http://nexus.sonatype.org/oss-repository-hosting.html/google-api-services-storage)
+ (Apache 2.0) gcsio.jar (com.google.cloud.bigdataoss:gcsio:1.4.5 - https://github.com/GoogleCloudPlatform/BigData-interop/gcsio/)
+ (Apache 2.0) util (com.google.cloud.bigdataoss:util:1.4.5 - https://github.com/GoogleCloudPlatform/BigData-interop/util/)
+ (Apache 2.0) Google Guice - Core Library (com.google.inject:guice:3.0 - http://code.google.com/p/google-guice/guice/)
+ (Apache 2.0) OkHttp (com.squareup.okhttp:okhttp:2.5.0 - https://github.com/square/okhttp/okhttp)
+ (Apache 2.0) Okio (com.squareup.okio:okio:1.6.0 - https://github.com/square/okio/okio)
+ (Apache 2.0) config (com.typesafe:config:1.2.1 - https://github.com/typesafehub/config)
+ (Apache 2.0) akka-actor (com.typesafe.akka:akka-actor_2.10:2.3.7 - http://akka.io/)
+ (Apache 2.0) akka-remote (com.typesafe.akka:akka-remote_2.10:2.3.7 - http://akka.io/)
+ (Apache 2.0) akka-slf4j (com.typesafe.akka:akka-slf4j_2.10:2.3.7 - http://akka.io/)
+ (Apache 2.0) Metrics Core Library (com.yammer.metrics:metrics-core:2.2.0 - http://metrics.codahale.com/metrics-core/)
+ (Apache 2.0) Commons BeanUtils Bean Collections (commons-beanutils:commons-beanutils-bean-collections:1.8.3 - http://commons.apache.org/beanutils/)
+ (Apache 2.0) Metrics Core (io.dropwizard.metrics:metrics-core:3.1.0 - http://metrics.codahale.com/metrics-core/)
+ (Apache 2.0) Graphite Integration for Metrics (io.dropwizard.metrics:metrics-graphite:3.1.0 - http://metrics.codahale.com/metrics-graphite/)
+ (Apache 2.0) Jackson Integration for Metrics (io.dropwizard.metrics:metrics-json:3.1.0 - http://metrics.codahale.com/metrics-json/)
+ (Apache 2.0) JVM Integration for Metrics (io.dropwizard.metrics:metrics-jvm:3.1.0 - http://metrics.codahale.com/metrics-jvm/)
+ (Apache 2.0) Apache Log4j (log4j:log4j:1.2.17 - http://logging.apache.org/log4j/1.2/)
+ (Apache 2.0) Apache Avro IPC (org.apache.avro:avro-ipc:1.8.1 - http://avro.apache.org)
+ (Apache 2.0) Apache Avro Mapred API (org.apache.avro:avro-mapred:1.8.1 - http://avro.apache.org/avro-mapred)
+ (Apache 2.0) Apache Ivy (org.apache.ivy:ivy:2.4.0 - http://ant.apache.org/ivy/)
+ (Apache 2.0) Apache Kafka (org.apache.kafka:kafka-clients:0.8.2.2 - http://kafka.apache.org)
+ (Apache 2.0) Apache Kafka (org.apache.kafka:kafka_2.10:0.8.2.2 - http://kafka.apache.org)
+ (Apache 2.0) mesos (org.apache.mesos:mesos:0.21.1 - http://mesos.apache.org)
+ (Apache 2.0) Apache Sling JSON Library (org.apache.sling:org.apache.sling.commons.json:2.0.6 - http://sling.apache.org/org.apache.sling.commons.json)
+ (Apache 2.0) Apache Velocity (org.apache.velocity:velocity:1.7 - http://velocity.apache.org/engine/devel/)
+ (Apache 2.0) jasper-compiler (tomcat:jasper-compiler:5.5.23 - http://tomcat.apache.org/jasper-compiler)
+ (Apache 2.0) jasper-runtime (tomcat:jasper-runtime:5.5.23 - http://tomcat.apache.org/jasper-runtime)
+ (Apache 2.0) Tachyon Project Core (org.tachyonproject:tachyon:0.6.4 - http://tachyonproject.org/tachyon/)
+ (Apache 2.0) Tachyon Project Client (org.tachyonproject:tachyon-client:0.6.4 - http://tachyonproject.org/tachyon-client/)
+ (Apache 2.0) javax.inject (javax.inject:javax.inject:1 - http://code.google.com/p/atinject/)
+ (Apache 2.0) Apache Pig (org.apache.pig:0.16 - http://pig.apache.org)
+ (Apache 2.0) tez-api (org.apache.tez:tez-api:0.7.0 - http://tez.apache.org)
+ (Apache 2.0) tez-common (org.apache.tez:tez-common:0.7.0 - http://tez.apache.org)
+ (Apache 2.0) tez-dag (org.apache.tez:tez-dag:0.7.0 - http://tez.apache.org)
+ (Apache 2.0) tez-runtime-library (org.apache.tez:runtime-library:0.7.0 - http://tez.apache.org)
+ (Apache 2.0) tez-runtime-internals (org.apache.tez:tez-runtime-internals:0.7.0 - http://tez.apache.org)
+ (Apache 2.0) tez-mapreduce (org.apache.tez:tez-mapreduce:0.7.0 - http://tez.apache.org)
+ (Apache 2.0) tez-yarn-timeline-history-with-acls (org.apache.tez:tez-yarn-timeline-history-with-acls:0.7.0 - http://tez.apache.org)
========================================================================
MIT licenses
@@ -151,7 +199,8 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
(The MIT License) bcprov-jdk15on v1.51 (org.bouncycastle:bcprov-jdk15on:jar:1.51 - http://www.bouncycastle.org/java.html) - http://www.bouncycastle.org/licence.html
(The MIT License) AnchorJS (https://github.com/bryanbraun/anchorjs) - https://github.com/bryanbraun/anchorjs/blob/master/README.md#license
(The MIT License) moment-duration-format v1.3.0 (https://github.com/jsmreese/moment-duration-format) - https://github.com/jsmreese/moment-duration-format/blob/master/LICENSE
-
+ (The MIT License) github-markdown-css 2.4.0 (https://github.com/sindresorhus/github-markdown-css) - https://github.com/sindresorhus/github-markdown-css/blob/gh-pages/license
+ (The MIT License) scopt (com.github.scopt:scopt_2.10:3.2.0 - https://github.com/scopt/scopt)
The following components are provided under the MIT License.
(The MIT License) Objenesis (org.objenesis:objenesis:2.1 - https://github.com/easymock/objenesis) - Copyright (c) 2006-2015 the original author and authors
@@ -176,9 +225,22 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
(BSD 3 Clause) highlightjs v9.4.0 (https://highlightjs.org/) - https://github.com/isagalaev/highlight.js/blob/9.4.0/LICENSE
(BSD 3 Clause) hamcrest v1.3 (http://hamcrest.org/JavaHamcrest/) - http://opensource.org/licenses/BSD-3-Clause
(BSD Style) JLine v2.12.1 (https://github.com/jline/jline2) - https://github.com/jline/jline2/blob/master/LICENSE.txt
-
-
-
+ (BSD New license) Google Auth Library for Java - Credentials (com.google.auth:google-auth-library-credentials:0.4.0 - https://github.com/google/google-auth-library-java/google-auth-library-credentials)
+ (BSD New license) Google Auth Library for Java - OAuth2 HTTP (com.google.auth:google-auth-library-oauth2-http:0.4.0 - https://github.com/google/google-auth-library-java/google-auth-library-oauth2-http)
+ (New BSD license) Protocol Buffer Java API (com.google.protobuf:protobuf-java-util:3.0.0-beta-2 - https://developers.google.com/protocol-buffers/)
+ (New BSD license) Protocol Buffer JavaNano API (com.google.protobuf.nano:protobuf-javanano:3.0.0-alpha-5 - https://developers.google.com/protocol-buffers/)
+ (BSD) JSch (com.jcraft:jsch:0.1.42 - http://www.jcraft.com/jsch/)
+ (BSD 3-Clause) io.grpc:grpc-all (io.grpc:grpc-all:0.14.1 - https://github.com/grpc/grpc-java)
+ (BSD 3-Clause) io.grpc:grpc-auth (io.grpc:grpc-auth:0.14.1 - https://github.com/grpc/grpc-java)
+ (BSD 3-Clause) io.grpc:grpc-core (io.grpc:grpc-core:0.14.1 - https://github.com/grpc/grpc-java)
+ (BSD 3-Clause) io.grpc:grpc-netty (io.grpc:grpc-netty:0.14.1 - https://github.com/grpc/grpc-java)
+ (BSD 3-Clause) io.grpc:grpc-okhttp (io.grpc:grpc-okhttp:0.14.1 - https://github.com/grpc/grpc-java)
+ (BSD 3-Clause) io.grpc:grpc-protobuf (io.grpc:grpc-protobuf:0.14.1 - https://github.com/grpc/grpc-java)
+ (BSD 3-Clause) io.grpc:grpc-protobuf-lite (io.grpc:grpc-protobuf-lite:0.14.1 - https://github.com/grpc/grpc-java)
+ (BSD 3-Clause) io.grpc:grpc-protobuf-nano (io.grpc:grpc-protobuf-nano:0.14.1 - https://github.com/grpc/grpc-java)
+ (BSD 3-Clause) io.grpc:grpc-stub (io.grpc:grpc-stub:0.14.1 - https://github.com/grpc/grpc-java)
+
+
The following components are provided under the BSD-style License.
(New BSD License) JGit (org.eclipse.jgit:org.eclipse.jgit:jar:4.1.1.201511131810-r - https://eclipse.org/jgit/)
@@ -191,9 +253,13 @@ The following components are provided under the BSD-style License.
(BSD-like) Scala Actors library (org.scala-lang:scala-actors:2.11.7 - http://www.scala-lang.org/)
(BSD-like) Scala Compiler (org.scala-lang:scala-compiler:2.11.7 - http://www.scala-lang.org/)
(BSD-like) Scala Compiler (org.scala-lang:scala-reflect:2.11.7 - http://www.scala-lang.org/)
- (BSD-like) ASM (asm:asm:jar:3.1 - http://asm.ow2.org/) - Copyright (c) 2000-2011 INRIA, France Telecom
+ (BSD-like) ASM asm (asm:asm:jar:3.1 - http://asm.ow2.org/) - Copyright (c) 2000-2011 INRIA, France Telecom
+ (BSD-like) ASM asm-tree (org.ow2.asm:asm-tree:5.0.3 - http://asm.ow2.org/) - Copyright (c) 2000-2011 INRIA, France Telecom
+ (BSD-like) ASM asm-analysis (org.ow2.asm:asm-analysis:5.0.3 - http://asm.ow2.org/) - Copyright (c) 2000-2011 INRIA, France Telecom
+ (BSD-like) ASM asm-utils (org.ow2.asm:asm-utils:5.0.3 - http://asm.ow2.org/) - Copyright (c) 2000-2011 INRIA, France Telecom
+ (New BSD License) Markdown4j (org.commonjava.googlecode.markdown4j:markdown4j:jar:2.2-cj-1.0 - https://code.google.com/p/markdown4j/)
(New BSD License) Py4J (net.sf.py4j:py4j:0.9 - http://py4j.sourceforge.net/)
- (New BSD License) Py4J (net.sf.py4j:py4j:0.10.1 - http://py4j.sourceforge.net/) - https://github.com/bartdag/py4j/blob/0.10.1/LICENSE.txt
+ (New BSD License) Py4J (net.sf.py4j:py4j:0.10.3 - http://py4j.sourceforge.net/) - https://github.com/bartdag/py4j/blob/0.10.3/LICENSE.txt
(New BSD License) Markdown4j (org.commonjava.googlecode.markdown4j:markdown4j:jar:2.2-cj-1.0 - https://code.google.com/p/markdown4j/)
(BSD 3 Clause) Paranamer (com.thoughtworks.paranamer:paranamer:jar:2.6) - https://github.com/paul-hammant/paranamer/blob/paranamer-parent-2.6/LICENSE.txt
(BSD 3 Clause) netlib core (com.github.fommil.netlib:core:1.1.2 - https://github.com/fommil/netlib-java/core)
@@ -218,8 +284,15 @@ The following components are provided under the CDDL License.
(CDDL 1.1) Jersey (com.sun.jersey:jersey:jar:1.9 - https://jersey.java.net/)
(CDDL 1.1) jersey-core (org.glassfish.jersey.core:jersey-core:2.22.2 - https://jersey.java.net/)
(CDDL 1.1) hk2 (org.glassfish.hk2 - https://hk2.java.net/2.5.0-b03/)
-
-
+ (CDDL 1.1) jersey-core (com.sun.jersey:jersey-core:1.9 - https://jersey.java.net/jersey-core/)
+ (CDDL 1.1) jersey-json (com.sun.jersey:jersey-json:1.9 - https://jersey.java.net/jersey-json/)
+ (CDDL 1.1) jersey-server (com.sun.jersey:jersey-server:1.9 - https://jersey.java.net/jersey-server/)
+ (CDDL 1.1) jersey-guice (com.sun.jersey.contribs:jersey-guice:1.9 - https://jersey.java.net/jersey-contribs/jersey-guice/)
+ (CDDL 1.1) JAXB RI (com.sun.xml.bind:jaxb-impl:2.2.3-1 - http://jaxb.java.net/)
+ (CDDL 1.0) Java Servlet API (javax.servlet:javax.servlet-api:3.1.0 - http://servlet-spec.java.net)
+ (CDDL 1.1) (GPL2 w/ CPE) JAXB API bundle for GlassFish V3 (javax.xml.bind:jaxb-api:2.2.2 - https://jaxb.dev.java.net/)
+ (CDDL 1.0) (GNU General Public Library) Streaming API for XML (javax.xml.stream:stax-api:1.0-2 - no url defined)
+
========================================================================
EPL license
@@ -264,3 +337,5 @@ Creative Commons CC0 (http://creativecommons.org/publicdomain/zero/1.0/)
(CC0 1.0 Universal) JSR166e (com.twitter:jsr166e:1.1.0 - http://github.com/twitter/jsr166e)
(Public Domain, per Creative Commons CC0) HdrHistogram (org.hdrhistogram:HdrHistogram:2.1.6 - http://hdrhistogram.github.io/HdrHistogram/)
+ (Public Domain) XZ for Java (org.tukaani:xz:1.0 - http://tukaani.org/xz/java.html)
+ (Public Domain) AOP alliance (aopalliance:aopalliance:1.0 - http://aopalliance.sourceforge.net)
diff --git a/zeppelin-distribution/src/bin_license/licenses/LICENSE-github-markdown-css-2.4.0 b/zeppelin-distribution/src/bin_license/licenses/LICENSE-github-markdown-css-2.4.0
new file mode 100644
index 00000000000..654d0bfe943
--- /dev/null
+++ b/zeppelin-distribution/src/bin_license/licenses/LICENSE-github-markdown-css-2.4.0
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) Sindre Sorhus (sindresorhus.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java
index 1e0844efaac..da3d0c02b4f 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java
@@ -72,7 +72,7 @@ public void addRepo(String id, String url, boolean snapshot) {
}
}
- public void addRepo(String id, String url, boolean snapshot, Authentication auth) {
+ public void addRepo(String id, String url, boolean snapshot, Authentication auth, Proxy proxy) {
synchronized (repos) {
delRepo(id);
RemoteRepository rr = new RemoteRepository(id, "default", url);
@@ -81,6 +81,7 @@ public void addRepo(String id, String url, boolean snapshot, Authentication auth
RepositoryPolicy.UPDATE_POLICY_DAILY,
RepositoryPolicy.CHECKSUM_POLICY_WARN));
rr.setAuthentication(auth);
+ rr.setProxy(proxy);
repos.add(rr);
}
}
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
index 214175a2640..7ca4a4d173d 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
@@ -33,6 +33,7 @@
import org.sonatype.aether.RepositoryException;
import org.sonatype.aether.artifact.Artifact;
import org.sonatype.aether.collection.CollectRequest;
+import org.sonatype.aether.collection.DependencyCollectionException;
import org.sonatype.aether.graph.Dependency;
import org.sonatype.aether.graph.DependencyFilter;
import org.sonatype.aether.repository.RemoteRepository;
@@ -42,6 +43,7 @@
import org.sonatype.aether.util.artifact.JavaScopes;
import org.sonatype.aether.util.filter.DependencyFilterUtils;
import org.sonatype.aether.util.filter.PatternExclusionsDependencyFilter;
+import org.sonatype.aether.util.graph.DefaultDependencyNode;
/**
@@ -104,6 +106,21 @@ public List load(String artifact, Collection excludes, File destPa
return libs;
}
+ public synchronized void copyLocalDependency(String srcPath, File destPath)
+ throws IOException {
+ if (StringUtils.isBlank(srcPath)) {
+ return;
+ }
+
+ File srcFile = new File(srcPath);
+ File destFile = new File(destPath, srcFile.getName());
+
+ if (!destFile.exists() || !FileUtils.contentEquals(srcFile, destFile)) {
+ FileUtils.copyFile(srcFile, destFile);
+ logger.info("copy {} to {}", srcFile.getAbsolutePath(), destPath);
+ }
+ }
+
private List loadFromMvn(String artifact, Collection excludes)
throws RepositoryException {
Collection allExclusions = new LinkedList();
@@ -142,11 +159,11 @@ private List loadFromMvn(String artifact, Collection excludes)
*/
@Override
public List getArtifactsWithDep(String dependency,
- Collection excludes) throws RepositoryException {
+ Collection excludes) throws RepositoryException {
Artifact artifact = new DefaultArtifact(dependency);
DependencyFilter classpathFilter = DependencyFilterUtils.classpathFilter(JavaScopes.COMPILE);
PatternExclusionsDependencyFilter exclusionFilter =
- new PatternExclusionsDependencyFilter(excludes);
+ new PatternExclusionsDependencyFilter(excludes);
CollectRequest collectRequest = new CollectRequest();
collectRequest.setRoot(new Dependency(artifact, JavaScopes.COMPILE));
@@ -157,7 +174,11 @@ public List getArtifactsWithDep(String dependency,
}
}
DependencyRequest dependencyRequest = new DependencyRequest(collectRequest,
- DependencyFilterUtils.andFilter(exclusionFilter, classpathFilter));
- return system.resolveDependencies(session, dependencyRequest).getArtifactResults();
+ DependencyFilterUtils.andFilter(exclusionFilter, classpathFilter));
+ try {
+ return system.resolveDependencies(session, dependencyRequest).getArtifactResults();
+ } catch (NullPointerException ex) {
+ throw new RepositoryException(String.format("Cannot fetch dependencies for %s", dependency));
+ }
}
}
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Repository.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Repository.java
index d2b00924607..34fe4f05478 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Repository.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Repository.java
@@ -16,7 +16,11 @@
*/
package org.apache.zeppelin.dep;
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+
import org.sonatype.aether.repository.Authentication;
+import org.sonatype.aether.repository.Proxy;
+
/**
*
*
@@ -27,6 +31,11 @@ public class Repository {
private String url;
private String username = null;
private String password = null;
+ private String proxyProtocol = "HTTP";
+ private String proxyHost = null;
+ private Integer proxyPort = null;
+ private String proxyLogin = null;
+ private String proxyPassword = null;
public Repository(String id){
this.id = id;
@@ -77,4 +86,16 @@ public Authentication getAuthentication() {
}
return auth;
}
+
+ public Proxy getProxy() {
+ if (isNotBlank(proxyHost) && proxyPort != null) {
+ if (isNotBlank(proxyLogin)) {
+ return new Proxy(proxyProtocol, proxyHost, proxyPort,
+ new Authentication(proxyLogin, proxyPassword));
+ } else {
+ return new Proxy(proxyProtocol, proxyHost, proxyPort, null);
+ }
+ }
+ return null;
+ }
}
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
index 42caafdfc61..3e323200f6e 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
@@ -27,6 +27,7 @@
import com.google.gson.annotations.SerializedName;
import org.apache.zeppelin.annotation.ZeppelinApi;
+import org.apache.zeppelin.annotation.Experimental;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
@@ -203,6 +204,71 @@ public void setClassloaderUrls(URL[] classloaderUrls) {
this.classloaderUrls = classloaderUrls;
}
+ /**
+ * General function to register hook event
+ * @param noteId - Note to bind hook to
+ * @param event The type of event to hook to (pre_exec, post_exec)
+ * @param cmd The code to be executed by the interpreter on given event
+ */
+ @Experimental
+ public void registerHook(String noteId, String event, String cmd) {
+ InterpreterHookRegistry hooks = interpreterGroup.getInterpreterHookRegistry();
+ String className = getClassName();
+ hooks.register(noteId, className, event, cmd);
+ }
+
+ /**
+ * registerHook() wrapper for global scope
+ * @param event The type of event to hook to (pre_exec, post_exec)
+ * @param cmd The code to be executed by the interpreter on given event
+ */
+ @Experimental
+ public void registerHook(String event, String cmd) {
+ registerHook(null, event, cmd);
+ }
+
+ /**
+ * Get the hook code
+ * @param noteId - Note to bind hook to
+ * @param event The type of event to hook to (pre_exec, post_exec)
+ */
+ @Experimental
+ public String getHook(String noteId, String event) {
+ InterpreterHookRegistry hooks = interpreterGroup.getInterpreterHookRegistry();
+ String className = getClassName();
+ return hooks.get(noteId, className, event);
+ }
+
+ /**
+ * getHook() wrapper for global scope
+ * @param event The type of event to hook to (pre_exec, post_exec)
+ */
+ @Experimental
+ public String getHook(String event) {
+ return getHook(null, event);
+ }
+
+ /**
+ * Unbind code from given hook event
+ * @param noteId - Note to bind hook to
+ * @param event The type of event to hook to (pre_exec, post_exec)
+ */
+ @Experimental
+ public void unregisterHook(String noteId, String event) {
+ InterpreterHookRegistry hooks = interpreterGroup.getInterpreterHookRegistry();
+ String className = getClassName();
+ hooks.unregister(noteId, className, event);
+ }
+
+ /**
+ * unregisterHook() wrapper for global scope
+ * @param event The type of event to hook to (pre_exec, post_exec)
+ */
+ @Experimental
+ public void unregisterHook(String event) {
+ unregisterHook(null, event);
+ }
+
@ZeppelinApi
public Interpreter getInterpreterInTheSameSessionByClassName(String className) {
synchronized (interpreterGroup) {
@@ -251,6 +317,7 @@ public static class RegisteredInterpreter {
private String className;
private boolean defaultInterpreter;
private Map properties;
+ private Map editor;
private String path;
public RegisteredInterpreter(String name, String group, String className,
@@ -266,6 +333,7 @@ public RegisteredInterpreter(String name, String group, String className,
this.className = className;
this.defaultInterpreter = defaultInterpreter;
this.properties = properties;
+ this.editor = new HashMap<>();
}
public String getName() {
@@ -292,6 +360,10 @@ public Map getProperties() {
return properties;
}
+ public Map getEditor() {
+ return editor;
+ }
+
public void setPath(String path) {
this.path = path;
}
@@ -338,9 +410,9 @@ public static void register(String name, String group, String className,
@Deprecated
public static void register(String name, String group, String className,
boolean defaultInterpreter, Map properties) {
- logger.error("Static initialization is deprecated. You should change it to use " +
- "interpreter-setting.json in your jar or " +
- "interpreter/{interpreter}/interpreter-setting.json");
+ logger.warn("Static initialization is deprecated for interpreter {}, You should change it " +
+ "to use interpreter-setting.json in your jar or " +
+ "interpreter/{interpreter}/interpreter-setting.json", name);
register(new RegisteredInterpreter(name, group, className, defaultInterpreter, properties));
}
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java
index 21ca2e67b72..e33b9352252 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java
@@ -57,6 +57,7 @@ public static void remove() {
private AngularObjectRegistry angularObjectRegistry;
private ResourcePool resourcePool;
private List runners;
+ private String className;
public InterpreterContext(String noteId,
String paragraphId,
@@ -124,4 +125,11 @@ public List getRunners() {
return runners;
}
+ public String getClassName() {
+ return className;
+ }
+
+ public void setClassName(String className) {
+ this.className = className;
+ }
}
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
index bc56784b15b..f3b158c26a6 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
@@ -33,7 +33,7 @@
* and InterpreterGroup will have reference to these all interpreters.
*
* Remember, list of interpreters are dedicated to a note.
- * (when InterpreterOption.perNoteSession==true)
+ * (when InterpreterOption.session==true)
* So InterpreterGroup internally manages map of [noteId, list of interpreters]
*
* A InterpreterGroup runs on interpreter process.
@@ -45,6 +45,7 @@ public class InterpreterGroup extends ConcurrentHashMap intpForNote = this.get(noteId);
destroy(intpForNote);
+
+ if (remoteInterpreterProcess != null) {
+ remoteInterpreterProcess.dereference();
+ if (remoteInterpreterProcess.referenceCount() <= 0) {
+ remoteInterpreterProcess = null;
+ allInterpreterGroups.remove(id);
+ }
+ }
}
@@ -213,6 +230,7 @@ public void destroy() {
while (remoteInterpreterProcess.referenceCount() > 0) {
remoteInterpreterProcess.dereference();
}
+ remoteInterpreterProcess = null;
}
allInterpreterGroups.remove(id);
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterHookListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterHookListener.java
new file mode 100644
index 00000000000..c70212c7b7e
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterHookListener.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter;
+
+/**
+ * An interface for processing custom callback code into the interpreter.
+ */
+public interface InterpreterHookListener {
+ /**
+ * Prepends pre-execute hook code to the script that will be interpreted
+ */
+ public void onPreExecute(String script);
+
+ /**
+ * Appends post-execute hook code to the script that will be interpreted
+ */
+ public void onPostExecute(String script);
+}
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterHookRegistry.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterHookRegistry.java
new file mode 100644
index 00000000000..0917775d692
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterHookRegistry.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * The InterpreterinterpreterHookRegistry specifies code to be conditionally executed by an
+ * interpreter. The constants defined in this class denote currently
+ * supported events. Each instance is bound to a single InterpreterGroup.
+ * Scope is determined on a per-note basis (except when null for global scope).
+ */
+public class InterpreterHookRegistry {
+ public static final String GLOBAL_KEY = "_GLOBAL_";
+ private String interpreterId;
+ private Map>> registry =
+ new HashMap>>();
+
+ /**
+ * hookRegistry constructor.
+ *
+ * @param interpreterId The Id of the InterpreterGroup instance to bind to
+ */
+ public InterpreterHookRegistry(final String interpreterId) {
+ this.interpreterId = interpreterId;
+ }
+
+ /**
+ * Get the interpreterGroup id this instance is bound to
+ */
+ public String getInterpreterId() {
+ return interpreterId;
+ }
+
+ /**
+ * Adds a note to the registry
+ *
+ * @param noteId The Id of the Note instance to add
+ */
+ public void addNote(String noteId) {
+ synchronized (registry) {
+ if (registry.get(noteId) == null) {
+ registry.put(noteId, new HashMap>());
+ }
+ }
+ }
+
+ /**
+ * Adds a className to the registry
+ *
+ * @param noteId The note id
+ * @param className The name of the interpreter repl to map the hooks to
+ */
+ public void addRepl(String noteId, String className) {
+ synchronized (registry) {
+ addNote(noteId);
+ if (registry.get(noteId).get(className) == null) {
+ registry.get(noteId).put(className, new HashMap());
+ }
+ }
+ }
+
+ /**
+ * Register a hook for a specific event.
+ *
+ * @param noteId Denotes the note this instance belongs to
+ * @param className The name of the interpreter repl to map the hooks to
+ * @param event hook event (see constants defined in this class)
+ * @param cmd Code to be executed by the interpreter
+ */
+ public void register(String noteId, String className,
+ String event, String cmd) throws IllegalArgumentException {
+ synchronized (registry) {
+ if (noteId == null) {
+ noteId = GLOBAL_KEY;
+ }
+ addRepl(noteId, className);
+ if (!event.equals(HookType.POST_EXEC) && !event.equals(HookType.PRE_EXEC) &&
+ !event.equals(HookType.POST_EXEC_DEV) && !event.equals(HookType.PRE_EXEC_DEV)) {
+ throw new IllegalArgumentException("Must be " + HookType.POST_EXEC + ", " +
+ HookType.POST_EXEC_DEV + ", " +
+ HookType.PRE_EXEC + " or " +
+ HookType.PRE_EXEC_DEV);
+ }
+ registry.get(noteId).get(className).put(event, cmd);
+ }
+ }
+
+ /**
+ * Unregister a hook for a specific event.
+ *
+ * @param noteId Denotes the note this instance belongs to
+ * @param className The name of the interpreter repl to map the hooks to
+ * @param event hook event (see constants defined in this class)
+ */
+ public void unregister(String noteId, String className, String event) {
+ synchronized (registry) {
+ if (noteId == null) {
+ noteId = GLOBAL_KEY;
+ }
+ addRepl(noteId, className);
+ registry.get(noteId).get(className).remove(event);
+ }
+ }
+
+ /**
+ * Get a hook for a specific event.
+ *
+ * @param noteId Denotes the note this instance belongs to
+ * @param className The name of the interpreter repl to map the hooks to
+ * @param event hook event (see constants defined in this class)
+ */
+ public String get(String noteId, String className, String event) {
+ synchronized (registry) {
+ if (noteId == null) {
+ noteId = GLOBAL_KEY;
+ }
+ addRepl(noteId, className);
+ return registry.get(noteId).get(className).get(event);
+ }
+ }
+
+ /**
+ * Container for hook event type constants
+ */
+ public static final class HookType {
+ // Execute the hook code PRIOR to main paragraph code execution
+ public static final String PRE_EXEC = "pre_exec";
+
+ // Execute the hook code AFTER main paragraph code execution
+ public static final String POST_EXEC = "post_exec";
+
+ // Same as above but reserved for interpreter developers, in order to allow
+ // notebook users to use the above without overwriting registry settings
+ // that are initialized directly in subclasses of Interpreter.
+ public static final String PRE_EXEC_DEV = "pre_exec_dev";
+ public static final String POST_EXEC_DEV = "post_exec_dev";
+ }
+
+}
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/LazyOpenInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/LazyOpenInterpreter.java
index c62ab05eb1f..425ae20a4f1 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/LazyOpenInterpreter.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/LazyOpenInterpreter.java
@@ -147,4 +147,34 @@ public void setInterpreterGroup(InterpreterGroup interpreterGroup) {
public void setClassloaderUrls(URL [] urls) {
intp.setClassloaderUrls(urls);
}
+
+ @Override
+ public void registerHook(String noteId, String event, String cmd) {
+ intp.registerHook(noteId, event, cmd);
+ }
+
+ @Override
+ public void registerHook(String event, String cmd) {
+ intp.registerHook(event, cmd);
+ }
+
+ @Override
+ public String getHook(String noteId, String event) {
+ return intp.getHook(noteId, event);
+ }
+
+ @Override
+ public String getHook(String event) {
+ return intp.getHook(event);
+ }
+
+ @Override
+ public void unregisterHook(String noteId, String event) {
+ intp.unregisterHook(noteId, event);
+ }
+
+ @Override
+ public void unregisterHook(String event) {
+ intp.unregisterHook(event);
+ }
}
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
index 073b84bbda2..e0cdaa338b1 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
@@ -298,6 +298,7 @@ public InterpreterResult interpret(String st, InterpreterContext context) {
if (logger.isDebugEnabled()) {
logger.debug("st:\n{}", st);
}
+
FormType form = getFormType();
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
Client client = null;
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
index 7ddb92838f4..0a7b1ed6912 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
@@ -33,6 +33,8 @@
import org.apache.zeppelin.display.*;
import org.apache.zeppelin.helium.*;
import org.apache.zeppelin.interpreter.*;
+import org.apache.zeppelin.interpreter.InterpreterHookRegistry.HookType;
+import org.apache.zeppelin.interpreter.InterpreterHookListener;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.dev.ZeppelinDevServer;
import org.apache.zeppelin.interpreter.thrift.*;
@@ -60,6 +62,7 @@ public class RemoteInterpreterServer
InterpreterGroup interpreterGroup;
AngularObjectRegistry angularObjectRegistry;
+ InterpreterHookRegistry hookRegistry;
DistributedResourcePool resourcePool;
private ApplicationLoader appLoader;
@@ -152,7 +155,9 @@ public void createInterpreter(String interpreterGroupId, String noteId, String
if (interpreterGroup == null) {
interpreterGroup = new InterpreterGroup(interpreterGroupId);
angularObjectRegistry = new AngularObjectRegistry(interpreterGroup.getId(), this);
+ hookRegistry = new InterpreterHookRegistry(interpreterGroup.getId());
resourcePool = new DistributedResourcePool(interpreterGroup.getId(), eventClient);
+ interpreterGroup.setInterpreterHookRegistry(hookRegistry);
interpreterGroup.setAngularObjectRegistry(angularObjectRegistry);
interpreterGroup.setResourcePool(resourcePool);
@@ -290,6 +295,7 @@ public RemoteInterpreterResult interpret(String noteId, String className, String
}
Interpreter intp = getInterpreter(noteId, className);
InterpreterContext context = convert(interpreterContext);
+ context.setClassName(intp.getClassName());
Scheduler scheduler = intp.getScheduler();
InterpretJobListener jobListener = new InterpretJobListener();
@@ -383,10 +389,61 @@ public Map info() {
return infos;
}
+ private void processInterpreterHooks(final String noteId) {
+ InterpreterHookListener hookListener = new InterpreterHookListener() {
+ @Override
+ public void onPreExecute(String script) {
+ String cmdDev = interpreter.getHook(noteId, HookType.PRE_EXEC_DEV);
+ String cmdUser = interpreter.getHook(noteId, HookType.PRE_EXEC);
+
+ // User defined hook should be executed before dev hook
+ List cmds = Arrays.asList(cmdDev, cmdUser);
+ for (String cmd : cmds) {
+ if (cmd != null) {
+ script = cmd + '\n' + script;
+ }
+ }
+
+ InterpretJob.this.script = script;
+ }
+
+ @Override
+ public void onPostExecute(String script) {
+ String cmdDev = interpreter.getHook(noteId, HookType.POST_EXEC_DEV);
+ String cmdUser = interpreter.getHook(noteId, HookType.POST_EXEC);
+
+ // User defined hook should be executed after dev hook
+ List cmds = Arrays.asList(cmdUser, cmdDev);
+ for (String cmd : cmds) {
+ if (cmd != null) {
+ script += '\n' + cmd;
+ }
+ }
+
+ InterpretJob.this.script = script;
+ }
+ };
+ hookListener.onPreExecute(script);
+ hookListener.onPostExecute(script);
+ }
+
@Override
protected Object jobRun() throws Throwable {
try {
InterpreterContext.set(context);
+
+ // Open the interpreter instance prior to calling interpret().
+ // This is necessary because the earliest we can register a hook
+ // is from within the open() method.
+ LazyOpenInterpreter lazy = (LazyOpenInterpreter) interpreter;
+ if (!lazy.isOpen()) {
+ lazy.open();
+ }
+
+ // Add hooks to script from registry.
+ // Global scope first, followed by notebook scope
+ processInterpreterHooks(null);
+ processInterpreterHooks(context.getNoteId());
InterpreterResult result = interpreter.interpret(script, context);
// data from context.out is prepended to InterpreterResult if both defined
@@ -504,11 +561,13 @@ protected InterpreterOutput createInterpreterOutput(final String noteId, final S
return new InterpreterOutput(new InterpreterOutputListener() {
@Override
public void onAppend(InterpreterOutput out, byte[] line) {
+ logger.debug("Output Append:" + new String(line));
eventClient.onInterpreterOutputAppend(noteId, paragraphId, new String(line));
}
@Override
public void onUpdate(InterpreterOutput out, byte[] output) {
+ logger.debug("Output Update:" + new String(output));
eventClient.onInterpreterOutputUpdate(noteId, paragraphId, new String(output));
}
});
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/user/AuthenticationInfo.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/user/AuthenticationInfo.java
index de41692607a..11d15629612 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/user/AuthenticationInfo.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/user/AuthenticationInfo.java
@@ -18,13 +18,20 @@
package org.apache.zeppelin.user;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/***
*
*/
public class AuthenticationInfo {
+ private static final Logger LOG = LoggerFactory.getLogger(AuthenticationInfo.class);
String user;
String ticket;
UserCredentials userCredentials;
+ public static final AuthenticationInfo ANONYMOUS = new AuthenticationInfo("anonymous",
+ "anonymous");
public AuthenticationInfo() {}
@@ -66,4 +73,17 @@ public void setUserCredentials(UserCredentials userCredentials) {
this.userCredentials = userCredentials;
}
+ public static boolean isAnonymous(AuthenticationInfo subject) {
+ if (subject == null) {
+ LOG.warn("Subject is null, assuming anonymous. "
+ + "Not recommended to use subject as null except in tests");
+ return true;
+ }
+ return subject.isAnonymous();
+ }
+
+ public boolean isAnonymous() {
+ return ANONYMOUS.equals(this) || "anonymous".equalsIgnoreCase(this.getUser())
+ || StringUtils.isEmpty(this.getUser());
+ }
}
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java
index af2c7ff6963..876e8e7c88c 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java
@@ -20,6 +20,7 @@
import static org.junit.Assert.assertEquals;
import java.io.File;
+import java.io.FileNotFoundException;
import java.util.Collections;
import org.apache.commons.io.FileUtils;
@@ -36,6 +37,9 @@ public class DependencyResolverTest {
private static File testCopyPath;
private static File tmpDir;
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
@BeforeClass
public static void setUp() throws Exception {
tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
@@ -90,4 +94,13 @@ public void testLoad() throws Exception {
exception.expect(RepositoryException.class);
resolver.load("com.agimatec:agimatec-validation:0.9.3", testCopyPath);
}
+
+ @Test
+ public void should_throw_exception_if_dependency_not_found() throws Exception {
+ expectedException.expectMessage("Source 'one.two:1.0' does not exist");
+ expectedException.expect(FileNotFoundException.class);
+
+ resolver.load("one.two:1.0", testCopyPath);
+ }
+
}
\ No newline at end of file
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterHookRegistryTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterHookRegistryTest.java
new file mode 100644
index 00000000000..7614e9eb204
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterHookRegistryTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Test;
+
+public class InterpreterHookRegistryTest {
+
+ @Test
+ public void testBasic() {
+ final String PRE_EXEC = InterpreterHookRegistry.HookType.PRE_EXEC;
+ final String POST_EXEC = InterpreterHookRegistry.HookType.POST_EXEC;
+ final String PRE_EXEC_DEV = InterpreterHookRegistry.HookType.PRE_EXEC_DEV;
+ final String POST_EXEC_DEV = InterpreterHookRegistry.HookType.POST_EXEC_DEV;
+ final String GLOBAL_KEY = InterpreterHookRegistry.GLOBAL_KEY;
+ final String noteId = "note";
+ final String className = "class";
+ final String preExecHook = "pre";
+ final String postExecHook = "post";
+ InterpreterHookRegistry registry = new InterpreterHookRegistry("intpId");
+
+ // Test register()
+ registry.register(noteId, className, PRE_EXEC, preExecHook);
+ registry.register(noteId, className, POST_EXEC, postExecHook);
+ registry.register(noteId, className, PRE_EXEC_DEV, preExecHook);
+ registry.register(noteId, className, POST_EXEC_DEV, postExecHook);
+
+ // Test get()
+ assertEquals(registry.get(noteId, className, PRE_EXEC), preExecHook);
+ assertEquals(registry.get(noteId, className, POST_EXEC), postExecHook);
+ assertEquals(registry.get(noteId, className, PRE_EXEC_DEV), preExecHook);
+ assertEquals(registry.get(noteId, className, POST_EXEC_DEV), postExecHook);
+
+ // Test Unregister
+ registry.unregister(noteId, className, PRE_EXEC);
+ registry.unregister(noteId, className, POST_EXEC);
+ registry.unregister(noteId, className, PRE_EXEC_DEV);
+ registry.unregister(noteId, className, POST_EXEC_DEV);
+ assertNull(registry.get(noteId, className, PRE_EXEC));
+ assertNull(registry.get(noteId, className, POST_EXEC));
+ assertNull(registry.get(noteId, className, PRE_EXEC_DEV));
+ assertNull(registry.get(noteId, className, POST_EXEC_DEV));
+
+ // Test Global Scope
+ registry.register(null, className, PRE_EXEC, preExecHook);
+ assertEquals(registry.get(GLOBAL_KEY, className, PRE_EXEC), preExecHook);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testValidEventCode() {
+ InterpreterHookRegistry registry = new InterpreterHookRegistry("intpId");
+
+ // Test that only valid event codes ("pre_exec", "post_exec") are accepted
+ registry.register("foo", "bar", "baz", "whatever");
+ }
+
+}
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/realm/ZeppelinHubRealm.java b/zeppelin-server/src/main/java/org/apache/zeppelin/realm/ZeppelinHubRealm.java
index cbe490d8de5..67ed5446842 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/realm/ZeppelinHubRealm.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/realm/ZeppelinHubRealm.java
@@ -20,6 +20,7 @@
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.HashSet;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.httpclient.HttpClient;
@@ -36,6 +37,7 @@
import org.apache.shiro.authz.AuthorizationInfo;
import org.apache.shiro.realm.AuthorizingRealm;
import org.apache.shiro.subject.PrincipalCollection;
+import org.apache.zeppelin.server.ZeppelinServer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -135,6 +137,7 @@ protected User authenticateUser(String requestBody) {
}
responseBody = put.getResponseBodyAsString();
put.releaseConnection();
+
} catch (IOException e) {
LOG.error("Cannot login user", e);
throw new AuthenticationException(e.getMessage());
@@ -147,6 +150,13 @@ protected User authenticateUser(String requestBody) {
LOG.error("Cannot deserialize ZeppelinHub response to User instance", e);
throw new AuthenticationException("Cannot login to ZeppelinHub");
}
+
+ /* TODO(khalid): add proper roles and add listener */
+ HashSet userAndRoles = new HashSet();
+ userAndRoles.add(account.login);
+ ZeppelinServer.notebookWsServer.broadcastReloadedNoteList(
+ new org.apache.zeppelin.user.AuthenticationInfo(account.login), userAndRoles);
+
return account;
}
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
index 6025b52c7f1..5b1773b584f 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
@@ -35,7 +35,6 @@
import org.apache.commons.lang.exception.ExceptionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.sonatype.aether.RepositoryException;
import org.sonatype.aether.repository.RemoteRepository;
import org.apache.zeppelin.annotation.ZeppelinApi;
@@ -73,9 +72,7 @@ public InterpreterRestApi(InterpreterFactory interpreterFactory) {
@Path("setting")
@ZeppelinApi
public Response listSettings() {
- List interpreterSettings;
- interpreterSettings = interpreterFactory.get();
- return new JsonResponse<>(Status.OK, "", interpreterSettings).build();
+ return new JsonResponse<>(Status.OK, "", interpreterFactory.get()).build();
}
/**
@@ -202,7 +199,7 @@ public Response addRepository(String message) {
try {
Repository request = gson.fromJson(message, Repository.class);
interpreterFactory.addRepository(request.getId(), request.getUrl(), request.isSnapshot(),
- request.getAuthentication());
+ request.getAuthentication(), request.getProxy());
logger.info("New repository {} added", request.getId());
} catch (Exception e) {
logger.error("Exception in InterpreterRestApi while adding repository ", e);
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRepoRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRepoRestApi.java
new file mode 100644
index 00000000000..89cb47e6b2a
--- /dev/null
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRepoRestApi.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.rest;
+
+import java.util.Collections;
+import java.util.List;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.annotation.ZeppelinApi;
+import org.apache.zeppelin.notebook.repo.NotebookRepoSync;
+import org.apache.zeppelin.notebook.repo.NotebookRepoWithSettings;
+import org.apache.zeppelin.rest.message.NotebookRepoSettingsRequest;
+import org.apache.zeppelin.server.JsonResponse;
+import org.apache.zeppelin.socket.NotebookServer;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.apache.zeppelin.utils.SecurityUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.gson.Gson;
+import com.google.gson.JsonSyntaxException;
+
+/**
+ * NoteRepo rest API endpoint.
+ *
+ */
+@Path("/notebook-repositories")
+@Produces("application/json")
+public class NotebookRepoRestApi {
+
+ private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoRestApi.class);
+
+ private Gson gson = new Gson();
+ private NotebookRepoSync noteRepos;
+ private NotebookServer notebookWsServer;
+
+ public NotebookRepoRestApi() {}
+
+ public NotebookRepoRestApi(NotebookRepoSync noteRepos, NotebookServer notebookWsServer) {
+ this.noteRepos = noteRepos;
+ this.notebookWsServer = notebookWsServer;
+ }
+
+ /**
+ * List all notebook repository
+ */
+ @GET
+ @ZeppelinApi
+ public Response listRepoSettings() {
+ AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
+ LOG.info("Getting list of NoteRepo with Settings for user {}", subject.getUser());
+ List settings = noteRepos.getNotebookRepos(subject);
+ return new JsonResponse<>(Status.OK, "", settings).build();
+ }
+
+ /**
+ * Update a specific note repo.
+ *
+ * @param message
+ * @param settingId
+ * @return
+ */
+ @PUT
+ @ZeppelinApi
+ public Response updateRepoSetting(String payload) {
+ if (StringUtils.isBlank(payload)) {
+ return new JsonResponse<>(Status.NOT_FOUND, "", Collections.emptyMap()).build();
+ }
+ AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
+ NotebookRepoSettingsRequest newSettings = NotebookRepoSettingsRequest.EMPTY;
+ try {
+ newSettings = gson.fromJson(payload, NotebookRepoSettingsRequest.class);
+ } catch (JsonSyntaxException e) {
+ LOG.error("Cannot update notebook repo settings", e);
+ return new JsonResponse<>(Status.NOT_ACCEPTABLE, "",
+ ImmutableMap.of("error", "Invalid payload structure")).build();
+ }
+
+ if (NotebookRepoSettingsRequest.isEmpty(newSettings)) {
+ LOG.error("Invalid property");
+ return new JsonResponse<>(Status.NOT_ACCEPTABLE, "",
+ ImmutableMap.of("error", "Invalid payload")).build();
+ }
+ LOG.info("User {} is going to change repo setting", subject.getUser());
+ NotebookRepoWithSettings updatedSettings =
+ noteRepos.updateNotebookRepo(newSettings.name, newSettings.settings, subject);
+ if (!updatedSettings.isEmpty()) {
+ LOG.info("Broadcasting note list to user {}", subject.getUser());
+ notebookWsServer.broadcastReloadedNoteList(subject, null);
+ }
+ return new JsonResponse<>(Status.OK, "", updatedSettings).build();
+ }
+}
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java
index 727211292b2..ae66330785d 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java
@@ -39,7 +39,6 @@
import com.google.gson.Gson;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.utils.InterpreterBindingUtils;
import org.quartz.CronExpression;
import org.slf4j.Logger;
@@ -52,7 +51,7 @@
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.rest.message.CronRequest;
import org.apache.zeppelin.types.InterpreterSettingsList;
-import org.apache.zeppelin.rest.message.NewNotebookRequest;
+import org.apache.zeppelin.rest.message.NewNoteRequest;
import org.apache.zeppelin.rest.message.NewParagraphRequest;
import org.apache.zeppelin.rest.message.RunParagraphWithParametersRequest;
import org.apache.zeppelin.search.SearchService;
@@ -62,7 +61,7 @@
import org.apache.zeppelin.utils.SecurityUtils;
/**
- * Rest api endpoint for the noteBook.
+ * Rest api endpoint for the notebook.
*/
@Path("/notebook")
@Produces("application/json")
@@ -71,7 +70,7 @@ public class NotebookRestApi {
Gson gson = new Gson();
private Notebook notebook;
private NotebookServer notebookServer;
- private SearchService notebookIndex;
+ private SearchService noteSearchService;
private NotebookAuthorization notebookAuthorization;
public NotebookRestApi() {
@@ -80,7 +79,7 @@ public NotebookRestApi() {
public NotebookRestApi(Notebook notebook, NotebookServer notebookServer, SearchService search) {
this.notebook = notebook;
this.notebookServer = notebookServer;
- this.notebookIndex = search;
+ this.noteSearchService = search;
this.notebookAuthorization = notebook.getNotebookAuthorization();
}
@@ -114,10 +113,6 @@ private String ownerPermissionError(Set current, Set allowed) th
@ZeppelinApi
public Response putNotePermissions(@PathParam("noteId") String noteId, String req)
throws IOException {
- /**
- * TODO(jl): Fixed the type of HashSet
- * https://issues.apache.org/jira/browse/ZEPPELIN-1162
- */
HashMap> permMap =
gson.fromJson(req, new TypeToken>>() {
}.getType());
@@ -162,6 +157,7 @@ public Response putNotePermissions(@PathParam("noteId") String noteId, String re
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
note.persist(subject);
notebookServer.broadcastNote(note);
+ notebookServer.broadcastNoteList(subject, userAndRoles);
return new JsonResponse<>(Status.OK).build();
}
@@ -176,12 +172,12 @@ public Response putNotePermissions(@PathParam("noteId") String noteId, String re
public Response bind(@PathParam("noteId") String noteId, String req) throws IOException {
List settingIdList = gson.fromJson(req, new TypeToken>() {
}.getType());
- notebook.bindInterpretersToNote(noteId, settingIdList);
+ notebook.bindInterpretersToNote(SecurityUtils.getPrincipal(), noteId, settingIdList);
return new JsonResponse<>(Status.OK).build();
}
/**
- * list binded setting
+ * list bound setting
*/
@GET
@Path("interpreter/bind/{noteId}")
@@ -196,17 +192,18 @@ public Response bind(@PathParam("noteId") String noteId) {
@GET
@Path("/")
@ZeppelinApi
- public Response getNotebookList() throws IOException {
+ public Response getNoteList() throws IOException {
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
- List> notesInfo = notebookServer.generateNotebooksInfo(false, subject);
+ List> notesInfo = notebookServer.generateNotesInfo(false, subject,
+ SecurityUtils.getRoles());
return new JsonResponse<>(Status.OK, "", notesInfo).build();
}
@GET
- @Path("{notebookId}")
+ @Path("{noteId}")
@ZeppelinApi
- public Response getNotebook(@PathParam("notebookId") String notebookId) throws IOException {
- Note note = notebook.getNote(notebookId);
+ public Response getNote(@PathParam("noteId") String noteId) throws IOException {
+ Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
@@ -222,9 +219,9 @@ public Response getNotebook(@PathParam("notebookId") String notebookId) throws I
* @throws IOException
*/
@GET
- @Path("export/{id}")
+ @Path("export/{noteId}")
@ZeppelinApi
- public Response exportNoteBook(@PathParam("id") String noteId) throws IOException {
+ public Response exportNote(@PathParam("noteId") String noteId) throws IOException {
String exportJson = notebook.exportNote(noteId);
return new JsonResponse<>(Status.OK, "", exportJson).build();
}
@@ -232,14 +229,14 @@ public Response exportNoteBook(@PathParam("id") String noteId) throws IOExceptio
/**
* import new note REST API
*
- * @param req - notebook Json
+ * @param req - note Json
* @return JSON with new note ID
* @throws IOException
*/
@POST
@Path("import")
@ZeppelinApi
- public Response importNotebook(String req) throws IOException {
+ public Response importNote(String req) throws IOException {
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
Note newNote = notebook.importNote(req, null, subject);
return new JsonResponse<>(Status.CREATED, "", newNote.getId()).build();
@@ -256,8 +253,8 @@ public Response importNotebook(String req) throws IOException {
@Path("/")
@ZeppelinApi
public Response createNote(String message) throws IOException {
- LOG.info("Create new notebook by JSON {}", message);
- NewNotebookRequest request = gson.fromJson(message, NewNotebookRequest.class);
+ LOG.info("Create new note by JSON {}", message);
+ NewNoteRequest request = gson.fromJson(message, NewNoteRequest.class);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
Note note = notebook.createNote(subject);
List initialParagraphs = request.getParagraphs();
@@ -277,56 +274,56 @@ public Response createNote(String message) throws IOException {
note.setName(noteName);
note.persist(subject);
notebookServer.broadcastNote(note);
- notebookServer.broadcastNoteList(subject);
+ notebookServer.broadcastNoteList(subject, SecurityUtils.getRoles());
return new JsonResponse<>(Status.CREATED, "", note.getId()).build();
}
/**
* Delete note REST API
*
- * @param notebookId ID of Notebook
+ * @param noteId ID of Note
* @return JSON with status.OK
* @throws IOException
*/
@DELETE
- @Path("{notebookId}")
+ @Path("{noteId}")
@ZeppelinApi
- public Response deleteNote(@PathParam("notebookId") String notebookId) throws IOException {
- LOG.info("Delete notebook {} ", notebookId);
+ public Response deleteNote(@PathParam("noteId") String noteId) throws IOException {
+ LOG.info("Delete note {} ", noteId);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
- if (!(notebookId.isEmpty())) {
- Note note = notebook.getNote(notebookId);
+ if (!(noteId.isEmpty())) {
+ Note note = notebook.getNote(noteId);
if (note != null) {
- notebook.removeNote(notebookId, subject);
+ notebook.removeNote(noteId, subject);
}
}
- notebookServer.broadcastNoteList(subject);
+ notebookServer.broadcastNoteList(subject, SecurityUtils.getRoles());
return new JsonResponse<>(Status.OK, "").build();
}
/**
* Clone note REST API
*
- * @param notebookId ID of Notebook
+ * @param noteId ID of Note
* @return JSON with status.CREATED
* @throws IOException, CloneNotSupportedException, IllegalArgumentException
*/
@POST
- @Path("{notebookId}")
+ @Path("{noteId}")
@ZeppelinApi
- public Response cloneNote(@PathParam("notebookId") String notebookId, String message)
+ public Response cloneNote(@PathParam("noteId") String noteId, String message)
throws IOException, CloneNotSupportedException, IllegalArgumentException {
- LOG.info("clone notebook by JSON {}", message);
- NewNotebookRequest request = gson.fromJson(message, NewNotebookRequest.class);
+ LOG.info("clone note by JSON {}", message);
+ NewNoteRequest request = gson.fromJson(message, NewNoteRequest.class);
String newNoteName = null;
if (request != null) {
newNoteName = request.getName();
}
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
- Note newNote = notebook.cloneNote(notebookId, newNoteName, subject);
+ Note newNote = notebook.cloneNote(noteId, newNoteName, subject);
notebookServer.broadcastNote(newNote);
- notebookServer.broadcastNoteList(subject);
+ notebookServer.broadcastNoteList(subject, SecurityUtils.getRoles());
return new JsonResponse<>(Status.CREATED, "", newNote.getId()).build();
}
@@ -338,13 +335,13 @@ public Response cloneNote(@PathParam("notebookId") String notebookId, String mes
* @throws IOException
*/
@POST
- @Path("{notebookId}/paragraph")
+ @Path("{noteId}/paragraph")
@ZeppelinApi
- public Response insertParagraph(@PathParam("notebookId") String notebookId, String message)
+ public Response insertParagraph(@PathParam("noteId") String noteId, String message)
throws IOException {
- LOG.info("insert paragraph {} {}", notebookId, message);
+ LOG.info("insert paragraph {} {}", noteId, message);
- Note note = notebook.getNote(notebookId);
+ Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
@@ -370,18 +367,18 @@ public Response insertParagraph(@PathParam("notebookId") String notebookId, Stri
/**
* Get paragraph REST API
*
- * @param notebookId ID of Notebook
+ * @param noteId ID of Note
* @return JSON with information of the paragraph
* @throws IOException
*/
@GET
- @Path("{notebookId}/paragraph/{paragraphId}")
+ @Path("{noteId}/paragraph/{paragraphId}")
@ZeppelinApi
- public Response getParagraph(@PathParam("notebookId") String notebookId,
+ public Response getParagraph(@PathParam("noteId") String noteId,
@PathParam("paragraphId") String paragraphId) throws IOException {
- LOG.info("get paragraph {} {}", notebookId, paragraphId);
+ LOG.info("get paragraph {} {}", noteId, paragraphId);
- Note note = notebook.getNote(notebookId);
+ Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse(Status.NOT_FOUND, "note not found.").build();
}
@@ -402,14 +399,14 @@ public Response getParagraph(@PathParam("notebookId") String notebookId,
* @throws IOException
*/
@POST
- @Path("{notebookId}/paragraph/{paragraphId}/move/{newIndex}")
+ @Path("{noteId}/paragraph/{paragraphId}/move/{newIndex}")
@ZeppelinApi
- public Response moveParagraph(@PathParam("notebookId") String notebookId,
+ public Response moveParagraph(@PathParam("noteId") String noteId,
@PathParam("paragraphId") String paragraphId, @PathParam("newIndex") String newIndex)
throws IOException {
- LOG.info("move paragraph {} {} {}", notebookId, paragraphId, newIndex);
+ LOG.info("move paragraph {} {} {}", noteId, paragraphId, newIndex);
- Note note = notebook.getNote(notebookId);
+ Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse(Status.NOT_FOUND, "note not found.").build();
}
@@ -435,18 +432,18 @@ public Response moveParagraph(@PathParam("notebookId") String notebookId,
/**
* Delete paragraph REST API
*
- * @param notebookId ID of Notebook
+ * @param noteId ID of Note
* @return JSON with status.OK
* @throws IOException
*/
@DELETE
- @Path("{notebookId}/paragraph/{paragraphId}")
+ @Path("{noteId}/paragraph/{paragraphId}")
@ZeppelinApi
- public Response deleteParagraph(@PathParam("notebookId") String notebookId,
+ public Response deleteParagraph(@PathParam("noteId") String noteId,
@PathParam("paragraphId") String paragraphId) throws IOException {
- LOG.info("delete paragraph {} {}", notebookId, paragraphId);
+ LOG.info("delete paragraph {} {}", noteId, paragraphId);
- Note note = notebook.getNote(notebookId);
+ Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse(Status.NOT_FOUND, "note not found.").build();
}
@@ -457,7 +454,7 @@ public Response deleteParagraph(@PathParam("notebookId") String notebookId,
}
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
- note.removeParagraph(paragraphId);
+ note.removeParagraph(SecurityUtils.getPrincipal(), paragraphId);
note.persist(subject);
notebookServer.broadcastNote(note);
@@ -465,19 +462,19 @@ public Response deleteParagraph(@PathParam("notebookId") String notebookId,
}
/**
- * Run notebook jobs REST API
+ * Run note jobs REST API
*
- * @param notebookId ID of Notebook
+ * @param noteId ID of Note
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@POST
- @Path("job/{notebookId}")
+ @Path("job/{noteId}")
@ZeppelinApi
- public Response runNoteJobs(@PathParam("notebookId") String notebookId)
+ public Response runNoteJobs(@PathParam("noteId") String noteId)
throws IOException, IllegalArgumentException {
- LOG.info("run notebook jobs {} ", notebookId);
- Note note = notebook.getNote(notebookId);
+ LOG.info("run note jobs {} ", noteId);
+ Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
@@ -494,19 +491,19 @@ public Response runNoteJobs(@PathParam("notebookId") String notebookId)
}
/**
- * Stop(delete) notebook jobs REST API
+ * Stop(delete) note jobs REST API
*
- * @param notebookId ID of Notebook
+ * @param noteId ID of Note
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@DELETE
- @Path("job/{notebookId}")
+ @Path("job/{noteId}")
@ZeppelinApi
- public Response stopNoteJobs(@PathParam("notebookId") String notebookId)
+ public Response stopNoteJobs(@PathParam("noteId") String noteId)
throws IOException, IllegalArgumentException {
- LOG.info("stop notebook jobs {} ", notebookId);
- Note note = notebook.getNote(notebookId);
+ LOG.info("stop note jobs {} ", noteId);
+ Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
@@ -520,19 +517,19 @@ public Response stopNoteJobs(@PathParam("notebookId") String notebookId)
}
/**
- * Get notebook job status REST API
+ * Get note job status REST API
*
- * @param notebookId ID of Notebook
+ * @param noteId ID of Note
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@GET
- @Path("job/{notebookId}")
+ @Path("job/{noteId}")
@ZeppelinApi
- public Response getNoteJobStatus(@PathParam("notebookId") String notebookId)
+ public Response getNoteJobStatus(@PathParam("noteId") String noteId)
throws IOException, IllegalArgumentException {
- LOG.info("get notebook job status.");
- Note note = notebook.getNote(notebookId);
+ LOG.info("get note job status.");
+ Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
@@ -541,21 +538,21 @@ public Response getNoteJobStatus(@PathParam("notebookId") String notebookId)
}
/**
- * Get notebook paragraph job status REST API
+ * Get note paragraph job status REST API
*
- * @param notebookId ID of Notebook
+ * @param noteId ID of Note
* @param paragraphId ID of Paragraph
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@GET
- @Path("job/{notebookId}/{paragraphId}")
+ @Path("job/{noteId}/{paragraphId}")
@ZeppelinApi
- public Response getNoteParagraphJobStatus(@PathParam("notebookId") String notebookId,
+ public Response getNoteParagraphJobStatus(@PathParam("noteId") String noteId,
@PathParam("paragraphId") String paragraphId)
throws IOException, IllegalArgumentException {
- LOG.info("get notebook paragraph job status.");
- Note note = notebook.getNote(notebookId);
+ LOG.info("get note paragraph job status.");
+ Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
@@ -578,14 +575,14 @@ public Response getNoteParagraphJobStatus(@PathParam("notebookId") String notebo
* @throws IOException, IllegalArgumentException
*/
@POST
- @Path("job/{notebookId}/{paragraphId}")
+ @Path("job/{noteId}/{paragraphId}")
@ZeppelinApi
- public Response runParagraph(@PathParam("notebookId") String notebookId,
+ public Response runParagraph(@PathParam("noteId") String noteId,
@PathParam("paragraphId") String paragraphId, String message)
throws IOException, IllegalArgumentException {
- LOG.info("run paragraph job asynchronously {} {} {}", notebookId, paragraphId, message);
+ LOG.info("run paragraph job asynchronously {} {} {}", noteId, paragraphId, message);
- Note note = notebook.getNote(notebookId);
+ Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
@@ -598,6 +595,11 @@ public Response runParagraph(@PathParam("notebookId") String notebookId,
// handle params if presented
handleParagraphParams(message, note, paragraph);
+ AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
+
+ paragraph.setAuthenticationInfo(subject);
+ note.persist(subject);
+
note.run(paragraph.getId());
return new JsonResponse<>(Status.OK).build();
}
@@ -614,9 +616,9 @@ public Response runParagraph(@PathParam("notebookId") String notebookId,
* @throws IOException, IllegalArgumentException
*/
@POST
- @Path("run/{notebookId}/{paragraphId}")
+ @Path("run/{noteId}/{paragraphId}")
@ZeppelinApi
- public Response runParagraphSynchronously(@PathParam("notebookId") String noteId,
+ public Response runParagraphSynchronously(@PathParam("noteId") String noteId,
@PathParam("paragraphId") String paragraphId,
String message) throws
IOException, IllegalArgumentException {
@@ -653,22 +655,18 @@ public Response runParagraphSynchronously(@PathParam("notebookId") String noteId
/**
* Stop(delete) paragraph job REST API
*
- * @param notebookId ID of Notebook
+ * @param noteId ID of Note
* @param paragraphId ID of Paragraph
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@DELETE
- @Path("job/{notebookId}/{paragraphId}")
+ @Path("job/{noteId}/{paragraphId}")
@ZeppelinApi
- public Response stopParagraph(@PathParam("notebookId") String notebookId,
+ public Response stopParagraph(@PathParam("noteId") String noteId,
@PathParam("paragraphId") String paragraphId) throws IOException, IllegalArgumentException {
- /**
- * TODO(jl): Fixed notebookId to noteId
- * https://issues.apache.org/jira/browse/ZEPPELIN-1163
- */
- LOG.info("stop paragraph job {} ", notebookId);
- Note note = notebook.getNote(notebookId);
+ LOG.info("stop paragraph job {} ", noteId);
+ Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
@@ -689,16 +687,15 @@ public Response stopParagraph(@PathParam("notebookId") String notebookId,
* @throws IOException, IllegalArgumentException
*/
@POST
- @Path("cron/{notebookId}")
+ @Path("cron/{noteId}")
@ZeppelinApi
- public Response registerCronJob(@PathParam("notebookId") String notebookId, String message)
+ public Response registerCronJob(@PathParam("noteId") String noteId, String message)
throws IOException, IllegalArgumentException {
- // TODO(jl): Fixed notebookId to noteId
- LOG.info("Register cron job note={} request cron msg={}", notebookId, message);
+ LOG.info("Register cron job note={} request cron msg={}", noteId, message);
CronRequest request = gson.fromJson(message, CronRequest.class);
- Note note = notebook.getNote(notebookId);
+ Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
@@ -718,19 +715,18 @@ public Response registerCronJob(@PathParam("notebookId") String notebookId, Stri
/**
* Remove cron job REST API
*
- * @param notebookId ID of Notebook
+ * @param noteId ID of Note
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@DELETE
- @Path("cron/{notebookId}")
+ @Path("cron/{noteId}")
@ZeppelinApi
- public Response removeCronJob(@PathParam("notebookId") String notebookId)
+ public Response removeCronJob(@PathParam("noteId") String noteId)
throws IOException, IllegalArgumentException {
- // TODO(jl): Fixed notebookId to noteId
- LOG.info("Remove cron job note {}", notebookId);
+ LOG.info("Remove cron job note {}", noteId);
- Note note = notebook.getNote(notebookId);
+ Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
@@ -746,19 +742,18 @@ public Response removeCronJob(@PathParam("notebookId") String notebookId)
/**
* Get cron job REST API
*
- * @param notebookId ID of Notebook
+ * @param noteId ID of Note
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@GET
- @Path("cron/{notebookId}")
+ @Path("cron/{noteId}")
@ZeppelinApi
- public Response getCronJob(@PathParam("notebookId") String notebookId)
+ public Response getCronJob(@PathParam("noteId") String noteId)
throws IOException, IllegalArgumentException {
- // TODO(jl): Fixed notebookId to noteId
- LOG.info("Get cron job note {}", notebookId);
+ LOG.info("Get cron job note {}", noteId);
- Note note = notebook.getNote(notebookId);
+ Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
@@ -767,7 +762,7 @@ public Response getCronJob(@PathParam("notebookId") String notebookId)
}
/**
- * Get notebook jobs for job manager
+ * Get note jobs for job manager
*
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
@@ -775,22 +770,22 @@ public Response getCronJob(@PathParam("notebookId") String notebookId)
@GET
@Path("jobmanager/")
@ZeppelinApi
- public Response getJobListforNotebook() throws IOException, IllegalArgumentException {
- LOG.info("Get notebook jobs for job manager");
+ public Response getJobListforNote() throws IOException, IllegalArgumentException {
+ LOG.info("Get note jobs for job manager");
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
- List> notebookJobs = notebook
- .getJobListByUnixTime(false, 0, subject);
+ List> noteJobs = notebook
+ .getJobListByUnixTime(false, 0, subject);
Map response = new HashMap<>();
response.put("lastResponseUnixTime", System.currentTimeMillis());
- response.put("jobs", notebookJobs);
+ response.put("jobs", noteJobs);
return new JsonResponse<>(Status.OK, response).build();
}
/**
- * Get updated notebook jobs for job manager
+ * Get updated note jobs for job manager
*
* Return the `Note` change information within the post unix timestamp.
*
@@ -800,18 +795,18 @@ public Response getJobListforNotebook() throws IOException, IllegalArgumentExcep
@GET
@Path("jobmanager/{lastUpdateUnixtime}/")
@ZeppelinApi
- public Response getUpdatedJobListforNotebook(
+ public Response getUpdatedJobListforNote(
@PathParam("lastUpdateUnixtime") long lastUpdateUnixTime)
throws IOException, IllegalArgumentException {
- LOG.info("Get updated notebook jobs lastUpdateTime {}", lastUpdateUnixTime);
+ LOG.info("Get updated note jobs lastUpdateTime {}", lastUpdateUnixTime);
- List> notebookJobs;
+ List> noteJobs;
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
- notebookJobs = notebook.getJobListByUnixTime(false, lastUpdateUnixTime, subject);
+ noteJobs = notebook.getJobListByUnixTime(false, lastUpdateUnixTime, subject);
Map response = new HashMap<>();
response.put("lastResponseUnixTime", System.currentTimeMillis());
- response.put("jobs", notebookJobs);
+ response.put("jobs", noteJobs);
return new JsonResponse<>(Status.OK, response).build();
}
@@ -823,25 +818,25 @@ public Response getUpdatedJobListforNotebook(
@Path("search")
@ZeppelinApi
public Response search(@QueryParam("q") String queryTerm) {
- LOG.info("Searching notebooks for: {}", queryTerm);
+ LOG.info("Searching notes for: {}", queryTerm);
String principal = SecurityUtils.getPrincipal();
HashSet roles = SecurityUtils.getRoles();
HashSet userAndRoles = new HashSet<>();
userAndRoles.add(principal);
userAndRoles.addAll(roles);
- List> notebooksFound = notebookIndex.query(queryTerm);
- for (int i = 0; i < notebooksFound.size(); i++) {
- String[] Id = notebooksFound.get(i).get("id").split("/", 2);
+ List> notesFound = noteSearchService.query(queryTerm);
+ for (int i = 0; i < notesFound.size(); i++) {
+ String[] Id = notesFound.get(i).get("id").split("/", 2);
String noteId = Id[0];
if (!notebookAuthorization.isOwner(noteId, userAndRoles) &&
!notebookAuthorization.isReader(noteId, userAndRoles) &&
!notebookAuthorization.isWriter(noteId, userAndRoles)) {
- notebooksFound.remove(i);
+ notesFound.remove(i);
i--;
}
}
- LOG.info("{} notebooks found", notebooksFound.size());
- return new JsonResponse<>(Status.OK, notebooksFound).build();
+ LOG.info("{} notes found", notesFound.size());
+ return new JsonResponse<>(Status.OK, notesFound).build();
}
@@ -850,7 +845,7 @@ private void handleParagraphParams(String message, Note note, Paragraph paragrap
// handle params if presented
if (!StringUtils.isEmpty(message)) {
RunParagraphWithParametersRequest request =
- gson.fromJson(message, RunParagraphWithParametersRequest.class);
+ gson.fromJson(message, RunParagraphWithParametersRequest.class);
Map paramsForUpdating = request.getParams();
if (paramsForUpdating != null) {
paragraph.settings.getParams().putAll(paramsForUpdating);
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewNotebookRequest.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewNoteRequest.java
similarity index 91%
rename from zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewNotebookRequest.java
rename to zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewNoteRequest.java
index 61ae6577c0d..67c1b674d7c 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewNotebookRequest.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewNoteRequest.java
@@ -23,14 +23,14 @@
import org.apache.zeppelin.interpreter.InterpreterOption;
/**
- * NewNotebookRequest rest api request message
+ * NewNoteRequest rest api request message
*
*/
-public class NewNotebookRequest {
+public class NewNoteRequest {
String name;
List paragraphs;
- public NewNotebookRequest (){
+ public NewNoteRequest (){
}
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NotebookRepoSettingsRequest.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NotebookRepoSettingsRequest.java
new file mode 100644
index 00000000000..9884476b411
--- /dev/null
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NotebookRepoSettingsRequest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.rest.message;
+
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+
+/**
+ * Represent payload of a notebook repo settings.
+ */
+public class NotebookRepoSettingsRequest {
+
+ public static final NotebookRepoSettingsRequest EMPTY = new NotebookRepoSettingsRequest();
+
+ public String name;
+ public Map settings;
+
+ public NotebookRepoSettingsRequest() {
+ name = StringUtils.EMPTY;
+ settings = Collections.emptyMap();
+ }
+
+ public boolean isEmpty() {
+ return this == EMPTY;
+ }
+
+ public static boolean isEmpty(NotebookRepoSettingsRequest repoSetting) {
+ if (repoSetting == null) {
+ return true;
+ }
+ return repoSetting.isEmpty();
+ }
+}
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
index d352c080176..5e8e3d57e83 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
@@ -22,9 +22,7 @@
import java.util.EnumSet;
import java.util.HashSet;
import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import javax.net.ssl.SSLContext;
import javax.servlet.DispatcherType;
import javax.ws.rs.core.Application;
@@ -37,9 +35,16 @@
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.notebook.Notebook;
import org.apache.zeppelin.notebook.NotebookAuthorization;
-import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.notebook.repo.NotebookRepoSync;
-import org.apache.zeppelin.rest.*;
+import org.apache.zeppelin.rest.ConfigurationsRestApi;
+import org.apache.zeppelin.rest.CredentialRestApi;
+import org.apache.zeppelin.rest.HeliumRestApi;
+import org.apache.zeppelin.rest.InterpreterRestApi;
+import org.apache.zeppelin.rest.LoginRestApi;
+import org.apache.zeppelin.rest.NotebookRepoRestApi;
+import org.apache.zeppelin.rest.NotebookRestApi;
+import org.apache.zeppelin.rest.SecurityRestApi;
+import org.apache.zeppelin.rest.ZeppelinRestApi;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.apache.zeppelin.search.LuceneSearch;
import org.apache.zeppelin.search.SearchService;
@@ -47,7 +52,12 @@
import org.apache.zeppelin.user.Credentials;
import org.apache.zeppelin.utils.SecurityUtils;
import org.eclipse.jetty.http.HttpVersion;
-import org.eclipse.jetty.server.*;
+import org.eclipse.jetty.server.HttpConfiguration;
+import org.eclipse.jetty.server.HttpConnectionFactory;
+import org.eclipse.jetty.server.SecureRequestCustomizer;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.SslConnectionFactory;
import org.eclipse.jetty.server.handler.ContextHandlerCollection;
import org.eclipse.jetty.server.session.SessionHandler;
import org.eclipse.jetty.servlet.DefaultServlet;
@@ -73,8 +83,8 @@ public class ZeppelinServer extends Application {
private SchedulerFactory schedulerFactory;
private InterpreterFactory replFactory;
- private NotebookRepo notebookRepo;
- private SearchService notebookIndex;
+ private SearchService noteSearchService;
+ private NotebookRepoSync notebookRepo;
private NotebookAuthorization notebookAuthorization;
private Credentials credentials;
private DependencyResolver depResolver;
@@ -89,14 +99,14 @@ public ZeppelinServer() throws Exception {
this.heliumApplicationFactory = new HeliumApplicationFactory();
this.schedulerFactory = new SchedulerFactory();
this.replFactory = new InterpreterFactory(conf, notebookWsServer,
- notebookWsServer, heliumApplicationFactory, depResolver);
+ notebookWsServer, heliumApplicationFactory, depResolver, SecurityUtils.isAuthenticated());
this.notebookRepo = new NotebookRepoSync(conf);
- this.notebookIndex = new LuceneSearch();
- this.notebookAuthorization = new NotebookAuthorization(conf);
+ this.noteSearchService = new LuceneSearch();
+ this.notebookAuthorization = NotebookAuthorization.init(conf);
this.credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath());
notebook = new Notebook(conf,
notebookRepo, schedulerFactory, replFactory, notebookWsServer,
- notebookIndex, notebookAuthorization, credentials);
+ noteSearchService, notebookAuthorization, credentials);
// to update notebook from application event from remote process.
heliumApplicationFactory.setNotebook(notebook);
@@ -174,29 +184,24 @@ private static Server setupJettyServer(ZeppelinConfiguration conf) {
ServerConnector connector;
if (conf.useSsl()) {
-
HttpConfiguration httpConfig = new HttpConfiguration();
httpConfig.setSecureScheme("https");
- httpConfig.setSecurePort(conf.getServerPort());
+ httpConfig.setSecurePort(conf.getServerSslPort());
httpConfig.setOutputBufferSize(32768);
HttpConfiguration httpsConfig = new HttpConfiguration(httpConfig);
SecureRequestCustomizer src = new SecureRequestCustomizer();
// Only with Jetty 9.3.x
-// src.setStsMaxAge(2000);
-// src.setStsIncludeSubDomains(true);
+ // src.setStsMaxAge(2000);
+ // src.setStsIncludeSubDomains(true);
httpsConfig.addCustomizer(src);
connector = new ServerConnector(
server,
new SslConnectionFactory(getSslContextFactory(conf), HttpVersion.HTTP_1_1.asString()),
new HttpConnectionFactory(httpsConfig));
-
-
} else {
-
connector = new ServerConnector(server);
-
}
// Set some timeout options to make debugging easier.
@@ -309,9 +314,13 @@ public Set getSingletons() {
ZeppelinRestApi root = new ZeppelinRestApi();
singletons.add(root);
- NotebookRestApi notebookApi = new NotebookRestApi(notebook, notebookWsServer, notebookIndex);
+ NotebookRestApi notebookApi
+ = new NotebookRestApi(notebook, notebookWsServer, noteSearchService);
singletons.add(notebookApi);
+ NotebookRepoRestApi notebookRepoApi = new NotebookRepoRestApi(notebookRepo, notebookWsServer);
+ singletons.add(notebookRepoApi);
+
HeliumRestApi heliumApi = new HeliumRestApi(helium, heliumApplicationFactory, notebook);
singletons.add(heliumApi);
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
index e3328024818..a3dff401429 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
@@ -20,8 +20,8 @@
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
-
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.vfs2.FileSystemException;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.display.AngularObject;
@@ -59,6 +59,7 @@
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
/**
@@ -85,6 +86,8 @@ String getKey() {
Gson gson = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create();
final Map> noteSocketMap = new HashMap<>();
final Queue connectedSockets = new ConcurrentLinkedQueue<>();
+ final Map> userConnectedSockets =
+ new ConcurrentHashMap>();
private Notebook notebook() {
return ZeppelinServer.notebook;
@@ -130,7 +133,7 @@ public void onMessage(NotebookSocket conn, String msg) {
if (LOG.isTraceEnabled()) {
LOG.trace("RECEIVE MSG = " + messagereceived);
}
-
+
String ticket = TicketContainer.instance.getTicket(messagereceived.principal);
if (ticket != null && !ticket.equals(messagereceived.ticket)){
/* not to pollute logs, log instead of exception */
@@ -160,15 +163,18 @@ public void onMessage(NotebookSocket conn, String msg) {
userAndRoles.addAll(roles);
}
}
+ if (StringUtils.isEmpty(conn.getUser())) {
+ addUserConnection(messagereceived.principal, conn);
+ }
AuthenticationInfo subject = new AuthenticationInfo(messagereceived.principal);
/** Lets be elegant here */
switch (messagereceived.op) {
case LIST_NOTES:
- unicastNoteList(conn, subject);
+ unicastNoteList(conn, subject, userAndRoles);
break;
case RELOAD_NOTES_FROM_REPO:
- broadcastReloadedNoteList(subject);
+ broadcastReloadedNoteList(subject, userAndRoles);
break;
case GET_HOME_NOTE:
sendHomeNote(conn, userAndRoles, notebook, messagereceived);
@@ -229,8 +235,8 @@ public void onMessage(NotebookSocket conn, String msg) {
case LIST_CONFIGURATIONS:
sendAllConfigurations(conn, userAndRoles, notebook);
break;
- case CHECKPOINT_NOTEBOOK:
- checkpointNotebook(conn, notebook, messagereceived);
+ case CHECKPOINT_NOTE:
+ checkpointNote(conn, notebook, messagereceived);
break;
case LIST_REVISION_HISTORY:
listRevisionHistory(conn, notebook, messagereceived);
@@ -238,11 +244,11 @@ public void onMessage(NotebookSocket conn, String msg) {
case NOTE_REVISION:
getNoteByRevision(conn, notebook, messagereceived);
break;
- case LIST_NOTEBOOK_JOBS:
- unicastNotebookJobInfo(conn, messagereceived);
+ case LIST_NOTE_JOBS:
+ unicastNoteJobInfo(conn, messagereceived);
break;
- case UNSUBSCRIBE_UPDATE_NOTEBOOK_JOBS:
- unsubscribeNotebookJobInfo(conn);
+ case UNSUBSCRIBE_UPDATE_NOTE_JOBS:
+ unsubscribeNoteJobInfo(conn);
break;
case GET_INTERPRETER_BINDINGS:
getInterpreterBindings(conn, messagereceived);
@@ -250,6 +256,9 @@ public void onMessage(NotebookSocket conn, String msg) {
case SAVE_INTERPRETER_BINDINGS:
saveInterpreterBindings(conn, messagereceived);
break;
+ case EDITOR_SETTING:
+ getEditorSetting(conn, messagereceived);
+ break;
default:
break;
}
@@ -264,6 +273,26 @@ public void onClose(NotebookSocket conn, int code, String reason) {
.getRemoteAddr(), conn.getRequest().getRemotePort(), code, reason);
removeConnectionFromAllNote(conn);
connectedSockets.remove(conn);
+ removeUserConnection(conn.getUser(), conn);
+ }
+
+ private void removeUserConnection(String user, NotebookSocket conn) {
+ if (userConnectedSockets.containsKey(user)) {
+ userConnectedSockets.get(user).remove(conn);
+ } else {
+ LOG.warn("Closing connection that is absent in user connections");
+ }
+ }
+
+ private void addUserConnection(String user, NotebookSocket conn) {
+ conn.setUser(user);
+ if (userConnectedSockets.containsKey(user)) {
+ userConnectedSockets.get(user).add(conn);
+ } else {
+ Queue socketQueue = new ConcurrentLinkedQueue<>();
+ socketQueue.add(conn);
+ userConnectedSockets.put(user, socketQueue);
+ }
}
protected Message deserializeMessage(String msg) {
@@ -379,8 +408,12 @@ private void broadcastExcept(String noteId, Message m, NotebookSocket exclude) {
}
}
- private void broadcastAll(Message m) {
- for (NotebookSocket conn : connectedSockets) {
+ private void multicastToUser(String user, Message m) {
+ if (!userConnectedSockets.containsKey(user)) {
+ LOG.warn("Broadcasting to user that is not in connections map");
+ return;
+ }
+ for (NotebookSocket conn: userConnectedSockets.get(user)) {
try {
conn.send(serializeMessage(m));
} catch (IOException e) {
@@ -397,48 +430,49 @@ private void unicast(Message m, NotebookSocket conn) {
}
}
- public void unicastNotebookJobInfo(NotebookSocket conn, Message fromMessage) throws IOException {
+ public void unicastNoteJobInfo(NotebookSocket conn, Message fromMessage) throws IOException {
addConnectionToNote(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(), conn);
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
- List> notebookJobs = notebook()
+ List> noteJobs = notebook()
.getJobListByUnixTime(false, 0, subject);
Map response = new HashMap<>();
response.put("lastResponseUnixTime", System.currentTimeMillis());
- response.put("jobs", notebookJobs);
+ response.put("jobs", noteJobs);
- conn.send(serializeMessage(new Message(OP.LIST_NOTEBOOK_JOBS)
- .put("notebookJobs", response)));
+ conn.send(serializeMessage(new Message(OP.LIST_NOTE_JOBS)
+ .put("noteJobs", response)));
}
- public void broadcastUpdateNotebookJobInfo(long lastUpdateUnixTime) throws IOException {
- List> notebookJobs = new LinkedList<>();
+ public void broadcastUpdateNoteJobInfo(long lastUpdateUnixTime) throws IOException {
+ List> noteJobs = new LinkedList<>();
Notebook notebookObject = notebook();
List> jobNotes = null;
if (notebookObject != null) {
jobNotes = notebook().getJobListByUnixTime(false, lastUpdateUnixTime, null);
- notebookJobs = jobNotes == null ? notebookJobs : jobNotes;
+ noteJobs = jobNotes == null ? noteJobs : jobNotes;
}
Map response = new HashMap<>();
response.put("lastResponseUnixTime", System.currentTimeMillis());
- response.put("jobs", notebookJobs != null ? notebookJobs : new LinkedList<>());
+ response.put("jobs", noteJobs != null ? noteJobs : new LinkedList<>());
broadcast(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(),
- new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS).put("notebookRunningJobs", response));
+ new Message(OP.LIST_UPDATE_NOTE_JOBS).put("noteRunningJobs", response));
}
- public void unsubscribeNotebookJobInfo(NotebookSocket conn) {
+ public void unsubscribeNoteJobInfo(NotebookSocket conn) {
removeConnectionFromNote(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(), conn);
}
public void saveInterpreterBindings(NotebookSocket conn, Message fromMessage) {
- String noteId = (String) fromMessage.data.get("noteID");
+ String noteId = (String) fromMessage.data.get("noteId");
try {
List settingIdList = gson.fromJson(String.valueOf(
fromMessage.data.get("selectedSettingIds")), new TypeToken>() {
}.getType());
- notebook().bindInterpretersToNote(noteId, settingIdList);
+ AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
+ notebook().bindInterpretersToNote(subject.getUser(), noteId, settingIdList);
broadcastInterpreterBindings(noteId,
InterpreterBindingUtils.getInterpreterBindings(notebook(), noteId));
} catch (Exception e) {
@@ -448,20 +482,20 @@ public void saveInterpreterBindings(NotebookSocket conn, Message fromMessage) {
public void getInterpreterBindings(NotebookSocket conn, Message fromMessage)
throws IOException {
- String noteID = (String) fromMessage.data.get("noteID");
+ String noteId = (String) fromMessage.data.get("noteId");
List settingList =
- InterpreterBindingUtils.getInterpreterBindings(notebook(), noteID);
+ InterpreterBindingUtils.getInterpreterBindings(notebook(), noteId);
conn.send(serializeMessage(new Message(OP.INTERPRETER_BINDINGS)
.put("interpreterBindings", settingList)));
}
- public List> generateNotebooksInfo(boolean needsReload,
- AuthenticationInfo subject) {
+ public List> generateNotesInfo(boolean needsReload,
+ AuthenticationInfo subject, HashSet userAndRoles) {
Notebook notebook = notebook();
ZeppelinConfiguration conf = notebook.getConf();
- String homescreenNotebookId = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN);
+ String homescreenNoteId = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN);
boolean hideHomeScreenNotebookFromList = conf
.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE);
@@ -473,12 +507,12 @@ public List> generateNotebooksInfo(boolean needsReload,
}
}
- List notes = notebook.getAllNotes();
+ List notes = notebook.getAllNotes(userAndRoles);
List> notesInfo = new LinkedList<>();
for (Note note : notes) {
Map info = new HashMap<>();
- if (hideHomeScreenNotebookFromList && note.getId().equals(homescreenNotebookId)) {
+ if (hideHomeScreenNotebookFromList && note.getId().equals(homescreenNoteId)) {
continue;
}
@@ -500,19 +534,45 @@ public void broadcastInterpreterBindings(String noteId,
.put("interpreterBindings", settingList));
}
- public void broadcastNoteList(AuthenticationInfo subject) {
- List> notesInfo = generateNotebooksInfo(false, subject);
- broadcastAll(new Message(OP.NOTES_INFO).put("notes", notesInfo));
+ public void broadcastNoteList(AuthenticationInfo subject, HashSet userAndRoles) {
+ if (subject == null) {
+ subject = new AuthenticationInfo(StringUtils.EMPTY);
+ }
+ //send first to requesting user
+ List> notesInfo = generateNotesInfo(false, subject, userAndRoles);
+ multicastToUser(subject.getUser(), new Message(OP.NOTES_INFO).put("notes", notesInfo));
+ //to others afterwards
+ for (String user: userConnectedSockets.keySet()) {
+ if (subject.getUser() == user) {
+ continue;
+ }
+ notesInfo = generateNotesInfo(false, new AuthenticationInfo(user), userAndRoles);
+ multicastToUser(user, new Message(OP.NOTES_INFO).put("notes", notesInfo));
+ }
}
- public void unicastNoteList(NotebookSocket conn, AuthenticationInfo subject) {
- List> notesInfo = generateNotebooksInfo(false, subject);
+ public void unicastNoteList(NotebookSocket conn, AuthenticationInfo subject,
+ HashSet userAndRoles) {
+ List> notesInfo = generateNotesInfo(false, subject, userAndRoles);
unicast(new Message(OP.NOTES_INFO).put("notes", notesInfo), conn);
}
- public void broadcastReloadedNoteList(AuthenticationInfo subject) {
- List> notesInfo = generateNotebooksInfo(true, subject);
- broadcastAll(new Message(OP.NOTES_INFO).put("notes", notesInfo));
+ public void broadcastReloadedNoteList(AuthenticationInfo subject, HashSet userAndRoles) {
+ if (subject == null) {
+ subject = new AuthenticationInfo(StringUtils.EMPTY);
+ }
+ //reload and reply first to requesting user
+ List> notesInfo = generateNotesInfo(true, subject, userAndRoles);
+ multicastToUser(subject.getUser(), new Message(OP.NOTES_INFO).put("notes", notesInfo));
+ //to others afterwards
+ for (String user: userConnectedSockets.keySet()) {
+ if (subject.getUser() == user) {
+ continue;
+ }
+ //reloaded already above; parameter - false
+ notesInfo = generateNotesInfo(false, new AuthenticationInfo(user), userAndRoles);
+ multicastToUser(user, new Message(OP.NOTES_INFO).put("notes", notesInfo));
+ }
}
void permissionError(NotebookSocket conn, String op,
@@ -541,6 +601,8 @@ private void sendNote(NotebookSocket conn, HashSet userAndRoles, Noteboo
return;
}
+ String user = fromMessage.principal;
+
Note note = notebook.getNote(noteId);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
if (note != null) {
@@ -551,7 +613,7 @@ private void sendNote(NotebookSocket conn, HashSet userAndRoles, Noteboo
}
addConnectionToNote(note.getId(), conn);
conn.send(serializeMessage(new Message(OP.NOTE).put("note", note)));
- sendAllAngularObjects(note, conn);
+ sendAllAngularObjects(note, user, conn);
} else {
conn.send(serializeMessage(new Message(OP.NOTE).put("note", null)));
}
@@ -560,6 +622,7 @@ private void sendNote(NotebookSocket conn, HashSet userAndRoles, Noteboo
private void sendHomeNote(NotebookSocket conn, HashSet userAndRoles,
Notebook notebook, Message fromMessage) throws IOException {
String noteId = notebook.getConf().getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN);
+ String user = fromMessage.principal;
Note note = null;
if (noteId != null) {
@@ -575,7 +638,7 @@ private void sendHomeNote(NotebookSocket conn, HashSet userAndRoles,
}
addConnectionToNote(note.getId(), conn);
conn.send(serializeMessage(new Message(OP.NOTE).put("note", note)));
- sendAllAngularObjects(note, conn);
+ sendAllAngularObjects(note, user, conn);
} else {
removeConnectionFromAllNote(conn);
conn.send(serializeMessage(new Message(OP.NOTE).put("note", null)));
@@ -615,7 +678,7 @@ private void updateNote(NotebookSocket conn, HashSet userAndRoles,
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
note.persist(subject);
broadcastNote(note);
- broadcastNoteList(subject);
+ broadcastNoteList(subject, userAndRoles);
}
}
@@ -650,7 +713,7 @@ private void createNote(NotebookSocket conn, HashSet userAndRoles,
note.persist(subject);
addConnectionToNote(note.getId(), (NotebookSocket) conn);
conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", note)));
- broadcastNoteList(subject);
+ broadcastNoteList(subject, userAndRoles);
}
private void removeNote(NotebookSocket conn, HashSet userAndRoles,
@@ -672,7 +735,7 @@ private void removeNote(NotebookSocket conn, HashSet userAndRoles,
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
notebook.removeNote(noteId, subject);
removeNote(noteId);
- broadcastNoteList(subject);
+ broadcastNoteList(subject, userAndRoles);
}
private void updateParagraph(NotebookSocket conn, HashSet userAndRoles,
@@ -714,7 +777,7 @@ private void cloneNote(NotebookSocket conn, HashSet userAndRoles,
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
addConnectionToNote(newNote.getId(), (NotebookSocket) conn);
conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", newNote)));
- broadcastNoteList(subject);
+ broadcastNoteList(subject, userAndRoles);
}
protected Note importNote(NotebookSocket conn, HashSet userAndRoles,
@@ -722,13 +785,18 @@ protected Note importNote(NotebookSocket conn, HashSet userAndRoles,
throws IOException {
Note note = null;
if (fromMessage != null) {
- String noteName = (String) ((Map) fromMessage.get("notebook")).get("name");
- String noteJson = gson.toJson(fromMessage.get("notebook"));
- AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
+ String noteName = (String) ((Map) fromMessage.get("note")).get("name");
+ String noteJson = gson.toJson(fromMessage.get("note"));
+ AuthenticationInfo subject = null;
+ if (fromMessage.principal != null) {
+ subject = new AuthenticationInfo(fromMessage.principal);
+ } else {
+ subject = new AuthenticationInfo("anonymous");
+ }
note = notebook.importNote(noteJson, noteName, subject);
note.persist(subject);
broadcastNote(note);
- broadcastNoteList(subject);
+ broadcastNoteList(subject, userAndRoles);
}
return note;
}
@@ -742,7 +810,7 @@ private void removeParagraph(NotebookSocket conn, HashSet userAndRoles,
String noteId = getOpenNoteId(conn);
final Note note = notebook.getNote(noteId);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
- AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
+ AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
if (!notebookAuthorization.isWriter(noteId, userAndRoles)) {
permissionError(conn, "write", fromMessage.principal,
userAndRoles, notebookAuthorization.getWriters(noteId));
@@ -751,7 +819,7 @@ private void removeParagraph(NotebookSocket conn, HashSet userAndRoles,
/** We dont want to remove the last paragraph */
if (!note.isLastParagraph(paragraphId)) {
- note.removeParagraph(paragraphId);
+ note.removeParagraph(subject.getUser(), paragraphId);
note.persist(subject);
broadcastNote(note);
}
@@ -807,6 +875,7 @@ private void angularObjectUpdated(NotebookSocket conn, HashSet userAndRo
String interpreterGroupId = (String) fromMessage.get("interpreterGroupId");
String varName = (String) fromMessage.get("name");
Object varValue = fromMessage.get("value");
+ String user = fromMessage.principal;
AngularObject ao = null;
boolean global = false;
// propagate change to (Remote) AngularObjectRegistry
@@ -815,12 +884,12 @@ private void angularObjectUpdated(NotebookSocket conn, HashSet userAndRo
List settings = notebook.getInterpreterFactory()
.getInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
- if (setting.getInterpreterGroup(note.getId()) == null) {
+ if (setting.getInterpreterGroup(user, note.getId()) == null) {
continue;
}
- if (interpreterGroupId.equals(setting.getInterpreterGroup(note.getId()).getId())) {
+ if (interpreterGroupId.equals(setting.getInterpreterGroup(user, note.getId()).getId())) {
AngularObjectRegistry angularObjectRegistry = setting
- .getInterpreterGroup(note.getId()).getAngularObjectRegistry();
+ .getInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
// first trying to get local registry
ao = angularObjectRegistry.get(varName, noteId, paragraphId);
@@ -857,12 +926,12 @@ private void angularObjectUpdated(NotebookSocket conn, HashSet userAndRo
List settings = notebook.getInterpreterFactory()
.getInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
- if (setting.getInterpreterGroup(n.getId()) == null) {
+ if (setting.getInterpreterGroup(user, n.getId()) == null) {
continue;
}
- if (interpreterGroupId.equals(setting.getInterpreterGroup(n.getId()).getId())) {
+ if (interpreterGroupId.equals(setting.getInterpreterGroup(user, n.getId()).getId())) {
AngularObjectRegistry angularObjectRegistry = setting
- .getInterpreterGroup(n.getId()).getAngularObjectRegistry();
+ .getInterpreterGroup(user, n.getId()).getAngularObjectRegistry();
this.broadcastExcept(
n.getId(),
new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao)
@@ -1027,12 +1096,12 @@ private void removeAngularObjectFromLocalRepo(String noteId, String paragraphId,
final AngularObject removed = registry.remove(varName, noteId, paragraphId);
if (removed != null) {
this.broadcastExcept(
- noteId,
- new Message(OP.ANGULAR_OBJECT_REMOVE).put("angularObject", removed)
- .put("interpreterGroupId", interpreterGroupId)
- .put("noteId", noteId)
- .put("paragraphId", paragraphId),
- conn);
+ noteId,
+ new Message(OP.ANGULAR_OBJECT_REMOVE).put("angularObject", removed)
+ .put("interpreterGroupId", interpreterGroupId)
+ .put("noteId", noteId)
+ .put("paragraphId", paragraphId),
+ conn);
}
}
@@ -1048,7 +1117,7 @@ private void moveParagraph(NotebookSocket conn, HashSet userAndRoles, No
String noteId = getOpenNoteId(conn);
final Note note = notebook.getNote(noteId);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
- AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
+ AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
if (!notebookAuthorization.isWriter(noteId, userAndRoles)) {
permissionError(conn, "write", fromMessage.principal,
userAndRoles, notebookAuthorization.getWriters(noteId));
@@ -1063,11 +1132,11 @@ private void moveParagraph(NotebookSocket conn, HashSet userAndRoles, No
private void insertParagraph(NotebookSocket conn, HashSet userAndRoles,
Notebook notebook, Message fromMessage) throws IOException {
final int index = (int) Double.parseDouble(fromMessage.get("index")
- .toString());
+ .toString());
String noteId = getOpenNoteId(conn);
final Note note = notebook.getNote(noteId);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
- AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
+ AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
if (!notebookAuthorization.isWriter(noteId, userAndRoles)) {
permissionError(conn, "write", fromMessage.principal,
userAndRoles, notebookAuthorization.getWriters(noteId));
@@ -1119,14 +1188,9 @@ private void runParagraph(NotebookSocket conn, HashSet userAndRoles, Not
String text = (String) fromMessage.get("paragraph");
p.setText(text);
p.setTitle((String) fromMessage.get("title"));
- if (!fromMessage.principal.equals("anonymous")) {
- AuthenticationInfo authenticationInfo = new AuthenticationInfo(fromMessage.principal,
- fromMessage.ticket);
- p.setAuthenticationInfo(authenticationInfo);
-
- } else {
- p.setAuthenticationInfo(new AuthenticationInfo());
- }
+ AuthenticationInfo authenticationInfo =
+ new AuthenticationInfo(fromMessage.principal, fromMessage.ticket);
+ p.setAuthenticationInfo(authenticationInfo);
Map params = (Map) fromMessage
.get("params");
@@ -1142,7 +1206,18 @@ private void runParagraph(NotebookSocket conn, HashSet userAndRoles, Not
}
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
- note.persist(subject);
+
+ try {
+ note.persist(subject);
+ } catch (FileSystemException ex) {
+ LOG.error("Exception from run", ex);
+ conn.send(serializeMessage(new Message(OP.ERROR_INFO).put("info",
+ "Oops! There is something wrong with the notebook file system. "
+ + "Please check the logs for more details.")));
+ // don't run the paragraph when there is error on persisting the note information
+ return;
+ }
+
try {
note.run(paragraphId);
} catch (Exception ex) {
@@ -1177,7 +1252,7 @@ public boolean apply(String key) {
.put("configurations", configurations)));
}
- private void checkpointNotebook(NotebookSocket conn, Notebook notebook,
+ private void checkpointNote(NotebookSocket conn, Notebook notebook,
Message fromMessage) throws IOException {
String noteId = (String) fromMessage.get("noteId");
String commitMessage = (String) fromMessage.get("commitMessage");
@@ -1309,7 +1384,7 @@ public NotebookInformationListener(NotebookServer notebookServer) {
@Override
public void onParagraphRemove(Paragraph p) {
try {
- notebookServer.broadcastUpdateNotebookJobInfo(System.currentTimeMillis() - 5000);
+ notebookServer.broadcastUpdateNoteJobInfo(System.currentTimeMillis() - 5000);
} catch (IOException ioe) {
LOG.error("can not broadcast for job manager {}", ioe.getMessage());
}
@@ -1318,14 +1393,14 @@ public void onParagraphRemove(Paragraph p) {
@Override
public void onNoteRemove(Note note) {
try {
- notebookServer.broadcastUpdateNotebookJobInfo(System.currentTimeMillis() - 5000);
+ notebookServer.broadcastUpdateNoteJobInfo(System.currentTimeMillis() - 5000);
} catch (IOException ioe) {
LOG.error("can not broadcast for job manager {}", ioe.getMessage());
}
List> notesInfo = new LinkedList<>();
Map info = new HashMap<>();
- info.put("notebookId", note.getId());
+ info.put("noteId", note.getId());
// set paragraphs
List> paragraphsInfo = new LinkedList<>();
@@ -1341,7 +1416,7 @@ public void onNoteRemove(Note note) {
response.put("jobs", notesInfo);
notebookServer.broadcast(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(),
- new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS).put("notebookRunningJobs", response));
+ new Message(OP.LIST_UPDATE_NOTE_JOBS).put("noteRunningJobs", response));
}
@@ -1349,35 +1424,35 @@ public void onNoteRemove(Note note) {
public void onParagraphCreate(Paragraph p) {
Notebook notebook = notebookServer.notebook();
List> notebookJobs = notebook.getJobListByParagraphId(
- p.getId()
+ p.getId()
);
Map response = new HashMap<>();
response.put("lastResponseUnixTime", System.currentTimeMillis());
response.put("jobs", notebookJobs);
notebookServer.broadcast(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(),
- new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS).put("notebookRunningJobs", response));
+ new Message(OP.LIST_UPDATE_NOTE_JOBS).put("noteRunningJobs", response));
}
@Override
public void onNoteCreate(Note note) {
Notebook notebook = notebookServer.notebook();
- List> notebookJobs = notebook.getJobListBymNotebookId(
- note.getId()
+ List> notebookJobs = notebook.getJobListByNoteId(
+ note.getId()
);
Map response = new HashMap<>();
response.put("lastResponseUnixTime", System.currentTimeMillis());
response.put("jobs", notebookJobs);
notebookServer.broadcast(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(),
- new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS).put("notebookRunningJobs", response));
+ new Message(OP.LIST_UPDATE_NOTE_JOBS).put("noteRunningJobs", response));
}
@Override
public void onParagraphStatusChange(Paragraph p, Status status) {
Notebook notebook = notebookServer.notebook();
List> notebookJobs = notebook.getJobListByParagraphId(
- p.getId()
+ p.getId()
);
Map response = new HashMap<>();
@@ -1385,21 +1460,21 @@ public void onParagraphStatusChange(Paragraph p, Status status) {
response.put("jobs", notebookJobs);
notebookServer.broadcast(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(),
- new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS).put("notebookRunningJobs", response));
+ new Message(OP.LIST_UPDATE_NOTE_JOBS).put("noteRunningJobs", response));
}
@Override
public void onUnbindInterpreter(Note note, InterpreterSetting setting) {
Notebook notebook = notebookServer.notebook();
- List> notebookJobs = notebook.getJobListBymNotebookId(
- note.getId()
+ List> notebookJobs = notebook.getJobListByNoteId(
+ note.getId()
);
Map response = new HashMap<>();
response.put("lastResponseUnixTime", System.currentTimeMillis());
response.put("jobs", notebookJobs);
notebookServer.broadcast(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(),
- new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS).put("notebookRunningJobs", response));
+ new Message(OP.LIST_UPDATE_NOTE_JOBS).put("noteRunningJobs", response));
}
}
@@ -1440,7 +1515,7 @@ public void afterStatusChange(Job job, Status before, Status after) {
LOG.info("Job {} is finished", job.getId());
try {
//TODO(khalid): may change interface for JobListener and pass subject from interpreter
- note.persist(null);
+ note.persist(job instanceof Paragraph ? ((Paragraph) job).getAuthenticationInfo() : null);
} catch (IOException e) {
LOG.error(e.toString(), e);
}
@@ -1448,14 +1523,14 @@ public void afterStatusChange(Job job, Status before, Status after) {
notebookServer.broadcastNote(note);
try {
- notebookServer.broadcastUpdateNotebookJobInfo(System.currentTimeMillis() - 5000);
+ notebookServer.broadcastUpdateNoteJobInfo(System.currentTimeMillis() - 5000);
} catch (IOException e) {
LOG.error("can not broadcast for job manager {}", e);
}
}
/**
- * This callback is for praragraph that runs on RemoteInterpreterProcess
+ * This callback is for paragraph that runs on RemoteInterpreterProcess
* @param paragraph
* @param out
* @param output
@@ -1463,9 +1538,9 @@ public void afterStatusChange(Job job, Status before, Status after) {
@Override
public void onOutputAppend(Paragraph paragraph, InterpreterOutput out, String output) {
Message msg = new Message(OP.PARAGRAPH_APPEND_OUTPUT)
- .put("noteId", paragraph.getNote().getId())
- .put("paragraphId", paragraph.getId())
- .put("data", output);
+ .put("noteId", paragraph.getNote().getId())
+ .put("paragraphId", paragraph.getId())
+ .put("data", output);
notebookServer.broadcast(paragraph.getNote().getId(), msg);
}
@@ -1479,9 +1554,9 @@ public void onOutputAppend(Paragraph paragraph, InterpreterOutput out, String ou
@Override
public void onOutputUpdate(Paragraph paragraph, InterpreterOutput out, String output) {
Message msg = new Message(OP.PARAGRAPH_UPDATE_OUTPUT)
- .put("noteId", paragraph.getNote().getId())
- .put("paragraphId", paragraph.getId())
- .put("data", output);
+ .put("noteId", paragraph.getNote().getId())
+ .put("paragraphId", paragraph.getId())
+ .put("data", output);
notebookServer.broadcast(paragraph.getNote().getId(), msg);
}
@@ -1496,7 +1571,8 @@ public NotebookEventListener getNotebookInformationListener() {
return new NotebookInformationListener(this);
}
- private void sendAllAngularObjects(Note note, NotebookSocket conn) throws IOException {
+ private void sendAllAngularObjects(Note note, String user, NotebookSocket conn)
+ throws IOException {
List settings =
notebook().getInterpreterFactory().getInterpreterSettings(note.getId());
if (settings == null || settings.size() == 0) {
@@ -1504,17 +1580,15 @@ private void sendAllAngularObjects(Note note, NotebookSocket conn) throws IOExce
}
for (InterpreterSetting intpSetting : settings) {
- AngularObjectRegistry registry = intpSetting.getInterpreterGroup(note.getId())
- .getAngularObjectRegistry();
+ AngularObjectRegistry registry =
+ intpSetting.getInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
List objects = registry.getAllWithGlobal(note.getId());
for (AngularObject object : objects) {
- conn.send(serializeMessage(new Message(OP.ANGULAR_OBJECT_UPDATE)
- .put("angularObject", object)
- .put("interpreterGroupId",
- intpSetting.getInterpreterGroup(note.getId()).getId())
- .put("noteId", note.getId())
- .put("paragraphId", object.getParagraphId())
- ));
+ conn.send(serializeMessage(
+ new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", object)
+ .put("interpreterGroupId",
+ intpSetting.getInterpreterGroup(user, note.getId()).getId())
+ .put("noteId", note.getId()).put("paragraphId", object.getParagraphId())));
}
}
}
@@ -1567,11 +1641,26 @@ public void onRemove(String interpreterGroupId, String name, String noteId, Stri
if (id.equals(interpreterGroupId)) {
broadcast(
note.getId(),
- new Message(OP.ANGULAR_OBJECT_REMOVE).put("name", name).put(
- "noteId", noteId).put("paragraphId", paragraphId));
+ new Message(OP.ANGULAR_OBJECT_REMOVE)
+ .put("name", name)
+ .put("noteId", noteId)
+ .put("paragraphId", paragraphId));
}
}
}
}
+
+ private void getEditorSetting(NotebookSocket conn, Message fromMessage)
+ throws IOException {
+ String paragraphId = (String) fromMessage.get("paragraphId");
+ String replName = (String) fromMessage.get("magic");
+ String noteId = getOpenNoteId(conn);
+ String user = fromMessage.principal;
+ Message resp = new Message(OP.EDITOR_SETTING);
+ resp.put("paragraphId", paragraphId);
+ resp.put("editor", notebook().getInterpreterFactory().getEditorSetting(user, noteId, replName));
+ conn.send(serializeMessage(resp));
+ return;
+ }
}
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookSocket.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookSocket.java
index 5d68bf5ec2d..baee746ed0f 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookSocket.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookSocket.java
@@ -20,6 +20,7 @@
import javax.servlet.http.HttpServletRequest;
+import org.apache.commons.lang.StringUtils;
import org.eclipse.jetty.websocket.api.Session;
import org.eclipse.jetty.websocket.api.WebSocketAdapter;
@@ -32,12 +33,14 @@ public class NotebookSocket extends WebSocketAdapter {
private NotebookSocketListener listener;
private HttpServletRequest request;
private String protocol;
+ private String user;
public NotebookSocket(HttpServletRequest req, String protocol,
NotebookSocketListener listener) {
this.listener = listener;
this.request = req;
this.protocol = protocol;
+ this.user = StringUtils.EMPTY;
}
@Override
@@ -65,8 +68,15 @@ public String getProtocol() {
return protocol;
}
- public void send(String serializeMessage) throws IOException {
+ public synchronized void send(String serializeMessage) throws IOException {
connection.getRemote().sendString(serializeMessage);
}
+ public String getUser() {
+ return user;
+ }
+
+ public void setUser(String user) {
+ this.user = user;
+ }
}
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java b/zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java
index f9e5929a882..d81d2e6150b 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java
@@ -119,4 +119,10 @@ public static HashSet getRoles() {
return roles;
}
+ /**
+ * Checked if shiro enabled or not
+ */
+ public static boolean isAuthenticated() {
+ return org.apache.shiro.SecurityUtils.getSubject().isAuthenticated();
+ }
}
diff --git a/zeppelin-server/src/main/resources/log4j.properties b/zeppelin-server/src/main/resources/log4j.properties
new file mode 100644
index 00000000000..2f644074c89
--- /dev/null
+++ b/zeppelin-server/src/main/resources/log4j.properties
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+log4j.rootLogger = INFO, stdout
+
+log4j.appender.stdout = org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n
+
+log4j.additivity.org.apache.zeppelin.interpreter = false
+log4j.logger.org.apache.zeppelin.interpreter = DEBUG, stdout
diff --git a/zeppelin-server/src/main/resources/shiro.ini b/zeppelin-server/src/main/resources/shiro.ini
index 371a44e11e1..050c9d99bb5 100644
--- a/zeppelin-server/src/main/resources/shiro.ini
+++ b/zeppelin-server/src/main/resources/shiro.ini
@@ -18,14 +18,19 @@
[users]
# List of users with their password allowed to access Zeppelin.
# To use a different strategy (LDAP / Database / ...) check the shiro doc at http://shiro.apache.org/configuration.html#Configuration-INISections
-admin = password
+admin = password, admin
+user1 = user1, role1
[urls]
-
# anon means the access is anonymous.
# authcBasic means Basic Auth Security
# To enfore security, comment the line below and uncomment the next one
/** = anon
-#/** = authcBasic
+#/** = authc
+[roles]
+role1 = *
+role2 = *
+role3 = *
+admin = *
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java b/zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java
index c99bf0a2b5e..9ef4af019b8 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java
@@ -122,7 +122,7 @@ protected void deleteTestNotebook(final WebDriver driver) {
driver.findElement(By.xpath(".//*[@id='main']//button[@ng-click='removeNote(note.id)']"))
.sendKeys(Keys.ENTER);
ZeppelinITUtils.sleep(1000, true);
- driver.findElement(By.xpath("//div[@class='modal-dialog'][contains(.,'delete this notebook')]" +
+ driver.findElement(By.xpath("//div[@class='modal-dialog'][contains(.,'delete this note')]" +
"//div[@class='modal-footer']//button[contains(.,'OK')]")).click();
ZeppelinITUtils.sleep(100, true);
}
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/AuthenticationIT.java b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/AuthenticationIT.java
index ea3f3637537..fea67d6e6c3 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/AuthenticationIT.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/AuthenticationIT.java
@@ -36,6 +36,7 @@
import java.io.File;
import java.io.IOException;
+import java.net.URI;
import java.util.List;
@@ -176,13 +177,21 @@ public void testGroupPermission() throws Exception {
authenticationIT.logoutUser("finance1");
authenticationIT.authenticationUser("hr1", "hr1");
- pollingWait(By.xpath("//*[@id='notebook-names']//a[contains(@href, '" + noteId + "')]"),
- MAX_BROWSER_TIMEOUT_SEC).click();
+ try {
+ WebElement element = pollingWait(By.xpath("//*[@id='notebook-names']//a[contains(@href, '" + noteId + "')]"),
+ MAX_BROWSER_TIMEOUT_SEC);
+ collector.checkThat("Check is user has permission to view this note link", false,
+ CoreMatchers.equalTo(element.isDisplayed()));
+ } catch (Exception e) {
+ //This should have failed, nothing to worry.
+ }
+
+ driver.get(new URI(driver.getCurrentUrl()).resolve("/#/notebook/" + noteId).toString());
List privilegesModal = driver.findElements(
By.xpath("//div[@class='modal-content']//div[@class='bootstrap-dialog-header']" +
"//div[contains(.,'Insufficient privileges')]"));
- collector.checkThat("Check is user has permission to view this notebook", 1,
+ collector.checkThat("Check is user has permission to view this note", 1,
CoreMatchers.equalTo(privilegesModal.size()));
driver.findElement(
By.xpath("//div[@class='modal-content'][contains(.,'Insufficient privileges')]" +
@@ -190,13 +199,21 @@ public void testGroupPermission() throws Exception {
authenticationIT.logoutUser("hr1");
authenticationIT.authenticationUser("finance2", "finance2");
- pollingWait(By.xpath("//*[@id='notebook-names']//a[contains(@href, '" + noteId + "')]"),
- MAX_BROWSER_TIMEOUT_SEC).click();
+ try {
+ WebElement element = pollingWait(By.xpath("//*[@id='notebook-names']//a[contains(@href, '" + noteId + "')]"),
+ MAX_BROWSER_TIMEOUT_SEC);
+ collector.checkThat("Check is user has permission to view this note link", true,
+ CoreMatchers.equalTo(element.isDisplayed()));
+ } catch (Exception e) {
+ //This should have failed, nothing to worry.
+ }
+
+ driver.get(new URI(driver.getCurrentUrl()).resolve("/#/notebook/" + noteId).toString());
privilegesModal = driver.findElements(
By.xpath("//div[@class='modal-content']//div[@class='bootstrap-dialog-header']" +
"//div[contains(.,'Insufficient privileges')]"));
- collector.checkThat("Check is user has permission to view this notebook", 0,
+ collector.checkThat("Check is user has permission to view this note", 0,
CoreMatchers.equalTo(privilegesModal.size()));
deleteTestNotebook(driver);
authenticationIT.logoutUser("finance2");
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java
index b7b8ea2f259..1d60fce5e4b 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java
@@ -182,7 +182,7 @@ public void testAngularDisplay() throws Exception {
driver.findElement(By.xpath(".//*[@id='main']//button[@ng-click='removeNote(note.id)']"))
.sendKeys(Keys.ENTER);
ZeppelinITUtils.sleep(1000, true);
- driver.findElement(By.xpath("//div[@class='modal-dialog'][contains(.,'delete this notebook')]" +
+ driver.findElement(By.xpath("//div[@class='modal-dialog'][contains(.,'delete this note')]" +
"//div[@class='modal-footer']//button[contains(.,'OK')]")).click();
ZeppelinITUtils.sleep(100, true);
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
index 580e5a08e94..de2607f17b4 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
@@ -152,9 +152,9 @@ protected static void startUp() throws Exception {
}
// set spark master and other properties
- sparkIntpSetting.getProperties().setProperty("master", "spark://" + getHostname() + ":7071");
+ sparkIntpSetting.getProperties().setProperty("master", "local[2]");
sparkIntpSetting.getProperties().setProperty("spark.cores.max", "2");
-
+ sparkIntpSetting.getProperties().setProperty("zeppelin.spark.useHiveContext", "false");
// set spark home for pyspark
sparkIntpSetting.getProperties().setProperty("spark.home", getSparkHome());
pySpark = true;
@@ -171,10 +171,16 @@ protected static void startUp() throws Exception {
String sparkHome = getSparkHome();
if (sparkHome != null) {
- sparkIntpSetting.getProperties().setProperty("master", "spark://" + getHostname() + ":7071");
+ if (System.getenv("SPARK_MASTER") != null) {
+ sparkIntpSetting.getProperties().setProperty("master", System.getenv("SPARK_MASTER"));
+ } else {
+ sparkIntpSetting.getProperties()
+ .setProperty("master", "local[2]");
+ }
sparkIntpSetting.getProperties().setProperty("spark.cores.max", "2");
// set spark home for pyspark
sparkIntpSetting.getProperties().setProperty("spark.home", sparkHome);
+ sparkIntpSetting.getProperties().setProperty("zeppelin.spark.useHiveContext", "false");
pySpark = true;
sparkR = true;
}
@@ -194,7 +200,11 @@ private static String getHostname() {
}
private static String getSparkHome() {
- String sparkHome = getSparkHomeRecursively(new File(System.getProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName())));
+ String sparkHome = System.getenv("SPARK_HOME");
+ if (sparkHome != null) {
+ return sparkHome;
+ }
+ sparkHome = getSparkHomeRecursively(new File(System.getProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName())));
System.out.println("SPARK HOME detected " + sparkHome);
return sparkHome;
}
@@ -228,13 +238,7 @@ private static String getSparkHomeRecursively(File dir) {
}
private static boolean isActiveSparkHome(File dir) {
- if (dir.getName().matches("spark-[0-9\\.]+[A-Za-z-]*-bin-hadoop[0-9\\.]+")) {
- File pidDir = new File(dir, "run");
- if (pidDir.isDirectory() && pidDir.listFiles().length > 0) {
- return true;
- }
- }
- return false;
+ return dir.getName().matches("spark-[0-9\\.]+[A-Za-z-]*-bin-hadoop[0-9\\.]+");
}
protected static void shutDown() throws Exception {
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
index c767eb05786..6d4fb2c5472 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
@@ -30,7 +30,9 @@
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.server.ZeppelinServer;
+import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.AfterClass;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.FixMethodOrder;
import org.junit.Test;
@@ -47,6 +49,7 @@
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class InterpreterRestApiTest extends AbstractTestRestApi {
Gson gson = new Gson();
+ AuthenticationInfo anonymous;
@BeforeClass
public static void init() throws Exception {
@@ -58,6 +61,11 @@ public static void destroy() throws Exception {
AbstractTestRestApi.shutDown();
}
+ @Before
+ public void setUp() {
+ anonymous = new AuthenticationInfo("anonymous");
+ }
+
@Test
public void getAvailableInterpreters() throws IOException {
// when
@@ -90,7 +98,7 @@ public void testSettingsCRUD() throws IOException {
String jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"propvalue\"}," +
"\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
"\"dependencies\":[]," +
- "\"option\": { \"remote\": true, \"perNoteSession\": false }}";
+ "\"option\": { \"remote\": true, \"session\": false }}";
PostMethod post = httpPost("/interpreter/setting/", jsonRequest);
LOG.info("testSettingCRUD create response\n" + post.getResponseBodyAsString());
assertThat("test create method:", post, isCreated());
@@ -106,7 +114,7 @@ public void testSettingsCRUD() throws IOException {
jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"Otherpropvalue\"}," +
"\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
"\"dependencies\":[]," +
- "\"option\": { \"remote\": true, \"perNoteSession\": false }}";
+ "\"option\": { \"remote\": true, \"session\": false }}";
PutMethod put = httpPut("/interpreter/setting/" + newSettingId, jsonRequest);
LOG.info("testSettingCRUD update response\n" + put.getResponseBodyAsString());
assertThat("test update method:", put, isAllowed());
@@ -131,7 +139,7 @@ public void testSettingsCreateWithEmptyJson() throws IOException {
@Test
public void testInterpreterAutoBinding() throws IOException {
// create note
- Note note = ZeppelinServer.notebook.createNote(null);
+ Note note = ZeppelinServer.notebook.createNote(anonymous);
// check interpreter is binded
GetMethod get = httpGet("/notebook/interpreter/bind/" + note.getId());
@@ -144,13 +152,13 @@ public void testInterpreterAutoBinding() throws IOException {
get.releaseConnection();
//cleanup
- ZeppelinServer.notebook.removeNote(note.getId(), null);
+ ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testInterpreterRestart() throws IOException, InterruptedException {
// create new note
- Note note = ZeppelinServer.notebook.createNote(null);
+ Note note = ZeppelinServer.notebook.createNote(anonymous);
note.addParagraph();
Paragraph p = note.getLastParagraph();
Map config = p.getConfig();
@@ -159,6 +167,7 @@ public void testInterpreterRestart() throws IOException, InterruptedException {
// run markdown paragraph
p.setConfig(config);
p.setText("%md markdown");
+ p.setAuthenticationInfo(anonymous);
note.run(p.getId());
while (p.getStatus() != Status.FINISHED) {
Thread.sleep(100);
@@ -181,13 +190,14 @@ public void testInterpreterRestart() throws IOException, InterruptedException {
p = note.addParagraph();
p.setConfig(config);
p.setText("%md markdown restarted");
+ p.setAuthenticationInfo(anonymous);
note.run(p.getId());
while (p.getStatus() != Status.FINISHED) {
Thread.sleep(100);
}
assertEquals("markdown restarted
\n", p.getResult().message());
//cleanup
- ZeppelinServer.notebook.removeNote(note.getId(), null);
+ ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRepoRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRepoRestApiTest.java
new file mode 100644
index 00000000000..d2217929ed3
--- /dev/null
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRepoRestApiTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.rest;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertThat;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.commons.httpclient.methods.PutMethod;
+import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runners.MethodSorters;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+
+/**
+ * NotebookRepo rest api test.
+ */
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class NotebookRepoRestApiTest extends AbstractTestRestApi {
+
+ Gson gson = new Gson();
+ AuthenticationInfo anonymous;
+
+ @BeforeClass
+ public static void init() throws Exception {
+ AbstractTestRestApi.startUp();
+ }
+
+ @AfterClass
+ public static void destroy() throws Exception {
+ AbstractTestRestApi.shutDown();
+ }
+
+ @Before
+ public void setUp() {
+ anonymous = new AuthenticationInfo("anonymous");
+ }
+
+ private List> getListOfReposotiry() throws IOException {
+ GetMethod get = httpGet("/notebook-repositories");
+ Map responce = gson.fromJson(get.getResponseBodyAsString(), new TypeToken>() {}.getType());
+ get.releaseConnection();
+ return (List>) responce.get("body");
+ }
+
+ private void updateNotebookRepoWithNewSetting(String payload) throws IOException {
+ PutMethod put = httpPut("/notebook-repositories", payload);
+ int status = put.getStatusCode();
+ put.releaseConnection();
+ assertThat(status, is(200));
+ }
+
+ @Test public void ThatCanGetNotebookRepositoiesSettings() throws IOException {
+ List> listOfRepositories = getListOfReposotiry();
+ assertThat(listOfRepositories.size(), is(not(0)));
+ }
+
+ @Test public void setNewDirectoryForLocalDirectory() throws IOException {
+ List> listOfRepositories = getListOfReposotiry();
+ String localVfs = StringUtils.EMPTY;
+ String className = StringUtils.EMPTY;
+
+ for (int i = 0; i < listOfRepositories.size(); i++) {
+ if (listOfRepositories.get(i).get("name").equals("VFSNotebookRepo")) {
+ localVfs = (String) ((List>)listOfRepositories.get(i).get("settings")).get(0).get("selected");
+ className = (String) listOfRepositories.get(i).get("className");
+ break;
+ }
+ }
+
+ if (StringUtils.isBlank(localVfs)) {
+ // no loval VFS set...
+ return;
+ }
+
+ String payload = "{ \"name\": \"" + className + "\", \"settings\" : { \"Notebook Path\" : \"/tmp/newDir\" } }";
+ updateNotebookRepoWithNewSetting(payload);
+
+ // Verify
+ listOfRepositories = getListOfReposotiry();
+ String updatedPath = StringUtils.EMPTY;
+ for (int i = 0; i < listOfRepositories.size(); i++) {
+ if (listOfRepositories.get(i).get("name").equals("VFSNotebookRepo")) {
+ updatedPath = (String) ((List>)listOfRepositories.get(i).get("settings")).get(0).get("selected");
+ break;
+ }
+ }
+ assertThat(updatedPath, is("/tmp/newDir"));
+
+ // go back to normal
+ payload = "{ \"name\": \"" + className + "\", \"settings\" : { \"Notebook Path\" : \"" + localVfs + "\" } }";
+ updateNotebookRepoWithNewSetting(payload);
+ }
+}
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
index d7f55f54744..36b0f1c97b3 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
@@ -28,7 +28,9 @@
import org.apache.zeppelin.notebook.NotebookAuthorization;
import org.apache.zeppelin.notebook.NotebookAuthorizationInfoSaving;
import org.apache.zeppelin.server.ZeppelinServer;
+import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.AfterClass;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.FixMethodOrder;
import org.junit.Test;
@@ -49,6 +51,7 @@
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class NotebookRestApiTest extends AbstractTestRestApi {
Gson gson = new Gson();
+ AuthenticationInfo anonymous;
@BeforeClass
public static void init() throws Exception {
@@ -60,9 +63,14 @@ public static void destroy() throws Exception {
AbstractTestRestApi.shutDown();
}
+ @Before
+ public void setUp() {
+ anonymous = new AuthenticationInfo("anonymous");
+ }
+
@Test
public void testPermissions() throws IOException {
- Note note1 = ZeppelinServer.notebook.createNote(null);
+ Note note1 = ZeppelinServer.notebook.createNote(anonymous);
// Set only readers
String jsonRequest = "{\"readers\":[\"admin-team\"],\"owners\":[]," +
"\"writers\":[]}";
@@ -85,7 +93,7 @@ public void testPermissions() throws IOException {
get.releaseConnection();
- Note note2 = ZeppelinServer.notebook.createNote(null);
+ Note note2 = ZeppelinServer.notebook.createNote(anonymous);
// Set only writers
jsonRequest = "{\"readers\":[],\"owners\":[]," +
"\"writers\":[\"admin-team\"]}";
@@ -119,14 +127,14 @@ public void testPermissions() throws IOException {
assertEquals(authInfo.get("owners"), Lists.newArrayList());
get.releaseConnection();
//cleanup
- ZeppelinServer.notebook.removeNote(note1.getId(), null);
- ZeppelinServer.notebook.removeNote(note2.getId(), null);
+ ZeppelinServer.notebook.removeNote(note1.getId(), anonymous);
+ ZeppelinServer.notebook.removeNote(note2.getId(), anonymous);
}
@Test
public void testGetNoteParagraphJobStatus() throws IOException {
- Note note1 = ZeppelinServer.notebook.createNote(null);
+ Note note1 = ZeppelinServer.notebook.createNote(anonymous);
note1.addParagraph();
String paragraphId = note1.getLastParagraph().getId();
@@ -142,33 +150,33 @@ public void testGetNoteParagraphJobStatus() throws IOException {
assertEquals(paragraphStatus.get("status"), "READY");
//cleanup
- ZeppelinServer.notebook.removeNote(note1.getId(), null);
+ ZeppelinServer.notebook.removeNote(note1.getId(), anonymous);
}
@Test
- public void testCloneNotebook() throws IOException {
- Note note1 = ZeppelinServer.notebook.createNote(null);
+ public void testCloneNote() throws IOException {
+ Note note1 = ZeppelinServer.notebook.createNote(anonymous);
PostMethod post = httpPost("/notebook/" + note1.getId(), "");
- LOG.info("testCloneNotebook response\n" + post.getResponseBodyAsString());
+ LOG.info("testCloneNote response\n" + post.getResponseBodyAsString());
assertThat(post, isCreated());
Map resp = gson.fromJson(post.getResponseBodyAsString(), new TypeToken>() {
}.getType());
- String clonedNotebookId = (String) resp.get("body");
+ String clonedNoteId = (String) resp.get("body");
post.releaseConnection();
- GetMethod get = httpGet("/notebook/" + clonedNotebookId);
+ GetMethod get = httpGet("/notebook/" + clonedNoteId);
assertThat(get, isAllowed());
Map resp2 = gson.fromJson(get.getResponseBodyAsString(), new TypeToken>() {
}.getType());
Map resp2Body = (Map) resp2.get("body");
- assertEquals((String)resp2Body.get("name"), "Note " + clonedNotebookId);
+ assertEquals((String)resp2Body.get("name"), "Note " + clonedNoteId);
get.releaseConnection();
//cleanup
- ZeppelinServer.notebook.removeNote(note1.getId(), null);
- ZeppelinServer.notebook.removeNote(clonedNotebookId, null);
+ ZeppelinServer.notebook.removeNote(note1.getId(), anonymous);
+ ZeppelinServer.notebook.removeNote(clonedNoteId, anonymous);
}
}
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java
index 4390d74b495..692b4da1fbe 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java
@@ -19,20 +19,22 @@
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import com.google.common.collect.Sets;
import org.apache.commons.httpclient.methods.DeleteMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.PutMethod;
import org.apache.commons.lang3.StringUtils;
-import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Paragraph;
-import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.server.ZeppelinServer;
+import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.AfterClass;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.FixMethodOrder;
import org.junit.Test;
@@ -50,6 +52,7 @@
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class ZeppelinRestApiTest extends AbstractTestRestApi {
Gson gson = new Gson();
+ AuthenticationInfo anonymous;
@BeforeClass
public static void init() throws Exception {
@@ -61,6 +64,11 @@ public static void destroy() throws Exception {
AbstractTestRestApi.shutDown();
}
+ @Before
+ public void setUp() {
+ anonymous = new AuthenticationInfo("anonymous");
+ }
+
/***
* ROOT API TEST
***/
@@ -74,10 +82,10 @@ public void getApiRoot() throws IOException {
}
@Test
- public void testGetNotebookInfo() throws IOException {
- LOG.info("testGetNotebookInfo");
+ public void testGetNoteInfo() throws IOException {
+ LOG.info("testGetNoteInfo");
// Create note to get info
- Note note = ZeppelinServer.notebook.createNote(null);
+ Note note = ZeppelinServer.notebook.createNote(anonymous);
assertNotNull("can't create new note", note);
note.setName("note");
Paragraph paragraph = note.addParagraph();
@@ -86,12 +94,12 @@ public void testGetNotebookInfo() throws IOException {
paragraph.setConfig(config);
String paragraphText = "%md This is my new paragraph in my new note";
paragraph.setText(paragraphText);
- note.persist(null);
+ note.persist(anonymous);
- String sourceNoteID = note.getId();
- GetMethod get = httpGet("/notebook/" + sourceNoteID);
- LOG.info("testGetNotebookInfo \n" + get.getResponseBodyAsString());
- assertThat("test notebook get method:", get, isAllowed());
+ String sourceNoteId = note.getId();
+ GetMethod get = httpGet("/notebook/" + sourceNoteId);
+ LOG.info("testGetNoteInfo \n" + get.getResponseBodyAsString());
+ assertThat("test note get method:", get, isAllowed());
Map resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken>() {
}.getType());
@@ -104,44 +112,46 @@ public void testGetNotebookInfo() throws IOException {
assertTrue(paragraphs.size() > 0);
assertEquals(paragraphText, paragraphs.get(0).get("text"));
+ //
+ ZeppelinServer.notebook.removeNote(sourceNoteId, anonymous);
}
@Test
- public void testNotebookCreateWithName() throws IOException {
+ public void testNoteCreateWithName() throws IOException {
String noteName = "Test note name";
- testNotebookCreate(noteName);
+ testNoteCreate(noteName);
}
@Test
- public void testNotebookCreateNoName() throws IOException {
- testNotebookCreate("");
+ public void testNoteCreateNoName() throws IOException {
+ testNoteCreate("");
}
@Test
- public void testNotebookCreateWithParagraphs() throws IOException {
- // Call Create Notebook REST API
+ public void testNoteCreateWithParagraphs() throws IOException {
+ // Call Create Note REST API
String noteName = "test";
String jsonRequest = "{\"name\":\"" + noteName + "\", \"paragraphs\": [" +
"{\"title\": \"title1\", \"text\": \"text1\"}," +
"{\"title\": \"title2\", \"text\": \"text2\"}" +
"]}";
PostMethod post = httpPost("/notebook/", jsonRequest);
- LOG.info("testNotebookCreate \n" + post.getResponseBodyAsString());
- assertThat("test notebook create method:", post, isCreated());
+ LOG.info("testNoteCreate \n" + post.getResponseBodyAsString());
+ assertThat("test note create method:", post, isCreated());
Map resp = gson.fromJson(post.getResponseBodyAsString(), new TypeToken>() {
}.getType());
- String newNotebookId = (String) resp.get("body");
- LOG.info("newNotebookId:=" + newNotebookId);
- Note newNote = ZeppelinServer.notebook.getNote(newNotebookId);
+ String newNoteId = (String) resp.get("body");
+ LOG.info("newNoteId:=" + newNoteId);
+ Note newNote = ZeppelinServer.notebook.getNote(newNoteId);
assertNotNull("Can not find new note by id", newNote);
// This is partial test as newNote is in memory but is not persistent
String newNoteName = newNote.getName();
LOG.info("new note name is: " + newNoteName);
String expectedNoteName = noteName;
if (noteName.isEmpty()) {
- expectedNoteName = "Note " + newNotebookId;
+ expectedNoteName = "Note " + newNoteId;
}
assertEquals("compare note name", expectedNoteName, newNoteName);
assertEquals("initial paragraph check failed", 3, newNote.getParagraphs().size());
@@ -153,34 +163,34 @@ public void testNotebookCreateWithParagraphs() throws IOException {
assertTrue("paragraph text check failed", p.getText().startsWith("text"));
}
// cleanup
- ZeppelinServer.notebook.removeNote(newNotebookId, null);
+ ZeppelinServer.notebook.removeNote(newNoteId, anonymous);
post.releaseConnection();
}
- private void testNotebookCreate(String noteName) throws IOException {
- // Call Create Notebook REST API
+ private void testNoteCreate(String noteName) throws IOException {
+ // Call Create Note REST API
String jsonRequest = "{\"name\":\"" + noteName + "\"}";
PostMethod post = httpPost("/notebook/", jsonRequest);
- LOG.info("testNotebookCreate \n" + post.getResponseBodyAsString());
- assertThat("test notebook create method:", post, isCreated());
+ LOG.info("testNoteCreate \n" + post.getResponseBodyAsString());
+ assertThat("test note create method:", post, isCreated());
Map resp = gson.fromJson(post.getResponseBodyAsString(), new TypeToken>() {
}.getType());
- String newNotebookId = (String) resp.get("body");
- LOG.info("newNotebookId:=" + newNotebookId);
- Note newNote = ZeppelinServer.notebook.getNote(newNotebookId);
+ String newNoteId = (String) resp.get("body");
+ LOG.info("newNoteId:=" + newNoteId);
+ Note newNote = ZeppelinServer.notebook.getNote(newNoteId);
assertNotNull("Can not find new note by id", newNote);
// This is partial test as newNote is in memory but is not persistent
String newNoteName = newNote.getName();
LOG.info("new note name is: " + newNoteName);
String expectedNoteName = noteName;
if (noteName.isEmpty()) {
- expectedNoteName = "Note " + newNotebookId;
+ expectedNoteName = "Note " + newNoteId;
}
assertEquals("compare note name", expectedNoteName, newNoteName);
// cleanup
- ZeppelinServer.notebook.removeNote(newNotebookId, null);
+ ZeppelinServer.notebook.removeNote(newNoteId, anonymous);
post.releaseConnection();
}
@@ -189,23 +199,23 @@ private void testNotebookCreate(String noteName) throws IOException {
public void testDeleteNote() throws IOException {
LOG.info("testDeleteNote");
//Create note and get ID
- Note note = ZeppelinServer.notebook.createNote(null);
+ Note note = ZeppelinServer.notebook.createNote(anonymous);
String noteId = note.getId();
- testDeleteNotebook(noteId);
+ testDeleteNote(noteId);
}
@Test
public void testDeleteNoteBadId() throws IOException {
LOG.info("testDeleteNoteBadId");
- testDeleteNotebook("2AZFXEX97");
- testDeleteNotebook("bad_ID");
+ testDeleteNote("2AZFXEX97");
+ testDeleteNote("bad_ID");
}
@Test
- public void testExportNotebook() throws IOException {
- LOG.info("testExportNotebook");
- Note note = ZeppelinServer.notebook.createNote(null);
+ public void testexportNote() throws IOException {
+ LOG.info("testexportNote");
+ Note note = ZeppelinServer.notebook.createNote(anonymous);
assertNotNull("can't create new note", note);
note.setName("source note for export");
Paragraph paragraph = note.addParagraph();
@@ -213,12 +223,12 @@ public void testExportNotebook() throws IOException {
config.put("enabled", true);
paragraph.setConfig(config);
paragraph.setText("%md This is my new paragraph in my new note");
- note.persist(null);
- String sourceNoteID = note.getId();
- // Call export Notebook REST API
- GetMethod get = httpGet("/notebook/export/" + sourceNoteID);
- LOG.info("testNotebookExport \n" + get.getResponseBodyAsString());
- assertThat("test notebook export method:", get, isAllowed());
+ note.persist(anonymous);
+ String sourceNoteId = note.getId();
+ // Call export Note REST API
+ GetMethod get = httpGet("/notebook/export/" + sourceNoteId);
+ LOG.info("testNoteExport \n" + get.getResponseBodyAsString());
+ assertThat("test note export method:", get, isAllowed());
Map resp =
gson.fromJson(get.getResponseBodyAsString(),
@@ -227,7 +237,7 @@ public void testExportNotebook() throws IOException {
String exportJSON = (String) resp.get("body");
assertNotNull("Can not find new notejson", exportJSON);
LOG.info("export JSON:=" + exportJSON);
- ZeppelinServer.notebook.removeNote(sourceNoteID, null);
+ ZeppelinServer.notebook.removeNote(sourceNoteId, anonymous);
get.releaseConnection();
}
@@ -236,9 +246,9 @@ public void testExportNotebook() throws IOException {
public void testImportNotebook() throws IOException {
Map resp;
String noteName = "source note for import";
- LOG.info("testImortNotebook");
- // create test notebook
- Note note = ZeppelinServer.notebook.createNote(null);
+ LOG.info("testImortNote");
+ // create test note
+ Note note = ZeppelinServer.notebook.createNote(anonymous);
assertNotNull("can't create new note", note);
note.setName(noteName);
Paragraph paragraph = note.addParagraph();
@@ -246,11 +256,11 @@ public void testImportNotebook() throws IOException {
config.put("enabled", true);
paragraph.setConfig(config);
paragraph.setText("%md This is my new paragraph in my new note");
- note.persist(null);
- String sourceNoteID = note.getId();
+ note.persist(anonymous);
+ String sourceNoteId = note.getId();
// get note content as JSON
- String oldJson = getNoteContent(sourceNoteID);
- // call notebook post
+ String oldJson = getNoteContent(sourceNoteId);
+ // call note post
PostMethod importPost = httpPost("/notebook/import/", oldJson);
assertThat(importPost, isCreated());
resp =
@@ -258,14 +268,14 @@ public void testImportNotebook() throws IOException {
new TypeToken>() {}.getType());
String importId = (String) resp.get("body");
- assertNotNull("Did not get back a notebook id in body", importId);
+ assertNotNull("Did not get back a note id in body", importId);
Note newNote = ZeppelinServer.notebook.getNote(importId);
assertEquals("Compare note names", noteName, newNote.getName());
assertEquals("Compare paragraphs count", note.getParagraphs().size(), newNote.getParagraphs()
.size());
// cleanup
- ZeppelinServer.notebook.removeNote(note.getId(), null);
- ZeppelinServer.notebook.removeNote(newNote.getId(), null);
+ ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
+ ZeppelinServer.notebook.removeNote(newNote.getId(), anonymous);
importPost.releaseConnection();
}
@@ -283,24 +293,24 @@ private String getNoteContent(String id) throws IOException {
return body;
}
- private void testDeleteNotebook(String notebookId) throws IOException {
+ private void testDeleteNote(String noteId) throws IOException {
- DeleteMethod delete = httpDelete(("/notebook/" + notebookId));
- LOG.info("testDeleteNotebook delete response\n" + delete.getResponseBodyAsString());
+ DeleteMethod delete = httpDelete(("/notebook/" + noteId));
+ LOG.info("testDeleteNote delete response\n" + delete.getResponseBodyAsString());
assertThat("Test delete method:", delete, isAllowed());
delete.releaseConnection();
// make sure note is deleted
- if (!notebookId.isEmpty()) {
- Note deletedNote = ZeppelinServer.notebook.getNote(notebookId);
+ if (!noteId.isEmpty()) {
+ Note deletedNote = ZeppelinServer.notebook.getNote(noteId);
assertNull("Deleted note should be null", deletedNote);
}
}
@Test
- public void testCloneNotebook() throws IOException, CloneNotSupportedException, IllegalArgumentException {
- LOG.info("testCloneNotebook");
+ public void testCloneNote() throws IOException, CloneNotSupportedException, IllegalArgumentException {
+ LOG.info("testCloneNote");
// Create note to clone
- Note note = ZeppelinServer.notebook.createNote(null);
+ Note note = ZeppelinServer.notebook.createNote(anonymous);
assertNotNull("can't create new note", note);
note.setName("source note for clone");
Paragraph paragraph = note.addParagraph();
@@ -308,40 +318,42 @@ public void testCloneNotebook() throws IOException, CloneNotSupportedException,
config.put("enabled", true);
paragraph.setConfig(config);
paragraph.setText("%md This is my new paragraph in my new note");
- note.persist(null);
- String sourceNoteID = note.getId();
+ note.persist(anonymous);
+ String sourceNoteId = note.getId();
String noteName = "clone Note Name";
- // Call Clone Notebook REST API
+ // Call Clone Note REST API
String jsonRequest = "{\"name\":\"" + noteName + "\"}";
- PostMethod post = httpPost("/notebook/" + sourceNoteID, jsonRequest);
- LOG.info("testNotebookClone \n" + post.getResponseBodyAsString());
- assertThat("test notebook clone method:", post, isCreated());
+ PostMethod post = httpPost("/notebook/" + sourceNoteId, jsonRequest);
+ LOG.info("testNoteClone \n" + post.getResponseBodyAsString());
+ assertThat("test note clone method:", post, isCreated());
Map resp = gson.fromJson(post.getResponseBodyAsString(), new TypeToken>() {
}.getType());
- String newNotebookId = (String) resp.get("body");
- LOG.info("newNotebookId:=" + newNotebookId);
- Note newNote = ZeppelinServer.notebook.getNote(newNotebookId);
+ String newNoteId = (String) resp.get("body");
+ LOG.info("newNoteId:=" + newNoteId);
+ Note newNote = ZeppelinServer.notebook.getNote(newNoteId);
assertNotNull("Can not find new note by id", newNote);
assertEquals("Compare note names", noteName, newNote.getName());
assertEquals("Compare paragraphs count", note.getParagraphs().size(), newNote.getParagraphs().size());
//cleanup
- ZeppelinServer.notebook.removeNote(note.getId(), null);
- ZeppelinServer.notebook.removeNote(newNote.getId(), null);
+ ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
+ ZeppelinServer.notebook.removeNote(newNote.getId(), anonymous);
post.releaseConnection();
}
@Test
- public void testListNotebooks() throws IOException {
- LOG.info("testListNotebooks");
+ public void testListNotes() throws IOException {
+ LOG.info("testListNotes");
GetMethod get = httpGet("/notebook/ ");
- assertThat("List notebooks method", get, isAllowed());
+ assertThat("List notes method", get, isAllowed());
Map resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken>() {
}.getType());
List> body = (List>) resp.get("body");
- assertEquals("List notebooks are equal", ZeppelinServer.notebook.getAllNotes().size(), body.size());
+ //TODO(khalid): anonymous or specific user notes?
+ HashSet anonymous = Sets.newHashSet("anonymous");
+ assertEquals("List notes are equal", ZeppelinServer.notebook.getAllNotes(anonymous).size(), body.size());
get.releaseConnection();
}
@@ -349,7 +361,7 @@ public void testListNotebooks() throws IOException {
public void testNoteJobs() throws IOException, InterruptedException {
LOG.info("testNoteJobs");
// Create note to run test.
- Note note = ZeppelinServer.notebook.createNote(null);
+ Note note = ZeppelinServer.notebook.createNote(anonymous);
assertNotNull("can't create new note", note);
note.setName("note for run test");
Paragraph paragraph = note.addParagraph();
@@ -359,8 +371,8 @@ public void testNoteJobs() throws IOException, InterruptedException {
paragraph.setConfig(config);
paragraph.setText("%md This is test paragraph.");
- note.persist(null);
- String noteID = note.getId();
+ note.persist(anonymous);
+ String noteId = note.getId();
note.runAll();
// wait until job is finished or timeout.
@@ -373,38 +385,38 @@ public void testNoteJobs() throws IOException, InterruptedException {
}
}
- // Call Run Notebook Jobs REST API
- PostMethod postNoteJobs = httpPost("/notebook/job/" + noteID, "");
- assertThat("test notebook jobs run:", postNoteJobs, isAllowed());
+ // Call Run note jobs REST API
+ PostMethod postNoteJobs = httpPost("/notebook/job/" + noteId, "");
+ assertThat("test note jobs run:", postNoteJobs, isAllowed());
postNoteJobs.releaseConnection();
- // Call Stop Notebook Jobs REST API
- DeleteMethod deleteNoteJobs = httpDelete("/notebook/job/" + noteID);
- assertThat("test notebook stop:", deleteNoteJobs, isAllowed());
+ // Call Stop note jobs REST API
+ DeleteMethod deleteNoteJobs = httpDelete("/notebook/job/" + noteId);
+ assertThat("test note stop:", deleteNoteJobs, isAllowed());
deleteNoteJobs.releaseConnection();
Thread.sleep(1000);
// Call Run paragraph REST API
- PostMethod postParagraph = httpPost("/notebook/job/" + noteID + "/" + paragraph.getId(), "");
+ PostMethod postParagraph = httpPost("/notebook/job/" + noteId + "/" + paragraph.getId(), "");
assertThat("test paragraph run:", postParagraph, isAllowed());
postParagraph.releaseConnection();
Thread.sleep(1000);
// Call Stop paragraph REST API
- DeleteMethod deleteParagraph = httpDelete("/notebook/job/" + noteID + "/" + paragraph.getId());
+ DeleteMethod deleteParagraph = httpDelete("/notebook/job/" + noteId + "/" + paragraph.getId());
assertThat("test paragraph stop:", deleteParagraph, isAllowed());
deleteParagraph.releaseConnection();
Thread.sleep(1000);
//cleanup
- ZeppelinServer.notebook.removeNote(note.getId(), null);
+ ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
- public void testGetNotebookJob() throws IOException, InterruptedException {
- LOG.info("testGetNotebookJob");
+ public void testGetNoteJob() throws IOException, InterruptedException {
+ LOG.info("testGetNoteJob");
// Create note to run test.
- Note note = ZeppelinServer.notebook.createNote(null);
+ Note note = ZeppelinServer.notebook.createNote(anonymous);
assertNotNull("can't create new note", note);
note.setName("note for run test");
Paragraph paragraph = note.addParagraph();
@@ -414,8 +426,9 @@ public void testGetNotebookJob() throws IOException, InterruptedException {
paragraph.setConfig(config);
paragraph.setText("%sh sleep 1");
- note.persist(null);
- String noteID = note.getId();
+ paragraph.setAuthenticationInfo(anonymous);
+ note.persist(anonymous);
+ String noteId = note.getId();
note.runAll();
@@ -425,12 +438,12 @@ public void testGetNotebookJob() throws IOException, InterruptedException {
}
// assume that status of the paragraph is running
- GetMethod get = httpGet("/notebook/job/" + noteID);
- assertThat("test get notebook job: ", get, isAllowed());
+ GetMethod get = httpGet("/notebook/job/" + noteId);
+ assertThat("test get note job: ", get, isAllowed());
String responseBody = get.getResponseBodyAsString();
get.releaseConnection();
- LOG.info("test get notebook job: \n" + responseBody);
+ LOG.info("test get note job: \n" + responseBody);
Map resp = gson.fromJson(responseBody, new TypeToken>() {
}.getType());
@@ -445,19 +458,19 @@ public void testGetNotebookJob() throws IOException, InterruptedException {
while (!paragraph.isTerminated()) {
Thread.sleep(100);
if (timeout++ > 10) {
- LOG.info("testGetNotebookJob timeout job.");
+ LOG.info("testGetNoteJob timeout job.");
break;
}
}
- ZeppelinServer.notebook.removeNote(note.getId(), null);
+ ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testRunParagraphWithParams() throws IOException, InterruptedException {
LOG.info("testRunParagraphWithParams");
// Create note to run test.
- Note note = ZeppelinServer.notebook.createNote(null);
+ Note note = ZeppelinServer.notebook.createNote(anonymous);
assertNotNull("can't create new note", note);
note.setName("note for run test");
Paragraph paragraph = note.addParagraph();
@@ -467,8 +480,8 @@ public void testRunParagraphWithParams() throws IOException, InterruptedExceptio
paragraph.setConfig(config);
paragraph.setText("%spark\nval param = z.input(\"param\").toString\nprintln(param)");
- note.persist(null);
- String noteID = note.getId();
+ note.persist(anonymous);
+ String noteId = note.getId();
note.runAll();
// wait until job is finished or timeout.
@@ -482,26 +495,26 @@ public void testRunParagraphWithParams() throws IOException, InterruptedExceptio
}
// Call Run paragraph REST API
- PostMethod postParagraph = httpPost("/notebook/job/" + noteID + "/" + paragraph.getId(),
+ PostMethod postParagraph = httpPost("/notebook/job/" + noteId + "/" + paragraph.getId(),
"{\"params\": {\"param\": \"hello\", \"param2\": \"world\"}}");
assertThat("test paragraph run:", postParagraph, isAllowed());
postParagraph.releaseConnection();
Thread.sleep(1000);
- Note retrNote = ZeppelinServer.notebook.getNote(noteID);
+ Note retrNote = ZeppelinServer.notebook.getNote(noteId);
Paragraph retrParagraph = retrNote.getParagraph(paragraph.getId());
Map params = retrParagraph.settings.getParams();
assertEquals("hello", params.get("param"));
assertEquals("world", params.get("param2"));
//cleanup
- ZeppelinServer.notebook.removeNote(note.getId(), null);
+ ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
- public void testCronJobs() throws InterruptedException, IOException{
+ public void testJobs() throws InterruptedException, IOException{
// create a note and a paragraph
- Note note = ZeppelinServer.notebook.createNote(null);
+ Note note = ZeppelinServer.notebook.createNote(anonymous);
note.setName("note for run test");
Paragraph paragraph = note.addParagraph();
@@ -545,21 +558,21 @@ public void testCronJobs() throws InterruptedException, IOException{
DeleteMethod deleteCron = httpDelete("/notebook/cron/" + note.getId());
assertThat("", deleteCron, isAllowed());
deleteCron.releaseConnection();
- ZeppelinServer.notebook.removeNote(note.getId(), null);
+ ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testRegressionZEPPELIN_527() throws IOException {
- Note note = ZeppelinServer.notebook.createNote(null);
+ Note note = ZeppelinServer.notebook.createNote(anonymous);
note.setName("note for run test");
Paragraph paragraph = note.addParagraph();
paragraph.setText("%spark\nval param = z.input(\"param\").toString\nprintln(param)");
- note.persist(null);
+ note.persist(anonymous);
GetMethod getNoteJobs = httpGet("/notebook/job/" + note.getId());
- assertThat("test notebook jobs run:", getNoteJobs, isAllowed());
+ assertThat("test note jobs run:", getNoteJobs, isAllowed());
Map resp = gson.fromJson(getNoteJobs.getResponseBodyAsString(), new TypeToken>() {
}.getType());
List> body = (List>) resp.get("body");
@@ -567,12 +580,12 @@ public void testRegressionZEPPELIN_527() throws IOException {
assertFalse(body.get(0).containsKey("finished"));
getNoteJobs.releaseConnection();
- ZeppelinServer.notebook.removeNote(note.getId(), null);
+ ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testInsertParagraph() throws IOException {
- Note note = ZeppelinServer.notebook.createNote(null);
+ Note note = ZeppelinServer.notebook.createNote(anonymous);
String jsonRequest = "{\"title\": \"title1\", \"text\": \"text1\"}";
PostMethod post = httpPost("/notebook/" + note.getId() + "/paragraph", jsonRequest);
@@ -607,17 +620,17 @@ public void testInsertParagraph() throws IOException {
assertEquals("title2", paragraphAtIdx0.getTitle());
assertEquals("text2", paragraphAtIdx0.getText());
- ZeppelinServer.notebook.removeNote(note.getId(), null);
+ ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testGetParagraph() throws IOException {
- Note note = ZeppelinServer.notebook.createNote(null);
+ Note note = ZeppelinServer.notebook.createNote(anonymous);
Paragraph p = note.addParagraph();
p.setTitle("hello");
p.setText("world");
- note.persist(null);
+ note.persist(anonymous);
GetMethod get = httpGet("/notebook/" + note.getId() + "/paragraph/" + p.getId());
LOG.info("testGetParagraph response\n" + get.getResponseBodyAsString());
@@ -636,12 +649,12 @@ public void testGetParagraph() throws IOException {
assertEquals("hello", body.get("title"));
assertEquals("world", body.get("text"));
- ZeppelinServer.notebook.removeNote(note.getId(), null);
+ ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testMoveParagraph() throws IOException {
- Note note = ZeppelinServer.notebook.createNote(null);
+ Note note = ZeppelinServer.notebook.createNote(anonymous);
Paragraph p = note.addParagraph();
p.setTitle("title1");
@@ -651,7 +664,7 @@ public void testMoveParagraph() throws IOException {
p2.setTitle("title2");
p2.setText("text2");
- note.persist(null);
+ note.persist(anonymous);
PostMethod post = httpPost("/notebook/" + note.getId() + "/paragraph/" + p2.getId() + "/move/" + 0, "");
assertThat("Test post method: ", post, isAllowed());
@@ -668,18 +681,18 @@ public void testMoveParagraph() throws IOException {
assertThat("Test post method: ", post2, isBadRequest());
post.releaseConnection();
- ZeppelinServer.notebook.removeNote(note.getId(), null);
+ ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testDeleteParagraph() throws IOException {
- Note note = ZeppelinServer.notebook.createNote(null);
+ Note note = ZeppelinServer.notebook.createNote(anonymous);
Paragraph p = note.addParagraph();
p.setTitle("title1");
p.setText("text1");
- note.persist(null);
+ note.persist(anonymous);
DeleteMethod delete = httpDelete("/notebook/" + note.getId() + "/paragraph/" + p.getId());
assertThat("Test delete method: ", delete, isAllowed());
@@ -689,7 +702,7 @@ public void testDeleteParagraph() throws IOException {
Paragraph retrParagrah = retrNote.getParagraph(p.getId());
assertNull("paragraph should be deleted", retrParagrah);
- ZeppelinServer.notebook.removeNote(note.getId(), null);
+ ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
@@ -705,15 +718,15 @@ public void testSearch() throws IOException {
String username = body.get("principal");
getSecurityTicket.releaseConnection();
- Note note1 = ZeppelinServer.notebook.createNote(null);
+ Note note1 = ZeppelinServer.notebook.createNote(anonymous);
String jsonRequest = "{\"title\": \"title1\", \"text\": \"ThisIsToTestSearchMethodWithPermissions 1\"}";
- PostMethod postNotebookText = httpPost("/notebook/" + note1.getId() + "/paragraph", jsonRequest);
- postNotebookText.releaseConnection();
+ PostMethod postNoteText = httpPost("/notebook/" + note1.getId() + "/paragraph", jsonRequest);
+ postNoteText.releaseConnection();
- Note note2 = ZeppelinServer.notebook.createNote(null);
+ Note note2 = ZeppelinServer.notebook.createNote(anonymous);
jsonRequest = "{\"title\": \"title1\", \"text\": \"ThisIsToTestSearchMethodWithPermissions 2\"}";
- postNotebookText = httpPost("/notebook/" + note2.getId() + "/paragraph", jsonRequest);
- postNotebookText.releaseConnection();
+ postNoteText = httpPost("/notebook/" + note2.getId() + "/paragraph", jsonRequest);
+ postNoteText.releaseConnection();
String jsonPermissions = "{\"owners\":[\"" + username + "\"],\"readers\":[\"" + username + "\"],\"writers\":[\"" + username + "\"]}";
PutMethod putPermission = httpPut("/notebook/" + note1.getId() + "/permissions", jsonPermissions);
@@ -723,9 +736,9 @@ public void testSearch() throws IOException {
putPermission = httpPut("/notebook/" + note2.getId() + "/permissions", jsonPermissions);
putPermission.releaseConnection();
- GetMethod searchNotebook = httpGet("/notebook/search?q='ThisIsToTestSearchMethodWithPermissions'");
- searchNotebook.addRequestHeader("Origin", "http://localhost");
- Map
@@ -41,7 +41,7 @@ With `%html` directive, Zeppelin treats your output as HTML
## Table
-If you have data that row seprated by '\n' (newline) and column separated by '\t' (tab) with first row as header row, for example
+If you have data that row separated by '\n' (newline) and column separated by '\t' (tab) with first row as header row, for example
diff --git a/docs/index.md b/docs/index.md
index 8c2ce95cc95..0f257500eec 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -172,6 +172,7 @@ Join to our [Mailing list](https://zeppelin.apache.org/community.html) and repor
* [Zeppelin on Spark Cluster Mode (Standalone via Docker)](./install/spark_cluster_mode.html#spark-standalone-mode)
* [Zeppelin on Spark Cluster Mode (YARN via Docker)](./install/spark_cluster_mode.html#spark-on-yarn-mode)
* [Zeppelin on Spark Cluster Mode (Mesos via Docker)](./install/spark_cluster_mode.html#spark-on-mesos-mode)
+ * [Zeppelin on CDH (via Docker)](./install/cdh.html)
* Contribute
* [Writing Zeppelin Interpreter](./development/writingzeppelininterpreter.html)
* [Writing Zeppelin Application (Experimental)](./development/writingzeppelinapplication.html)
diff --git a/docs/install/cdh.md b/docs/install/cdh.md
new file mode 100644
index 00000000000..f661417a3f6
--- /dev/null
+++ b/docs/install/cdh.md
@@ -0,0 +1,100 @@
+---
+layout: page
+title: "Apache Zeppelin on CDH"
+description: "This document will guide you how you can build and configure the environment on CDH with Apache Zeppelin using docker scripts."
+group: install
+---
+
+{% include JB/setup %}
+
+# Apache Zeppelin on CDH
+
+
+
+### 1. Import Cloudera QuickStart Docker image
+
+>[Cloudera](http://www.cloudera.com/) has officially provided CDH Docker Hub in their own container. Please check [this guide page](http://www.cloudera.com/documentation/enterprise/latest/topics/quickstart_docker_container.html#cloudera_docker_container) for more information.
+
+You can import the Docker image by pulling it from Cloudera Docker Hub.
+
+```
+docker pull cloudera/quickstart:latest
+```
+
+
+### 2. Run docker
+
+```
+docker run -it \
+ -p 80:80 \
+ -p 4040:4040 \
+ -p 8020:8020 \
+ -p 8022:8022 \
+ -p 8030:8030 \
+ -p 8032:8032 \
+ -p 8033:8033 \
+ -p 8040:8040 \
+ -p 8042:8042 \
+ -p 8088:8088 \
+ -p 8480:8480 \
+ -p 8485:8485 \
+ -p 8888:8888 \
+ -p 9083:9083 \
+ -p 10020:10020 \
+ -p 10033:10033 \
+ -p 18088:18088 \
+ -p 19888:19888 \
+ -p 25000:25000 \
+ -p 25010:25010 \
+ -p 25020:25020 \
+ -p 50010:50010 \
+ -p 50020:50020 \
+ -p 50070:50070 \
+ -p 50075:50075 \
+ -h quickstart.cloudera --privileged=true \
+ agitated_payne_backup /usr/bin/docker-quickstart;
+```
+
+### 3. Verify running CDH
+
+To verify the application is running well, check the web UI for HDFS on `http://
+
+### 5. Run Zeppelin with Spark interpreter
+After running a single paragraph with Spark interpreter in Zeppelin,
+
+
+
+
+
diff --git a/docs/install/install.md b/docs/install/install.md
index 4d81fe5ce7f..e14fc7c2773 100644
--- a/docs/install/install.md
+++ b/docs/install/install.md
@@ -1,7 +1,7 @@
---
layout: page
title: "Quick Start"
-description: "This page will help you to get started and guide you through installation of Apache Zeppelin, running it in the command line and basic configuration options."
+description: "This page will help you get started and will guide you through installing Apache Zeppelin, running it in the command line and configuring options."
group: install
---
+
+{% include JB/setup %}
+
+# Beam interpreter for Apache Zeppelin
+
+
+
+## Overview
+[Apache Beam](http://beam.incubator.apache.org) is an open source unified platform for data processing pipelines. A pipeline can be build using one of the Beam SDKs.
+The execution of the pipeline is done by different Runners. Currently, Beam supports Apache Flink Runner, Apache Spark Runner, and Google Dataflow Runner.
+
+## How to use
+Basically, you can write normal Beam java code where you can determine the Runner. You should write the main method inside a class becuase the interpreter invoke this main to execute the pipeline. Unlike Zeppelin normal pattern, each paragraph is considered as a separate job, there isn't any relation to any other paragraph.
+
+The following is a demonstration of a word count example with data represented in array of strings
+But it can read data from files by replacing `Create.of(SENTENCES).withCoder(StringUtf8Coder.of())` with `TextIO.Read.from("path/to/filename.txt")`
+
+```java
+%beam
+
+// most used imports
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.transforms.Create;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+import java.util.ArrayList;
+import org.apache.spark.api.java.*;
+import org.apache.spark.api.java.function.Function;
+import org.apache.spark.SparkConf;
+import org.apache.spark.streaming.*;
+import org.apache.spark.SparkContext;
+import org.apache.beam.runners.direct.*;
+import org.apache.beam.sdk.runners.*;
+import org.apache.beam.sdk.options.*;
+import org.apache.beam.runners.spark.*;
+import org.apache.beam.runners.spark.io.ConsoleIO;
+import org.apache.beam.runners.flink.*;
+import org.apache.beam.runners.flink.examples.WordCount.Options;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.io.TextIO;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.options.PipelineOptions;
+
+public class MinimalWordCount {
+ static List
+ ++
-## Create Interpreter
+Fill `Interpreter name` field with whatever you want to use as the alias(e.g. mysql, mysql2, hive, redshift, and etc..). Please note that this alias will be used as `%interpreter_name` to call the interpreter in the paragraph.
+Then select `jdbc` as an `Interpreter group`.
-When you create a interpreter by default use PostgreSQL with the next properties:
+
+
+The default driver of JDBC interpreter is set as `PostgreSQL`. It means Zeppelin includes `PostgreSQL` driver jar in itself.
+So you don't need to add any dependencies(e.g. the artifact name or path for `PostgreSQL` driver jar) for `PostgreSQL` connection.
+The JDBC interpreter properties are defined by default like below.
-Prior to creating the interpreter it is necessary to add maven coordinate or path of the JDBC driver to the Zeppelin classpath. To do this you must edit dependencies artifact(ex. `mysql:mysql-connector-java:5.1.38`) in interpreter menu as shown:
+The last step is **Dependency Setting**. Since Zeppelin only includes `PostgreSQL` driver jar by default, you need to add each driver's maven coordinates or JDBC driver's jar file path for the other databases.
--+--
- 
+
+That's it. You can find more JDBC connection setting examples([Mysql](#mysql), [Apache Hive](#apache-hive), [Apache Phoenix](#apache-phoenix), and [Apache Tajo](#apache-tajo)) in [this section](#examples).
-To create the interpreter you need to specify connection parameters as shown in the table.
+## More properties
+There are more JDBC interpreter properties you can specify like below.
- 
+
+Select(blue) or deselect(white) the interpreter buttons depending on your use cases.
+If you need to use more than one interpreter in the notebook, activate several buttons.
+Don't forget to click `Save` button, or you will face `Interpreter *** is not found` error.
+
+
+
+## How to use
+### Run the paragraph with JDBC interpreter
+To test whether your databases and Zeppelin are successfully connected or not, type `%jdbc_interpreter_name`(e.g. `%mysql`) at the top of the paragraph and run `show databases`.
+
+```sql
+%jdbc_interpreter_name
+show databases
+```
+If the paragraph is `FINISHED` without any errors, a new paragraph will be automatically added after the previous one with `%jdbc_interpreter_name`.
+So you don't need to type this prefix in every paragraphs' header.
+
+
+
+### Apply Zeppelin Dynamic Forms
+
+You can leverage [Zeppelin Dynamic Form](../manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parametrization features.
+
+```sql
+%jdbc_interpreter_name
+SELECT name, country, performer
+FROM demo.performers
+WHERE name='{{"{{performer=Sheryl Crow|Doof|Fanfarlo|Los Paranoia"}}}}'
+```
+
+## Examples
+Here are some examples you can refer to. Including the below connectors, you can connect every databases as long as it can be configured with it's JDBC driver.
+
+### Mysql
+
+
+
+##### Properties
+
+##### Properties
-```sql
-%jdbc(prefix)
-SELECT * FROM db_name;
+##### Properties
+
-```sql
-%jdbc(prefix)
-SELECT name, country, performer
-FROM demo.performers
-WHERE name='{{performer=Sheryl Crow|Doof|Fanfarlo|Los Paranoia}}'
-```
+##### Properties
+
+
+##### Properties
+
-
+## Configuration
+
+
+
+### Markdown4j Parser
+
+`markdown4j` parser provides [YUML](http://yuml.me/) and [Websequence](https://www.websequencediagrams.com/) extensions
+
+
+
+### Pegdown Parser
+
+`pegdown` parser provides github flavored markdown.
+
+
diff --git a/docs/interpreter/pig.md b/docs/interpreter/pig.md
new file mode 100644
index 00000000000..227656bba78
--- /dev/null
+++ b/docs/interpreter/pig.md
@@ -0,0 +1,97 @@
+---
+layout: page
+title: "Pig Interpreter for Apache Zeppelin"
+description: "Apache Pig is a platform for analyzing large data sets that consists of a high-level language for expressing data analysis programs, coupled with infrastructure for evaluating these programs."
+group: manual
+---
+{% include JB/setup %}
+
+
+# Pig Interpreter for Apache Zeppelin
+
+
+
+## Overview
+[Apache Pig](https://pig.apache.org/) is a platform for analyzing large data sets that consists of a high-level language for expressing data analysis programs, coupled with infrastructure for evaluating these programs. The salient property of Pig programs is that their structure is amenable to substantial parallelization, which in turns enables them to handle very large data sets.
+
+## Supported interpreter type
+ - `%pig.script` (default)
+
+ All the pig script can run in this type of interpreter, and display type is plain text.
+
+ - `%pig.query`
+
+ Almost the same as `%pig.script`. The only difference is that you don't need to add alias in the last statement. And the display type is table.
+
+## Supported runtime mode
+ - Local
+ - MapReduce
+ - Tez (Only Tez 0.7 is supported)
+
+## How to use
+
+### How to setup Pig
+
+- Local Mode
+
+ Nothing needs to be done for local mode
+
+- MapReduce Mode
+
+ HADOOP\_CONF\_DIR needs to be specified in `ZEPPELIN_HOME/conf/zeppelin-env.sh`.
+
+- Tez Mode
+
+ HADOOP\_CONF\_DIR and TEZ\_CONF\_DIR needs to be specified in `ZEPPELIN_HOME/conf/zeppelin-env.sh`.
+
+### How to configure interpreter
+
+At the Interpreters menu, you have to create a new Pig interpreter. Pig interpreter has below properties by default.
+
+
+
If you need further information about **Zeppelin Interpreter Setting** for using Shell interpreter, please read [What is interpreter setting?](../manual/interpreters.html#what-is-interpreter-setting) section first.
\ No newline at end of file
diff --git a/docs/interpreter/spark.md b/docs/interpreter/spark.md
index 90ac9b0b948..035a01e19c1 100644
--- a/docs/interpreter/spark.md
+++ b/docs/interpreter/spark.md
@@ -1,7 +1,7 @@
---
layout: page
title: "Apache Spark Interpreter for Apache Zeppelin"
-description: "Apache Spark is a fast and general-purpose cluster computing system. It provides high-level APIs in Java, Scala, Python and R, and an optimized engine that supports general execution graphs."
+description: "Apache Spark is a fast and general-purpose cluster computing system. It provides high-level APIs in Java, Scala, Python and R, and an optimized engine that supports general execution engine."
group: interpreter
---
+
+{% include JB/setup %}
+
+# Install with flink and spark cluster
+
+
+
+This tutorial is extremely entry-level. It assumes no prior knowledge of Linux, git, or other tools. If you carefully type what I tell you when I tell you, you should be able to get Zeppelin running.
+
+## Installing Zeppelin with Flink and Spark in cluster mode
+
+This tutorial assumes the user has a machine (real or [virtual](https://www.virtualbox.org/wiki/Downloads) with a fresh, minimal installation of [Ubuntu 14.04.3 Server](http://www.ubuntu.com/download/server).
+
+**Note:** On the size requirements of the Virtual Machine, some users reported trouble when using the default virtual machine sizes, specifically that the hard drive needed to be at least 16GB- other users did not have this issue.
+
+There are many good tutorials on how to install Ubuntu Server on a virtual box, [here is one of them](http://ilearnstack.com/2013/04/13/setting-ubuntu-vm-in-virtualbox/)
+
+### Required Programs
+
+Assuming the minimal install, there are several programs that we will need to install before Zeppelin, Flink, and Spark.
+
+- git
+- openssh-server
+- OpenJDK 7
+- Maven 3.1+
+
+For git, openssh-server, and OpenJDK 7 we will be using the apt package manager.
+
+##### git
+From the command prompt:
+
+```
+sudo apt-get install git
+```
+
+##### openssh-server
+
+```
+sudo apt-get install openssh-server
+```
+
+##### OpenJDK 7
+
+```
+sudo apt-get install openjdk-7-jdk openjdk-7-jre-lib
+```
+*A note for those using Ubuntu 16.04*: To install `openjdk-7` on Ubuntu 16.04, one must add a repository. [Source](http://askubuntu.com/questions/761127/ubuntu-16-04-and-openjdk-7)
+
+``` bash
+sudo add-apt-repository ppa:openjdk-r/ppa
+sudo apt-get update
+sudo apt-get install openjdk-7-jdk openjdk-7-jre-lib
+```
+
+##### Maven 3.1+
+Zeppelin requires maven version 3.x. The version available in the repositories at the time of writing is 2.x, so maven must be installed manually.
+
+Purge any existing versions of maven.
+
+```
+sudo apt-get purge maven maven2
+```
+
+Download the maven 3.3.9 binary.
+
+```
+wget "http://www.us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz"
+```
+
+Unarchive the binary and move to the `/usr/local` directory.
+
+```
+tar -zxvf apache-maven-3.3.9-bin.tar.gz
+sudo mv ./apache-maven-3.3.9 /usr/local
+```
+
+Create symbolic links in `/usr/bin`.
+
+```
+sudo ln -s /usr/local/apache-maven-3.3.9/bin/mvn /usr/bin/mvn
+```
+
+### Installing Zeppelin
+This provides a quick overview of Zeppelin installation from source, however the reader is encouraged to review the [Zeppelin Installation Guide](../install/install.html)
+
+From the command prompt:
+Clone Zeppelin.
+
+```
+git clone https://github.com/apache/zeppelin.git
+```
+
+Enter the Zeppelin root directory.
+
+```
+cd zeppelin
+```
+
+Package Zeppelin.
+
+```
+mvn clean package -DskipTests -Pspark-1.6 -Dflink.version=1.1.2
+```
+
+`-DskipTests` skips build tests- you're not developing (yet), so you don't need to do tests, the clone version *should* build.
+
+`-Pspark-1.6` tells maven to build a Zeppelin with Spark 1.6. This is important because Zeppelin has its own Spark interpreter and the versions must be the same.
+
+`-Dflink.version=1.1.2` tells maven specifically to build Zeppelin with Flink version 1.1.2.
+
+
+**Note:** You may wish to include additional build flags such as `-Ppyspark` or `-Psparkr`. See [the build section of github for more details](https://github.com/apache/zeppelin#build).
+
+**Note:** You can build against any version of Spark that has a Zeppelin build profile available. The key is to make sure you check out the matching version of Spark to build. At the time of this writing, Spark 1.6 was the most recent Spark version available.
+
+**Note:** On build failures. Having installed Zeppelin close to 30 times now, I will tell you that sometimes the build fails for seemingly no reason.
+As long as you didn't edit any code, it is unlikely the build is failing because of something you did. What does tend to happen, is some dependency that maven is trying to download is unreachable. If your build fails on this step here are some tips:
+- Don't get discouraged.
+- Scroll up and read through the logs. There will be clues there.
+- Retry (that is, run the `mvn clean package -DskipTests -Pspark-1.6` again)
+- If there were clues that a dependency couldn't be downloaded wait a few hours or even days and retry again. Open source software when compiling is trying to download all of the dependencies it needs, if a server is off-line there is nothing you can do but wait for it to come back.
+- Make sure you followed all of the steps carefully.
+- Ask the community to help you. Go [here](http://zeppelin.apache.org/community.html) and join the user mailing list. People are there to help you. Make sure to copy and paste the build output (everything that happened in the console) and include that in your message.
+
+
+Start the Zeppelin daemon.
+
+```
+bin/zeppelin-daemon.sh start
+```
+
+Use `ifconfig` to determine the host machine's IP address. If you are not familiar with how to do this, a fairly comprehensive post can be found [here](http://www.cyberciti.biz/faq/how-to-find-out-the-ip-address-assigned-to-eth0-and-display-ip-only/).
+
+Open a web-browser on a machine connected to the same network as the host (or in the host operating system if using a virtual machine). Navigate to http://`yourip`:8080, where yourip is the IP address you found in `ifconfig`.
+
+See the [Zeppelin tutorial](../tutorial/tutorial.md) for basic Zeppelin usage. It is also advised that you take a moment to check out the tutorial notebook that is included with each Zeppelin install, and to familiarize yourself with basic notebook functionality.
+
+##### Flink Test
+Create a new notebook named "Flink Test" and copy and paste the following code.
+
+
+```scala
+
+%flink // let Zeppelin know what interpreter to use.
+
+val text = env.fromElements("In the time of chimpanzees, I was a monkey", // some lines of text to analyze
+"Butane in my veins and I'm out to cut the junkie",
+"With the plastic eyeballs, spray paint the vegetables",
+"Dog food stalls with the beefcake pantyhose",
+"Kill the headlights and put it in neutral",
+"Stock car flamin' with a loser in the cruise control",
+"Baby's in Reno with the Vitamin D",
+"Got a couple of couches, sleep on the love seat",
+"Someone came in sayin' I'm insane to complain",
+"About a shotgun wedding and a stain on my shirt",
+"Don't believe everything that you breathe",
+"You get a parking violation and a maggot on your sleeve",
+"So shave your face with some mace in the dark",
+"Savin' all your food stamps and burnin' down the trailer park",
+"Yo, cut it")
+
+/* The meat and potatoes:
+ this tells Flink to iterate through the elements, in this case strings,
+ transform the string to lower case and split the string at white space into individual words
+ then finally aggregate the occurrence of each word.
+
+ This creates the count variable which is a list of tuples of the form (word, occurances)
+
+counts.collect().foreach(println(_)) // execute the script and print each element in the counts list
+
+*/
+val counts = text.flatMap{ _.toLowerCase.split("\\W+") }.map { (_,1) }.groupBy(0).sum(1)
+
+counts.collect().foreach(println(_)) // execute the script and print each element in the counts list
+
+```
+
+Run the code to make sure the built-in Zeppelin Flink interpreter is working properly.
+
+##### Spark Test
+Create a new notebook named "Spark Test" and copy and paste the following code.
+
+```scala
+%spark // let Zeppelin know what interpreter to use.
+
+val text = sc.parallelize(List("In the time of chimpanzees, I was a monkey", // some lines of text to analyze
+"Butane in my veins and I'm out to cut the junkie",
+"With the plastic eyeballs, spray paint the vegetables",
+"Dog food stalls with the beefcake pantyhose",
+"Kill the headlights and put it in neutral",
+"Stock car flamin' with a loser in the cruise control",
+"Baby's in Reno with the Vitamin D",
+"Got a couple of couches, sleep on the love seat",
+"Someone came in sayin' I'm insane to complain",
+"About a shotgun wedding and a stain on my shirt",
+"Don't believe everything that you breathe",
+"You get a parking violation and a maggot on your sleeve",
+"So shave your face with some mace in the dark",
+"Savin' all your food stamps and burnin' down the trailer park",
+"Yo, cut it"))
+
+
+/* The meat and potatoes:
+ this tells spark to iterate through the elements, in this case strings,
+ transform the string to lower case and split the string at white space into individual words
+ then finally aggregate the occurrence of each word.
+
+ This creates the count variable which is a list of tuples of the form (word, occurances)
+*/
+val counts = text.flatMap { _.toLowerCase.split("\\W+") }
+ .map { (_,1) }
+ .reduceByKey(_ + _)
+
+counts.collect().foreach(println(_)) // execute the script and print each element in the counts list
+```
+
+Run the code to make sure the built-in Zeppelin Flink interpreter is working properly.
+
+Finally, stop the Zeppelin daemon. From the command prompt run:
+
+```
+bin/zeppelin-daemon.sh stop
+```
+
+### Installing Clusters
+
+##### Flink Cluster
+
+###### Download Binaries
+
+Building from source is recommended where possible, for simplicity in this tutorial we will download Flink and Spark Binaries.
+
+To download the Flink Binary use `wget`
+
+```bash
+wget "http://mirror.cogentco.com/pub/apache/flink/flink-1.0.3/flink-1.0.3-bin-hadoop24-scala_2.10.tgz"
+tar -xzvf flink-1.0.3-bin-hadoop24-scala_2.10.tgz
+```
+
+This will download Flink 1.0.3, compatible with Hadoop 2.4. You do not have to install Hadoop for this binary to work, but if you are using Hadoop, please change `24` to your appropriate version.
+
+Start the Flink Cluster.
+
+```bash
+flink-1.0.3/bin/start-cluster.sh
+```
+
+###### Building From source
+
+If you wish to build Flink from source, the following will be instructive. Note that if you have downloaded and used the binary version this should be skipped. The changing nature of build tools and versions across platforms makes this section somewhat precarious. For example, Java8 and Maven 3.0.3 are recommended for building Flink, which are not recommended for Zeppelin at the time of writing. If the user wishes to attempt to build from source, this section will provide some reference. If errors are encountered, please contact the Apache Flink community.
+
+See the [Flink Installation guide](https://github.com/apache/flink/blob/master/README.md) for more detailed instructions.
+
+Return to the directory where you have been downloading, this tutorial assumes that is `$HOME`. Clone Flink, check out release-1.0, and build.
+
+```
+cd $HOME
+git clone https://github.com/apache/flink.git
+cd flink
+git checkout release-1.0
+mvn clean install -DskipTests
+```
+
+Start the Flink Cluster in stand-alone mode
+
+```
+build-target/bin/start-cluster.sh
+```
+
+###### Ensure the cluster is up
+
+In a browser, navigate to http://`yourip`:8082 to see the Flink Web-UI. Click on 'Task Managers' in the left navigation bar. Ensure there is at least one Task Manager present.
+
+