diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index 9f773d50add..76cb983e078 100644 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -66,7 +66,7 @@ zeppelin.interpreters - org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter + org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.jdbc.TajoInterpreter,org.apache.zeppelin.tajo.thrift.TajoInterpreter Comma separated interpreter configurations. First interpreter become a default diff --git a/pom.xml b/pom.xml index c1b6b94504f..66a884a371a 100644 --- a/pom.xml +++ b/pom.xml @@ -91,7 +91,8 @@ angular shell hive - tajo + tajo-jdbc + tajo-thrift zeppelin-web zeppelin-server zeppelin-distribution diff --git a/tajo/pom.xml b/tajo-jdbc/pom.xml similarity index 96% rename from tajo/pom.xml rename to tajo-jdbc/pom.xml index cbed17d3ba8..561b610dc76 100644 --- a/tajo/pom.xml +++ b/tajo-jdbc/pom.xml @@ -25,10 +25,10 @@ org.apache.zeppelin - zeppelin-tajo + zeppelin-tajo-jdbc jar 0.5.0-incubating-SNAPSHOT - Zeppelin: Tajo interpreter + Zeppelin: Tajo JDBC interpreter http://www.apache.org @@ -100,7 +100,7 @@ copy-dependencies - ${project.build.directory}/../../interpreter/tajo + ${project.build.directory}/../../interpreter/tajo-jdbc false false true @@ -114,7 +114,7 @@ copy - ${project.build.directory}/../../interpreter/tajo + ${project.build.directory}/../../interpreter/tajo-jdbc false false true diff --git a/tajo/src/main/java/org/apache/zeppelin/tajo/TajoInterpreter.java b/tajo-jdbc/src/main/java/org/apache/zeppelin/tajo/jdbc/TajoInterpreter.java similarity index 98% rename from tajo/src/main/java/org/apache/zeppelin/tajo/TajoInterpreter.java rename to tajo-jdbc/src/main/java/org/apache/zeppelin/tajo/jdbc/TajoInterpreter.java index d1979036f34..d9bdb792b37 100644 --- a/tajo/src/main/java/org/apache/zeppelin/tajo/TajoInterpreter.java +++ b/tajo-jdbc/src/main/java/org/apache/zeppelin/tajo/jdbc/TajoInterpreter.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.zeppelin.tajo; +package org.apache.zeppelin.tajo.jdbc; import java.sql.*; import java.util.List; @@ -45,8 +45,8 @@ public class TajoInterpreter extends Interpreter { static { Interpreter.register( - "tajo", - "tajo", + "tsqlj", + "tajo-jdbc", TajoInterpreter.class.getName(), new InterpreterPropertyBuilder() .add(TAJO_JDBC_URI, "jdbc:tajo://localhost:26002/default", "The URL for TajoServer.") diff --git a/tajo/src/test/java/org/apache/zeppelin/tajo/TajoInterpreterTest.java b/tajo-jdbc/src/test/java/org/apache/zeppelin/tajo/jdbc/TajoInterpreterTest.java similarity index 98% rename from tajo/src/test/java/org/apache/zeppelin/tajo/TajoInterpreterTest.java rename to tajo-jdbc/src/test/java/org/apache/zeppelin/tajo/jdbc/TajoInterpreterTest.java index abe1ca6bfbf..f8c50e20b0f 100644 --- a/tajo/src/test/java/org/apache/zeppelin/tajo/TajoInterpreterTest.java +++ b/tajo-jdbc/src/test/java/org/apache/zeppelin/tajo/jdbc/TajoInterpreterTest.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.zeppelin.tajo; +package org.apache.zeppelin.tajo.jdbc; import com.google.gson.JsonParseException; import org.apache.tajo.jdbc.TajoDriver; diff --git a/tajo/src/test/java/org/apache/zeppelin/tajo/TesterConnection.java b/tajo-jdbc/src/test/java/org/apache/zeppelin/tajo/jdbc/TesterConnection.java similarity index 99% rename from tajo/src/test/java/org/apache/zeppelin/tajo/TesterConnection.java rename to tajo-jdbc/src/test/java/org/apache/zeppelin/tajo/jdbc/TesterConnection.java index e0a68e2927f..0bcdd3f64a9 100644 --- a/tajo/src/test/java/org/apache/zeppelin/tajo/TesterConnection.java +++ b/tajo-jdbc/src/test/java/org/apache/zeppelin/tajo/jdbc/TesterConnection.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.zeppelin.tajo; +package org.apache.zeppelin.tajo.jdbc; import java.sql.*; diff --git a/tajo/src/test/java/org/apache/zeppelin/tajo/TesterDatabaseMetaData.java b/tajo-jdbc/src/test/java/org/apache/zeppelin/tajo/jdbc/TesterDatabaseMetaData.java similarity index 99% rename from tajo/src/test/java/org/apache/zeppelin/tajo/TesterDatabaseMetaData.java rename to tajo-jdbc/src/test/java/org/apache/zeppelin/tajo/jdbc/TesterDatabaseMetaData.java index b293991a737..5183db4d0b9 100644 --- a/tajo/src/test/java/org/apache/zeppelin/tajo/TesterDatabaseMetaData.java +++ b/tajo-jdbc/src/test/java/org/apache/zeppelin/tajo/jdbc/TesterDatabaseMetaData.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.zeppelin.tajo; +package org.apache.zeppelin.tajo.jdbc; import java.sql.Connection; import java.sql.DatabaseMetaData; diff --git a/tajo/src/test/java/org/apache/zeppelin/tajo/TesterResultSet.java b/tajo-jdbc/src/test/java/org/apache/zeppelin/tajo/jdbc/TesterResultSet.java similarity index 99% rename from tajo/src/test/java/org/apache/zeppelin/tajo/TesterResultSet.java rename to tajo-jdbc/src/test/java/org/apache/zeppelin/tajo/jdbc/TesterResultSet.java index dc174bed0f6..b6980e824ee 100644 --- a/tajo/src/test/java/org/apache/zeppelin/tajo/TesterResultSet.java +++ b/tajo-jdbc/src/test/java/org/apache/zeppelin/tajo/jdbc/TesterResultSet.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.zeppelin.tajo; +package org.apache.zeppelin.tajo.jdbc; import java.math.BigDecimal; import java.sql.*; diff --git a/tajo/src/test/java/org/apache/zeppelin/tajo/TesterResultSetMetaData.java b/tajo-jdbc/src/test/java/org/apache/zeppelin/tajo/jdbc/TesterResultSetMetaData.java similarity index 99% rename from tajo/src/test/java/org/apache/zeppelin/tajo/TesterResultSetMetaData.java rename to tajo-jdbc/src/test/java/org/apache/zeppelin/tajo/jdbc/TesterResultSetMetaData.java index d11e22fdc88..1e52b83b1a5 100644 --- a/tajo/src/test/java/org/apache/zeppelin/tajo/TesterResultSetMetaData.java +++ b/tajo-jdbc/src/test/java/org/apache/zeppelin/tajo/jdbc/TesterResultSetMetaData.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.zeppelin.tajo; +package org.apache.zeppelin.tajo.jdbc; import java.sql.ResultSetMetaData; import java.sql.SQLException; diff --git a/tajo/src/test/java/org/apache/zeppelin/tajo/TesterStatement.java b/tajo-jdbc/src/test/java/org/apache/zeppelin/tajo/jdbc/TesterStatement.java similarity index 99% rename from tajo/src/test/java/org/apache/zeppelin/tajo/TesterStatement.java rename to tajo-jdbc/src/test/java/org/apache/zeppelin/tajo/jdbc/TesterStatement.java index 22be68d252d..d12cc7c4796 100644 --- a/tajo/src/test/java/org/apache/zeppelin/tajo/TesterStatement.java +++ b/tajo-jdbc/src/test/java/org/apache/zeppelin/tajo/jdbc/TesterStatement.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.zeppelin.tajo; +package org.apache.zeppelin.tajo.jdbc; import java.sql.*; diff --git a/tajo/src/test/java/org/apache/zeppelin/tajo/TesterTajoInterpreter.java b/tajo-jdbc/src/test/java/org/apache/zeppelin/tajo/jdbc/TesterTajoInterpreter.java similarity index 96% rename from tajo/src/test/java/org/apache/zeppelin/tajo/TesterTajoInterpreter.java rename to tajo-jdbc/src/test/java/org/apache/zeppelin/tajo/jdbc/TesterTajoInterpreter.java index f0eacee7570..30f9e72a1c1 100644 --- a/tajo/src/test/java/org/apache/zeppelin/tajo/TesterTajoInterpreter.java +++ b/tajo-jdbc/src/test/java/org/apache/zeppelin/tajo/jdbc/TesterTajoInterpreter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.zeppelin.tajo; +package org.apache.zeppelin.tajo.jdbc; import java.sql.Connection; import java.sql.SQLException; diff --git a/tajo-thrift/README.md b/tajo-thrift/README.md new file mode 100644 index 00000000000..02a16662c1b --- /dev/null +++ b/tajo-thrift/README.md @@ -0,0 +1,10 @@ +# Tajo interpreter (thrift version) + +This is Tajo interpreter that is using thrift interface + +To build this branch, build Tajo thriftserver version https://github.com/jerryjung/tajo/tree/thriftserver first and install artifact into local maven repository (mvn install) +And then you can build Zeppelin. + +Tajo thrift driver keyword is %tsql + + diff --git a/tajo-thrift/pom.xml b/tajo-thrift/pom.xml new file mode 100644 index 00000000000..e6b3fa377fb --- /dev/null +++ b/tajo-thrift/pom.xml @@ -0,0 +1,166 @@ + + + + 4.0.0 + + + zeppelin + org.apache.zeppelin + 0.5.0-incubating-SNAPSHOT + + + org.apache.zeppelin + zeppelin-tajo-thrift + jar + 0.5.0-incubating-SNAPSHOT + Zeppelin: Tajo Thrift interpreter + http://zeppelin.incubator.apache.org + + + 2.5.0 + + + + + ${project.groupId} + zeppelin-interpreter + ${project.version} + provided + + + + org.apache.tajo + tajo-thrift-server + 0.11.0-SNAPSHOT + + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + + + + org.slf4j + slf4j-api + + + + org.slf4j + slf4j-log4j12 + + + + junit + junit + test + + + + + + + nflabs public repository + https://raw.github.com/NFLabs/mvn-repo/master/snapshots + + false + + + true + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.7 + + true + + + + + maven-enforcer-plugin + 1.3.1 + + + enforce + none + + + + + + org.apache.rat + apache-rat-plugin + + + none + + + + + + maven-dependency-plugin + 2.8 + + + copy-dependencies + package + + copy-dependencies + + + ${project.build.directory}/../../interpreter/tajo-thrift + false + false + true + runtime + + + + copy-artifact + package + + copy + + + ${project.build.directory}/../../interpreter/tajo-thrift + false + false + true + runtime + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${project.packaging} + + + + + + + + + + \ No newline at end of file diff --git a/tajo-thrift/src/main/java/org/apache/zeppelin/tajo/thrift/TajoInterpreter.java b/tajo-thrift/src/main/java/org/apache/zeppelin/tajo/thrift/TajoInterpreter.java new file mode 100644 index 00000000000..7e751617899 --- /dev/null +++ b/tajo-thrift/src/main/java/org/apache/zeppelin/tajo/thrift/TajoInterpreter.java @@ -0,0 +1,340 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.tajo.thrift; + +import java.io.IOException; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.thrift.ThriftServerConstants; +import org.apache.tajo.thrift.client.TajoThriftClient; +import org.apache.tajo.thrift.generated.TBriefQueryInfo; +import org.apache.tajo.thrift.generated.TColumn; +import org.apache.tajo.thrift.generated.TGetQueryStatusResponse; +import org.apache.tajo.thrift.generated.TSchema; +import org.apache.tajo.thrift.generated.TTableDesc; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.protobuf.ServiceException; + +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; + +/** + * Tajo interpreter implementation. with thrift protocol support + * https://github.com/jerryjung/tajo/tree/thriftserver + */ +public class TajoInterpreter extends Interpreter { + static { + Interpreter.register( + "tsqlt", + "tajo-thrift", + TajoInterpreter.class.getName(), + new InterpreterPropertyBuilder() + .add("tajo.thrift.server", + "localhost:" + ThriftServerConstants.DEFAULT_LISTEN_PORT, + "Tajo thrift server address, host:port") + .add("tajo.maxResult", + "1000", + "Maximum number of result to retreive") + .build()); + } + + Logger logger = LoggerFactory.getLogger(TajoInterpreter.class); + private TajoThriftClient tajoClient; + + + public TajoInterpreter(Properties property) { + super(property); + } + + @Override + public void open() { + TajoConf conf = new TajoConf(); + try { + tajoClient = new TajoThriftClient(conf, getProperty("tajo.thrift.server"), null); + } catch (IOException e) { + throw new InterpreterException(e); + } + } + + @Override + public void close() { + tajoClient.close(); + } + + private int getMaxResult() { + return Integer.parseInt(getProperty("tajo.maxResult")); + } + + private String getHelpMessage() { + StringBuilder builder = new StringBuilder(); + + builder.append("General\n"); + builder.append(" \\version\t\tshow Tajo version\n"); + builder.append(" \\?\t\tshow help\n"); + builder.append(" \\help\t\talias of \\?\n"); + builder.append("\n"); + builder.append("Informational\n"); + builder.append(" \\l\t\tlist databases\n"); + builder.append(" \\c\t\tshow current database\n"); + builder.append(" \\c [DBNAME]\t\tconnect to new database\n"); + builder.append(" \\d\t\tlist tables\n"); + builder.append(" \\d [TBNAME]\t\tdescribe tables\n"); + + return builder.toString(); + } + + private String listToOutput(List list) { + return listToOutput(list, ""); + } + + private String listToOutput(List list, String defaultValue) { + String out = ""; + if (list == null || list.size() == 0) return defaultValue; + for (String l : list) { + out += l + "\n"; + } + return out; + } + + @Override + public InterpreterResult interpret(String st, InterpreterContext context) { + ResultSet result; + + logger.info("interpret function"); + if (st != null && st.trim().startsWith("\\")) { + String tcmd = st.trim(); + try { + if (tcmd.equals("\\l")) { // list database + return new InterpreterResult( + Code.SUCCESS, + listToOutput(tajoClient.getAllDatabaseNames())); + } else if (tcmd.equals("\\c")) { // show current database + return new InterpreterResult( + Code.SUCCESS, + tajoClient.getCurrentDatabase()); + } else if (tcmd.startsWith("\\c")) { + String dbName = tcmd.split(" ")[1]; + return new InterpreterResult( + Code.SUCCESS, + (tajoClient.selectDatabase(dbName) ? + "connected to database " + dbName : + "failed")); + } else if (tcmd.equals("\\d")) { // list tables + return new InterpreterResult( + Code.SUCCESS, + listToOutput( + tajoClient.getTableList(tajoClient.getCurrentDatabase()), + "No Relation Found")); + } else if (tcmd.startsWith("\\d")) { // describe tables + String tableName = tcmd.split(" ")[1]; + TTableDesc tdesc = tajoClient.getTableDesc(tableName); + + if (tdesc == null) { + return new InterpreterResult( + Code.SUCCESS, + "no such a table:" + tableName); + } + + String out = ""; + out += "table name: " + tdesc.getTableName() + "\n"; + out += "table path: " + tdesc.getPath() + "\n"; + out += "store type: " + tdesc.getStoreType() + "\n"; + out += "number of rows: " + tdesc.getStats().getNumRows() + "\n"; + out += "volumne: " + tdesc.getStats().getNumBytes() + " B\n"; + out += "Options:\n"; + Map meta = tdesc.getTableMeta(); + if (meta != null) { + for (String k : meta.keySet()) { + out += k + "\t" + meta.get(k) + "\n"; + } + } + + out += "\n"; + out += "schema:\n"; + TSchema schema = tdesc.getSchema(); + for (TColumn col : schema.getColumns()) { + out += col.getSimpleName() + "\t" + col.getDataTypeName() + "\n"; + } + return new InterpreterResult( + Code.SUCCESS, + out); + } else if (tcmd.startsWith("\\set")) { + String[] set = tcmd.split(" "); + if (set.length != 3) { + return new InterpreterResult( + Code.SUCCESS, + "usage: \\set [[NAME] VALUE]"); + } + + String name = set[1]; + String value = set[2]; + + if (tajoClient.updateSessionVariable(name, value)) { + return new InterpreterResult( + Code.SUCCESS, + name + " = " + value); + } else { + return new InterpreterResult( + Code.ERROR, + "Error"); + } + } else if (tcmd.startsWith("\\unset")) { + String[] set = tcmd.split(" "); + if (set.length != 2) { + return new InterpreterResult( + Code.SUCCESS, + "usage: \\unset NAME"); + } + + String name = set[1]; + + if (tajoClient.unsetSessionVariable(name)) { + return new InterpreterResult( + Code.SUCCESS, + "unset " + name); + } else { + return new InterpreterResult( + Code.ERROR, + "Error"); + } + } else { + return new InterpreterResult( + Code.SUCCESS, + getHelpMessage()); + } + } catch (Exception e) { + throw new InterpreterException(e); + } + } + + try { + result = tajoClient.executeQueryAndGetResult(st); + // empty result + if (result == null) { + return new InterpreterResult(Code.SUCCESS, ""); + } + + String m = ""; + + // extract column info + ResultSetMetaData md = result.getMetaData(); + int numColumns = md.getColumnCount(); + + if (numColumns == 0) { + return new InterpreterResult(Code.SUCCESS, ""); + } + + for (int i = 1; i <= numColumns; i++) { + if (i != 1) { + m += "\t"; + } + m += md.getColumnName(i); + } + m += "\n"; + + int maxResult = getMaxResult(); + int currentRow = 0; + String extraMessage = ""; + + while (result.next()) { + + if (currentRow == maxResult) { + extraMessage = "\nResults are limited by " + maxResult + "."; + break; + } + currentRow++; + + for (int i = 1; i <= numColumns; i++) { + if (i != 1) { + m += "\t"; + } + + Object col = result.getObject(i); + if (col == null) { + m += "\t"; + } else { + m += col.toString(); + } + } + + m += "\n"; + } + + return new InterpreterResult(Code.SUCCESS, "%table " + m + extraMessage); + + } catch (ServiceException | IOException | SQLException e) { + logger.error("Error", e); + return new InterpreterResult(Code.ERROR, e.getMessage()); + } + + + } + + @Override + public void cancel(InterpreterContext context) { + } + + @Override + public FormType getFormType() { + return FormType.SIMPLE; + } + + @Override + public int getProgress(InterpreterContext context) { + + int result = 0; + logger.info("Getting progress information..."); + try { + List queryList = tajoClient.getQueryList(); + + logger.info("\t " + queryList.size() + " queries"); + if (queryList.isEmpty()) { + return result; + } + + String queryId = queryList.get(queryList.size() - 1).queryId; + TGetQueryStatusResponse query = tajoClient.getQueryStatus(queryId); + logger.info("Done! " + query); + result = (int) query.progress * 100; + + } catch (Exception e) { + logger.error("getProgress failed ", e); + } + return result; + } + + @Override + public List completion(String buf, int cursor) { + return new LinkedList(); + } + +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index bbf46fc88d4..ca36e46d6dd 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -389,7 +389,8 @@ public static enum ConfVars { + "org.apache.zeppelin.angular.AngularInterpreter," + "org.apache.zeppelin.shell.ShellInterpreter," + "org.apache.zeppelin.hive.HiveInterpreter," - + "org.apache.zeppelin.tajo.TajoInterpreter"), + + "org.apache.zeppelin.tajo.jdbc.TajoInterpreter," + + "org.apache.zeppelin.tajo.thrift.TajoInterpreter"), ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"), ZEPPELIN_ENCODING("zeppelin.encoding", "UTF-8"), ZEPPELIN_NOTEBOOK_DIR("zeppelin.notebook.dir", "notebook"),