From ebb0d7e5d62b2b9fec70756c7cd065262ae0301c Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Wed, 10 May 2017 14:02:28 +0200 Subject: [PATCH 01/18] WebHDFS Support --- docs/storage/storage.md | 26 ++ .../org/apache/zeppelin/file/HDFSCommand.java | 123 ++++++++-- .../zeppelin/file/HDFSFileInterpreter.java | 16 +- zeppelin-zengine/pom.xml | 12 + .../notebook/repo/HdfsNotebookRepo.java | 223 ++++++++++++++++++ .../org/apache/zeppelin/util/HdfsSite.java | 187 +++++++++++++++ 6 files changed, 556 insertions(+), 31 deletions(-) create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/HdfsNotebookRepo.java create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/util/HdfsSite.java diff --git a/docs/storage/storage.md b/docs/storage/storage.md index 9deca3dc75c..c99b135f9fb 100644 --- a/docs/storage/storage.md +++ b/docs/storage/storage.md @@ -33,6 +33,7 @@ There are few notebook storage systems available for a use out of the box: * storage using Amazon S3 service - `S3NotebookRepo` * storage using Azure service - `AzureNotebookRepo` * storage using MongoDB - `MongoNotebookRepo` + * storage using HDFS - `HdfsNotebookRepo` Multiple storage systems can be used at the same time by providing a comma-separated list of the class-names in the configuration. By default, only first two of them will be automatically kept in sync by Zeppelin. @@ -246,6 +247,31 @@ Optionally, you can specify Azure folder structure name in the file **zeppelin-s ```
+ +## Notebook Storage in Hdfs + + To enable your notebooks to be stored on HDFS - uncomment the next property in `zeppelin-site.xml` in order to use HdfsNotebookRepo class: + + ``` + + zeppelin.notebook.storage + org.apache.zeppelin.notebook.repo.HdfsNotebookRepo + notebook persistence layer implementation + + ``` + + and replace the notebook directory property below by an absolute HDFS location as follows : + ``` + + zeppelin.notebook.dir + hdfs://localhost:9000/tmp/notebook + path or URI for notebook persist + +``` + +
+ + ## Storage in ZeppelinHub ZeppelinHub storage layer allows out of the box connection of Zeppelin instance with your ZeppelinHub account. First of all, you need to either comment out the following property in **zeppelin-site.xml**: diff --git a/file/src/main/java/org/apache/zeppelin/file/HDFSCommand.java b/file/src/main/java/org/apache/zeppelin/file/HDFSCommand.java index 94508cd0364..5fa00c27988 100644 --- a/file/src/main/java/org/apache/zeppelin/file/HDFSCommand.java +++ b/file/src/main/java/org/apache/zeppelin/file/HDFSCommand.java @@ -18,12 +18,18 @@ package org.apache.zeppelin.file; -import java.net.URL; -import java.net.HttpURLConnection; +import org.slf4j.Logger; + +import javax.ws.rs.core.UriBuilder; import java.io.BufferedReader; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; import java.io.InputStreamReader; -import javax.ws.rs.core.UriBuilder; -import org.slf4j.Logger; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; /** * Definition and HTTP invocation methods for all WebHDFS commands @@ -36,7 +42,8 @@ public class HDFSCommand { */ public enum HttpType { GET, - PUT + PUT, + DELETE } /** @@ -76,6 +83,11 @@ public Arg(String key, String value) { // Define all the commands available public Op getFileStatus = new Op("GETFILESTATUS", HttpType.GET, 0); public Op listStatus = new Op("LISTSTATUS", HttpType.GET, 0); + public Op openFile = new Op("OPEN", HttpType.GET, 0); + public Op makeDirectory = new Op("MKDIRS", HttpType.PUT, 0); + public Op createWriteFile = new Op("CREATE", HttpType.PUT, 0); + public Op deleteFile = new Op("DELETE", HttpType.DELETE, 0); + public Op renameFile = new Op("RENAME", HttpType.PUT, 0); public HDFSCommand(String url, String user, Logger logger, int maxLength) { super(); @@ -102,9 +114,11 @@ public String checkArgs(Op op, String path, Arg[] args) throws Exception { } - // The operator that runs all commands - public String runCommand(Op op, String path, Arg[] args) throws Exception { + public String runCommand(Op op, String path, Arg[] args) throws Exception { + return runCommand(op, path, null, args); + } + public String runCommand(Op op, String path, byte[] argFile, Arg[] args) throws Exception { // Check arguments String error = checkArgs(op, path, args); if (error != null) { @@ -119,10 +133,20 @@ public String runCommand(Op op, String path, Arg[] args) throws Exception { .queryParam("op", op.op); if (args != null) { + boolean isUserName = false; for (Arg a : args) { builder = builder.queryParam(a.key, a.value); + if ("user.name".equals(a.key)) { + isUserName = true; + } + } + if (!isUserName) { + builder = builder.queryParam("user.name", this.user); } } + else { + builder = builder.queryParam("user.name", this.user); + } java.net.URI uri = builder.build(); // Connect and get response string @@ -131,26 +155,79 @@ public String runCommand(Op op, String path, Arg[] args) throws Exception { if (op.cmd == HttpType.GET) { con.setRequestMethod("GET"); + con.setInstanceFollowRedirects(true); + + String result = getReceivedResponse(con, HttpType.GET, hdfsUrl); + return result; + } else if (op.cmd == HttpType.PUT) { + con.setRequestMethod("PUT"); + con.setInstanceFollowRedirects(false); int responseCode = con.getResponseCode(); - logger.info("Sending 'GET' request to URL : " + hdfsUrl); - logger.info("Response Code : " + responseCode); + String result = getReceivedResponse(con, HttpType.PUT, hdfsUrl); - BufferedReader in = new BufferedReader( - new InputStreamReader(con.getInputStream())); - String inputLine; - StringBuffer response = new StringBuffer(); - - int i = 0; - while ((inputLine = in.readLine()) != null) { - if (inputLine.length() < maxLength) - response.append(inputLine); - i++; - if (i >= maxLength) - break; + if (responseCode == 307 && ("CREATE".equals(op.op) || "APPEND".equals(op.op))) { + String location = con.getHeaderField("Location"); + logger.debug("Redirect Location: " + location); + + hdfsUrl = new URL(location); + con = (HttpURLConnection) hdfsUrl.openConnection(); + + con.setRequestMethod("PUT"); + con.setRequestProperty("Content-Type", "application/octet-stream"); + con.setRequestProperty("Transfer-Encoding", "chunked"); + con.setDoOutput(true); + + DataOutputStream outputStream = new DataOutputStream(con.getOutputStream()); + outputStream.write(argFile); + outputStream.flush(); + + result = getReceivedResponse(con, HttpType.PUT, hdfsUrl); } - in.close(); - return response.toString(); + + return result; + } else if (op.cmd == HttpType.DELETE) { + con.setRequestMethod("DELETE"); + con.setDoInput(true); + con.setInstanceFollowRedirects(false); + return getReceivedResponse(con, HttpType.DELETE, hdfsUrl); } + return null; } + + private String getReceivedResponse(HttpURLConnection con, + HttpType type, URL url) throws IOException { + int responseCode = con.getResponseCode(); + + BufferedReader in; + if (responseCode == 200 || responseCode == 201 || responseCode == 307) { + logger.debug("Sending '{}' request to URL : {}", type.toString(), url); + logger.debug("Response Code : " + responseCode); + logger.debug("response message: " + con.getResponseMessage()); + in = new BufferedReader(new InputStreamReader(con.getInputStream())); + } else { + logger.info("Sending '{}' request to URL : {}", type.toString(), url); + logger.info("Response Code : " + responseCode); + logger.info("response message: " + con.getResponseMessage()); + in = new BufferedReader(new InputStreamReader(con.getErrorStream())); + } + String inputLine; + StringBuffer response = new StringBuffer(); + int i = 0; + while ((inputLine = in.readLine()) != null) { + if (inputLine.length() < maxLength) { + response.append(inputLine); + } + i++; + if (i >= maxLength) { + logger.warn("Input stream's length(" + inputLine.length() + + ") is greater than or equal to hdfs.maxlength(" + maxLength + + "). Please increase hdfs.maxlength in interpreter setting"); + break; + } + } + in.close(); + + return response.toString(); + } } diff --git a/file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java b/file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java index c4a173076ef..40aa8b2c62b 100644 --- a/file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java +++ b/file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java @@ -33,9 +33,9 @@ * */ public class HDFSFileInterpreter extends FileInterpreter { - static final String HDFS_URL = "hdfs.url"; - static final String HDFS_USER = "hdfs.user"; - static final String HDFS_MAXLENGTH = "hdfs.maxlength"; + public static final String HDFS_URL = "hdfs.url"; + public static final String HDFS_USER = "hdfs.user"; + public static final String HDFS_MAXLENGTH = "hdfs.maxlength"; Exception exceptionOnConnect = null; HDFSCommand cmd = null; @@ -214,8 +214,8 @@ public String listAll(String path) { AllFileStatus allFiles = gson.fromJson(sfs, AllFileStatus.class); if (allFiles != null && - allFiles.FileStatuses != null && - allFiles.FileStatuses.FileStatus != null) + allFiles.FileStatuses != null && + allFiles.FileStatuses.FileStatus != null) { for (OneFileStatus fs : allFiles.FileStatuses.FileStatus) all = all + listOne(path, fs) + '\n'; @@ -250,7 +250,7 @@ public boolean isDirectory(String path) { @Override public List completion(String buf, int cursor, - InterpreterContext interpreterContext) { + InterpreterContext interpreterContext) { logger.info("Completion request at position\t" + cursor + " in string " + buf); final List suggestions = new ArrayList<>(); if (StringUtils.isEmpty(buf)) { @@ -294,8 +294,8 @@ public List completion(String buf, int cursor, AllFileStatus allFiles = gson.fromJson(fileStatusString, AllFileStatus.class); if (allFiles != null && - allFiles.FileStatuses != null && - allFiles.FileStatuses.FileStatus != null) + allFiles.FileStatuses != null && + allFiles.FileStatuses.FileStatus != null) { for (OneFileStatus fs : allFiles.FileStatuses.FileStatus) { if (fs.pathSuffix.contains(unfinished)) { diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index 38b1e830068..2d831d07c46 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -59,6 +59,18 @@ ${project.version} + + ${project.groupId} + zeppelin-file + ${project.version} + + + javax.ws.rs + javax.ws.rs-api + + + + org.slf4j slf4j-api diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/HdfsNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/HdfsNotebookRepo.java new file mode 100644 index 00000000000..7accafaadb0 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/HdfsNotebookRepo.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.NoteInfo; +import org.apache.zeppelin.notebook.NotebookImportDeserializer; +import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.scheduler.Job.Status; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.apache.zeppelin.util.HdfsSite; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Date; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * + */ + +public class HdfsNotebookRepo implements NotebookRepo { + private static Logger logger = LoggerFactory.getLogger(HdfsNotebookRepo.class); + private HdfsSite hdfsSite; + private ZeppelinConfiguration conf; + private String rootDir; + + + public HdfsNotebookRepo(ZeppelinConfiguration conf) throws IOException { + this.conf = conf; + try { + rootDir = removeProtocol(conf.getNotebookDir()); + hdfsSite = new HdfsSite(conf); + hdfsSite.mkdirs(rootDir); + } catch (URISyntaxException e) { + throw new IOException(e); + } + } + + public String removeProtocol(String hdfsUrl) { + String newUrl = hdfsUrl.replaceAll("/$", ""); + if (newUrl.startsWith("hdfs://")) { + return "/" + newUrl.replaceAll("^hdfs://", "").split("/", 2)[1]; + } else { + return newUrl; + } + } + + + @Override + public List list(AuthenticationInfo subject) throws IOException { + String[] children = hdfsSite.listFiles(rootDir); + List infos = new LinkedList<>(); + for (String child : children) { + String fileName = child; + if (fileName.startsWith(".") + || fileName.startsWith("#") + || fileName.startsWith("~")) { + // skip hidden, temporary files + continue; + } + + if (!hdfsSite.isDirectory(rootDir + "/" + child)) { + // currently single note is saved like, [NOTE_ID]/note.json. + // so it must be a directory + continue; + } + + NoteInfo info = null; + + try { + info = getNoteInfo(rootDir + "/" + child); + if (info != null) { + infos.add(info); + } + } catch (Exception e) { + logger.error("Can't read note " + fileName, e); + } + } + + return infos; + } + + private Note getNote(String noteDir) throws IOException { + if (!hdfsSite.isDirectory(noteDir)) { + throw new IOException(noteDir + " is not a directory"); + } + + String noteJson = noteDir + "/" + "note.json"; + if (!hdfsSite.exists(noteJson)) { + throw new IOException(noteJson + " not found"); + } + + GsonBuilder gsonBuilder = new GsonBuilder(); + gsonBuilder.setPrettyPrinting(); + Gson gson = gsonBuilder.registerTypeAdapter(Date.class, new NotebookImportDeserializer()) + .create(); + + byte[] content = hdfsSite.readFile(noteJson); + String json = new String(content, conf.getString(ConfVars.ZEPPELIN_ENCODING)); + + Note note = gson.fromJson(json, Note.class); + + for (Paragraph p : note.getParagraphs()) { + if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) { + p.setStatus(Status.ABORT); + } + } + return note; + } + + private NoteInfo getNoteInfo(String noteDir) throws IOException { + Note note = getNote(noteDir); + return new NoteInfo(note); + } + + @Override + public Note get(String noteId, AuthenticationInfo subject) throws IOException { + String path = rootDir + "/" + noteId; + return getNote(path); + } + + protected String getRootDir() throws IOException { + if (!hdfsSite.exists(rootDir)) { + throw new IOException("Root path does not exists"); + } + + if (!hdfsSite.isDirectory(rootDir)) { + throw new IOException("Root path is not a directory"); + } + return rootDir; + } + + @Override + public synchronized void save(Note note, AuthenticationInfo subject) throws IOException { + GsonBuilder gsonBuilder = new GsonBuilder(); + gsonBuilder.setPrettyPrinting(); + Gson gson = gsonBuilder.create(); + String json = gson.toJson(note); + + String noteDir = rootDir + "/" + note.getId(); + + if (!hdfsSite.exists(noteDir)) { + hdfsSite.mkdirs(noteDir); + } + if (!hdfsSite.isDirectory(noteDir)) { + throw new IOException(noteDir + " is not a directory"); + } + + String noteJson = noteDir + "/" + "note.json"; + hdfsSite.writeFile(json.getBytes(conf.getString(ConfVars.ZEPPELIN_ENCODING)), noteJson); + } + + @Override + public void remove(String noteId, AuthenticationInfo subject) throws IOException { + String noteDir = rootDir + "/" + noteId; + + if (!hdfsSite.exists(noteDir)) { + throw new IOException("Can not remove " + noteDir); + } + hdfsSite.delete(noteDir); + } + + @Override + public void close() { + + } + + @Override + public Revision checkpoint(String noteId, String checkpointMsg, AuthenticationInfo subject) + throws IOException { + return null; + } + + @Override + public Note get(String noteId, String revId, AuthenticationInfo subject) throws IOException { + return null; + } + + @Override + public List revisionHistory(String noteId, AuthenticationInfo subject) { + return null; + } + + @Override + public Note setNoteRevision(String noteId, String revId, AuthenticationInfo subject) + throws IOException { + return null; + } + + @Override + public List getSettings(AuthenticationInfo subject) { + return null; + } + + @Override + public void updateSettings(Map settings, AuthenticationInfo subject) { + + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/HdfsSite.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/HdfsSite.java new file mode 100644 index 00000000000..cd5fbdb6191 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/HdfsSite.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.util; + +import com.google.gson.Gson; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.file.HDFSCommand; +import org.apache.zeppelin.file.HDFSFileInterpreter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.zeppelin.file.HDFSFileInterpreter.HDFS_MAXLENGTH; +import static org.apache.zeppelin.file.HDFSFileInterpreter.HDFS_URL; + +/** + * Class to create / move / delete / write to / read from HDFS filessytem + */ +public class HdfsSite { + private static Logger logger = LoggerFactory.getLogger(HdfsSite.class); + public static final String HDFS_NOTEBOOK_DIR = "hdfs.notebook.dir"; + static final String NOTE_JSON = "note.json"; + static final String NOTE_JSON_TEMP = "_note.json"; + + ZeppelinConfiguration conf; + boolean enableWebHDFS = true; + String hdfsUrl = null; + String hdfsUser = null; + int hdfsMaxLength = 0; + HDFSCommand hdfsCmd = null; + + + public HdfsSite(ZeppelinConfiguration conf) throws URISyntaxException { + this.conf = conf; + this.hdfsUrl = conf.getString(HDFS_URL, HDFS_URL, "http://localhost:50070/webhdfs/v1/"); + this.hdfsMaxLength = conf.getInt(HDFS_URL, HDFS_MAXLENGTH, 100000); + this.hdfsUser = System.getenv("HADOOP_USER_NAME"); + if (this.hdfsUser == null) { + this.hdfsUser = System.getenv("LOGNAME"); + } + if (this.hdfsUser == null) { + + this.enableWebHDFS = false; + } else { + this.hdfsCmd = new HDFSCommand(hdfsUrl, hdfsUser, logger, this.hdfsMaxLength); + this.enableWebHDFS = exists("/"); + } + } + + public boolean exists(String path) { + boolean ret = false; + HDFSFileInterpreter.SingleFileStatus fileStatus; + Gson gson = new Gson(); + try { + String notebookStatus = this.hdfsCmd.runCommand(this.hdfsCmd.getFileStatus, path, null); + fileStatus = gson.fromJson(notebookStatus, HDFSFileInterpreter.SingleFileStatus.class); + ret = fileStatus.FileStatus.modificationTime > 0; + } catch (Exception e) { + logger.info("disabled webHDFS. Please check webhdfs configurations"); + ret = false; + } finally { + return ret; + } + } + + + public String[] listFiles(String directory) throws IOException { + List hdfsNotebook = new ArrayList(); + Gson gson = new Gson(); + String hdfsDirStatus; + + try { + hdfsDirStatus = this.hdfsCmd.runCommand(this.hdfsCmd.listStatus, directory, null); + if (hdfsDirStatus != null) { + HDFSFileInterpreter.AllFileStatus allFiles = gson.fromJson(hdfsDirStatus, + HDFSFileInterpreter.AllFileStatus.class); + if (allFiles != null && allFiles.FileStatuses != null + && allFiles.FileStatuses.FileStatus != null) { + for (HDFSFileInterpreter.OneFileStatus fs : allFiles.FileStatuses.FileStatus) { + if ("DIRECTORY".equals(fs.type) && fs.pathSuffix.startsWith("_") == false) { + hdfsNotebook.add(fs.pathSuffix); + logger.info("read a notebook from HDFS: " + fs.pathSuffix); + } + } + } + } + } catch (Exception e) { + logger.error("exception occurred during getting notebook from hdfs : ", e); + } + return hdfsNotebook.toArray(new String[0]); + } + + public void delete(String path) throws IOException { + logger.debug("remove : " + path); + + HDFSCommand.Arg recursive = this.hdfsCmd.new Arg("recursive", "true"); + HDFSCommand.Arg[] args = {recursive}; + + try { + this.hdfsCmd.runCommand(this.hdfsCmd.deleteFile, path, args); + } catch (Exception e) { + logger.error("Exception: ", e); + throw new IOException(e.getCause()); + } + } + + public void mkdirs(String path) throws IOException { + + try { + this.hdfsCmd.runCommand(this.hdfsCmd.makeDirectory, path, null); + } catch (Exception e) { + logger.error("Exception: ", e); + throw new IOException(e.getCause()); + } + } + + + public void writeFile(byte[] content, String path) throws IOException { + try { + HDFSCommand.Arg dest = this.hdfsCmd.new Arg("overwrite", "true"); + HDFSCommand.Arg[] createArgs = {dest}; + this.hdfsCmd.runCommand(this.hdfsCmd.createWriteFile, path, content, createArgs); + } catch (Exception e) { + logger.error("Exception: ", e); + throw new IOException(e.getCause()); + } + } + + public void rename(String oldPath, String newPath) throws IOException { + try { + HDFSCommand.Arg dest = this.hdfsCmd.new Arg("destination", newPath); + HDFSCommand.Arg[] renameArgs = {dest}; + this.hdfsCmd.runCommand(this.hdfsCmd.renameFile, oldPath, renameArgs); + } catch (Exception e) { + logger.error("Exception: ", e); + throw new IOException(e.getCause()); + } finally { + } + } + + public boolean isDirectory(String path) throws IOException { + boolean ret = false; + HDFSFileInterpreter.SingleFileStatus fileStatus; + Gson gson = new Gson(); + try { + String notebookStatus = this.hdfsCmd.runCommand(this.hdfsCmd.getFileStatus, path, null); + fileStatus = gson.fromJson(notebookStatus, HDFSFileInterpreter.SingleFileStatus.class); + ret = fileStatus.FileStatus.type.equals("DIRECTORY"); + } catch (Exception e) { + logger.info("disabled webHDFS. Please check webhdfs configurations"); + ret = false; + } finally { + return ret; + } + } + + public byte[] readFile(String path) throws IOException { + byte[] res = new byte[0]; + try { + res = this.hdfsCmd.runCommand(this.hdfsCmd.openFile, path, null, null).getBytes(); + } catch (Exception e) { + logger.error("Exception: ", e); + throw new IOException(e.getCause()); + } finally { + return res; + } + } +} From 47cb49ff3e8c3954b2069f4221e6c819ddd7ddeb Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 11 May 2017 09:49:00 +0200 Subject: [PATCH 02/18] Update WEB HDFS doc --- docs/storage/storage.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/storage/storage.md b/docs/storage/storage.md index c99b135f9fb..e0d1ba033cc 100644 --- a/docs/storage/storage.md +++ b/docs/storage/storage.md @@ -258,13 +258,32 @@ Optionally, you can specify Azure folder structure name in the file **zeppelin-s org.apache.zeppelin.notebook.repo.HdfsNotebookRepo notebook persistence layer implementation + + + hdfs.url + http://localhost:50070/webhdfs/v1/ + HDFS url + + + + hdfs.user + hdfs + HDFS user + + + + hdfs.maxlength + 1000 + Maximum number of lines of results fetched + + ``` and replace the notebook directory property below by an absolute HDFS location as follows : ``` zeppelin.notebook.dir - hdfs://localhost:9000/tmp/notebook + /tmp/notebook path or URI for notebook persist ``` From a8e78947380c89dc6573cfe0b6d4d3a596d97404 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 11 May 2017 10:08:53 +0200 Subject: [PATCH 03/18] Move HDFSCommand to zeppelin-interpreter --- file/pom.xml | 7 ------- .../main/java/org/apache/zeppelin/file/HDFSCommand.java | 3 --- 2 files changed, 10 deletions(-) rename {file => zeppelin-interpreter}/src/main/java/org/apache/zeppelin/file/HDFSCommand.java (98%) diff --git a/file/pom.xml b/file/pom.xml index 2493c1fae4c..a987e26fe1a 100644 --- a/file/pom.xml +++ b/file/pom.xml @@ -33,7 +33,6 @@ - 2.0 2.22.2 @@ -48,12 +47,6 @@ provided - - javax.ws.rs - javax.ws.rs-api - ${ws.rsapi.version} - - org.slf4j slf4j-api diff --git a/file/src/main/java/org/apache/zeppelin/file/HDFSCommand.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSCommand.java similarity index 98% rename from file/src/main/java/org/apache/zeppelin/file/HDFSCommand.java rename to zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSCommand.java index 5fa00c27988..dd886edc1b9 100644 --- a/file/src/main/java/org/apache/zeppelin/file/HDFSCommand.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSCommand.java @@ -23,11 +23,8 @@ import javax.ws.rs.core.UriBuilder; import java.io.BufferedReader; import java.io.DataOutputStream; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URL; From 197a55921c3c3f27154cf6fad42e5eb336905418 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 11 May 2017 10:09:48 +0200 Subject: [PATCH 04/18] Update zeppelin-interpreter to include UriBuilder dependency --- zeppelin-interpreter/pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml index 109099cfc7b..5bd3dc70e32 100644 --- a/zeppelin-interpreter/pom.xml +++ b/zeppelin-interpreter/pom.xml @@ -44,6 +44,7 @@ 3.0.3 1.0 2.12.1 + 2.0 2.3 @@ -83,6 +84,12 @@ commons-lang + + javax.ws.rs + javax.ws.rs-api + ${ws.rsapi.version} + + org.slf4j slf4j-api From 6ce08a5c4a73f263776ac947beb11669c368afc8 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 11 May 2017 10:43:34 +0200 Subject: [PATCH 05/18] Update javax.ws milestone version from 2.0-m10 to release version 2.0 to avoid conflicts --- zeppelin-server/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index ac5cb2b1256..eed64c99f61 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -38,7 +38,7 @@ 2.6.0 2.2.1 1.13 - 2.0-m10 + 2.0 1.8 4.1.0 From 6f6ec73e8d32a9fade1e2805309db0296894cd3c Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 11 May 2017 11:58:39 +0200 Subject: [PATCH 06/18] remove zengine / file dependency --- .../zeppelin/file/HDFSFileInterpreter.java | 86 +++---------------- .../org/apache/zeppelin/file/HDFSStatus.java | 75 ++++++++++++++++ zeppelin-zengine/pom.xml | 12 --- .../org/apache/zeppelin/util/HdfsSite.java | 22 ++--- 4 files changed, 98 insertions(+), 97 deletions(-) create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSStatus.java diff --git a/file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java b/file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java index 40aa8b2c62b..05f488d1ae1 100644 --- a/file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java +++ b/file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java @@ -53,72 +53,7 @@ public HDFSFileInterpreter(Properties property){ super(property); prepare(); } - - /** - * Status of one file - * - * matches returned JSON - */ - public class OneFileStatus { - public long accessTime; - public int blockSize; - public int childrenNum; - public int fileId; - public String group; - public long length; - public long modificationTime; - public String owner; - public String pathSuffix; - public String permission; - public int replication; - public int storagePolicy; - public String type; - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("\nAccessTime = ").append(accessTime); - sb.append("\nBlockSize = ").append(blockSize); - sb.append("\nChildrenNum = ").append(childrenNum); - sb.append("\nFileId = ").append(fileId); - sb.append("\nGroup = ").append(group); - sb.append("\nLength = ").append(length); - sb.append("\nModificationTime = ").append(modificationTime); - sb.append("\nOwner = ").append(owner); - sb.append("\nPathSuffix = ").append(pathSuffix); - sb.append("\nPermission = ").append(permission); - sb.append("\nReplication = ").append(replication); - sb.append("\nStoragePolicy = ").append(storagePolicy); - sb.append("\nType = ").append(type); - return sb.toString(); - } - } - - /** - * Status of one file - * - * matches returned JSON - */ - public class SingleFileStatus { - public OneFileStatus FileStatus; - } - - /** - * Status of all files in a directory - * - * matches returned JSON - */ - public class MultiFileStatus { - public OneFileStatus[] FileStatus; - } - - /** - * Status of all files in a directory - * - * matches returned JSON - */ - public class AllFileStatus { - public MultiFileStatus FileStatuses; - } - + // tests whether we're able to connect to HDFS private void testConnection() { @@ -144,7 +79,7 @@ private String listDir(String path) throws Exception { return cmd.runCommand(cmd.listStatus, path, null); } - private String listPermission(OneFileStatus fs){ + private String listPermission(HDFSStatus.OneFileStatus fs){ StringBuilder sb = new StringBuilder(); sb.append(fs.type.equalsIgnoreCase("Directory") ? 'd' : '-'); int p = Integer.parseInt(fs.permission, 16); @@ -159,10 +94,10 @@ private String listPermission(OneFileStatus fs){ sb.append(((p & 0x1) == 0) ? '-' : 'x'); return sb.toString(); } - private String listDate(OneFileStatus fs) { + private String listDate(HDFSStatus.OneFileStatus fs) { return new SimpleDateFormat("yyyy-MM-dd HH:mm").format(new Date(fs.modificationTime)); } - private String listOne(String path, OneFileStatus fs) { + private String listOne(String path, HDFSStatus.OneFileStatus fs) { if (args.flags.contains(new Character('l'))) { StringBuilder sb = new StringBuilder(); sb.append(listPermission(fs) + "\t"); @@ -192,7 +127,7 @@ private String humanReadableByteCount(long bytes) { public String listFile(String filePath) { try { String str = cmd.runCommand(cmd.getFileStatus, filePath, null); - SingleFileStatus sfs = gson.fromJson(str, SingleFileStatus.class); + HDFSStatus.SingleFileStatus sfs = gson.fromJson(str, HDFSStatus.SingleFileStatus.class); if (sfs != null) { return listOne(filePath, sfs.FileStatus); } @@ -211,13 +146,13 @@ public String listAll(String path) { if (isDirectory(path)) { String sfs = listDir(path); if (sfs != null) { - AllFileStatus allFiles = gson.fromJson(sfs, AllFileStatus.class); + HDFSStatus.AllFileStatus allFiles = gson.fromJson(sfs, HDFSStatus.AllFileStatus.class); if (allFiles != null && allFiles.FileStatuses != null && allFiles.FileStatuses.FileStatus != null) { - for (OneFileStatus fs : allFiles.FileStatuses.FileStatus) + for (HDFSStatus.OneFileStatus fs : allFiles.FileStatuses.FileStatus) all = all + listOne(path, fs) + '\n'; } } @@ -237,7 +172,7 @@ public boolean isDirectory(String path) { return ret; try { String str = cmd.runCommand(cmd.getFileStatus, path, null); - SingleFileStatus sfs = gson.fromJson(str, SingleFileStatus.class); + HDFSStatus.SingleFileStatus sfs = gson.fromJson(str, HDFSStatus.SingleFileStatus.class); if (sfs != null) return sfs.FileStatus.type.equals("DIRECTORY"); } catch (Exception e) { @@ -291,13 +226,14 @@ public List completion(String buf, int cursor, try { String fileStatusString = listDir(globalPath); if (fileStatusString != null) { - AllFileStatus allFiles = gson.fromJson(fileStatusString, AllFileStatus.class); + HDFSStatus.AllFileStatus allFiles = gson.fromJson(fileStatusString, + HDFSStatus.AllFileStatus.class); if (allFiles != null && allFiles.FileStatuses != null && allFiles.FileStatuses.FileStatus != null) { - for (OneFileStatus fs : allFiles.FileStatuses.FileStatus) { + for (HDFSStatus.OneFileStatus fs : allFiles.FileStatuses.FileStatus) { if (fs.pathSuffix.contains(unfinished)) { //only suggest the text after the last . diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSStatus.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSStatus.java new file mode 100644 index 00000000000..f2dd1108a72 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSStatus.java @@ -0,0 +1,75 @@ +package org.apache.zeppelin.file; + +/** + * Created by hayssams on 11/05/2017. + */ +public class HDFSStatus { + /** + * Status of one file + * + * matches returned JSON + */ + public class OneFileStatus { + public long accessTime; + public int blockSize; + public int childrenNum; + public int fileId; + public String group; + public long length; + public long modificationTime; + public String owner; + public String pathSuffix; + public String permission; + public int replication; + public int storagePolicy; + public String type; + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("\nAccessTime = ").append(accessTime); + sb.append("\nBlockSize = ").append(blockSize); + sb.append("\nChildrenNum = ").append(childrenNum); + sb.append("\nFileId = ").append(fileId); + sb.append("\nGroup = ").append(group); + sb.append("\nLength = ").append(length); + sb.append("\nModificationTime = ").append(modificationTime); + sb.append("\nOwner = ").append(owner); + sb.append("\nPathSuffix = ").append(pathSuffix); + sb.append("\nPermission = ").append(permission); + sb.append("\nReplication = ").append(replication); + sb.append("\nStoragePolicy = ").append(storagePolicy); + sb.append("\nType = ").append(type); + return sb.toString(); + } + } + + /** + * Status of one file + * + * matches returned JSON + */ + public class SingleFileStatus { + public OneFileStatus FileStatus; + } + + /** + * Status of all files in a directory + * + * matches returned JSON + */ + public class MultiFileStatus { + public OneFileStatus[] FileStatus; + } + + /** + * Status of all files in a directory + * + * matches returned JSON + */ + public class AllFileStatus { + public MultiFileStatus FileStatuses; + } + + +} + + diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index 2d831d07c46..38b1e830068 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -59,18 +59,6 @@ ${project.version} - - ${project.groupId} - zeppelin-file - ${project.version} - - - javax.ws.rs - javax.ws.rs-api - - - - org.slf4j slf4j-api diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/HdfsSite.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/HdfsSite.java index cd5fbdb6191..daae473db14 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/HdfsSite.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/HdfsSite.java @@ -20,7 +20,7 @@ import com.google.gson.Gson; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.file.HDFSCommand; -import org.apache.zeppelin.file.HDFSFileInterpreter; +import org.apache.zeppelin.file.HDFSStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,13 +29,15 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.zeppelin.file.HDFSFileInterpreter.HDFS_MAXLENGTH; -import static org.apache.zeppelin.file.HDFSFileInterpreter.HDFS_URL; /** * Class to create / move / delete / write to / read from HDFS filessytem */ public class HdfsSite { + public static final String HDFS_URL = "hdfs.url"; + public static final String HDFS_USER = "hdfs.user"; + public static final String HDFS_MAXLENGTH = "hdfs.maxlength"; + private static Logger logger = LoggerFactory.getLogger(HdfsSite.class); public static final String HDFS_NOTEBOOK_DIR = "hdfs.notebook.dir"; static final String NOTE_JSON = "note.json"; @@ -68,11 +70,11 @@ public HdfsSite(ZeppelinConfiguration conf) throws URISyntaxException { public boolean exists(String path) { boolean ret = false; - HDFSFileInterpreter.SingleFileStatus fileStatus; + HDFSStatus.SingleFileStatus fileStatus; Gson gson = new Gson(); try { String notebookStatus = this.hdfsCmd.runCommand(this.hdfsCmd.getFileStatus, path, null); - fileStatus = gson.fromJson(notebookStatus, HDFSFileInterpreter.SingleFileStatus.class); + fileStatus = gson.fromJson(notebookStatus, HDFSStatus.SingleFileStatus.class); ret = fileStatus.FileStatus.modificationTime > 0; } catch (Exception e) { logger.info("disabled webHDFS. Please check webhdfs configurations"); @@ -91,11 +93,11 @@ public String[] listFiles(String directory) throws IOException { try { hdfsDirStatus = this.hdfsCmd.runCommand(this.hdfsCmd.listStatus, directory, null); if (hdfsDirStatus != null) { - HDFSFileInterpreter.AllFileStatus allFiles = gson.fromJson(hdfsDirStatus, - HDFSFileInterpreter.AllFileStatus.class); + HDFSStatus.AllFileStatus allFiles = gson.fromJson(hdfsDirStatus, + HDFSStatus.AllFileStatus.class); if (allFiles != null && allFiles.FileStatuses != null && allFiles.FileStatuses.FileStatus != null) { - for (HDFSFileInterpreter.OneFileStatus fs : allFiles.FileStatuses.FileStatus) { + for (HDFSStatus.OneFileStatus fs : allFiles.FileStatuses.FileStatus) { if ("DIRECTORY".equals(fs.type) && fs.pathSuffix.startsWith("_") == false) { hdfsNotebook.add(fs.pathSuffix); logger.info("read a notebook from HDFS: " + fs.pathSuffix); @@ -159,11 +161,11 @@ public void rename(String oldPath, String newPath) throws IOException { public boolean isDirectory(String path) throws IOException { boolean ret = false; - HDFSFileInterpreter.SingleFileStatus fileStatus; + HDFSStatus.SingleFileStatus fileStatus; Gson gson = new Gson(); try { String notebookStatus = this.hdfsCmd.runCommand(this.hdfsCmd.getFileStatus, path, null); - fileStatus = gson.fromJson(notebookStatus, HDFSFileInterpreter.SingleFileStatus.class); + fileStatus = gson.fromJson(notebookStatus, HDFSStatus.SingleFileStatus.class); ret = fileStatus.FileStatus.type.equals("DIRECTORY"); } catch (Exception e) { logger.info("disabled webHDFS. Please check webhdfs configurations"); From 4e1a05b65fd5091075d67935e71b75333c046121 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 11 May 2017 14:24:32 +0200 Subject: [PATCH 07/18] Add Licence --- .../org/apache/zeppelin/file/HDFSStatus.java | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSStatus.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSStatus.java index f2dd1108a72..939eef776a7 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSStatus.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSStatus.java @@ -1,8 +1,23 @@ -package org.apache.zeppelin.file; - /** - * Created by hayssams on 11/05/2017. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ + +package org.apache.zeppelin.file; + public class HDFSStatus { /** * Status of one file From 5f268d4fdc4d5d71e93edf22ee555b8b5afe679e Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 11 May 2017 17:18:30 +0200 Subject: [PATCH 08/18] add class comment --- .../src/main/java/org/apache/zeppelin/file/HDFSStatus.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSStatus.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSStatus.java index 939eef776a7..2674d8bf222 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSStatus.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSStatus.java @@ -18,6 +18,9 @@ package org.apache.zeppelin.file; +/** + * Class that hosts HDFS file statuses + */ public class HDFSStatus { /** * Status of one file From 74a42563b3d1f736ed2efddafc1a04bbeec466b8 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 11 May 2017 19:06:40 +0200 Subject: [PATCH 09/18] rollback javax.ws to 2.0-m10 --- zeppelin-interpreter/pom.xml | 2 +- zeppelin-server/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml index 5bd3dc70e32..2665e5561ba 100644 --- a/zeppelin-interpreter/pom.xml +++ b/zeppelin-interpreter/pom.xml @@ -44,7 +44,7 @@ 3.0.3 1.0 2.12.1 - 2.0 + 2.0-m10 2.3 diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index eed64c99f61..ac5cb2b1256 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -38,7 +38,7 @@ 2.6.0 2.2.1 1.13 - 2.0 + 2.0-m10 1.8 4.1.0 From a77c1dbd7e7cae38bbdf1efd3d109ed0696cffa0 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Fri, 19 May 2017 07:55:13 +0200 Subject: [PATCH 10/18] Update java.ws.api version --- zeppelin-interpreter/pom.xml | 2 +- zeppelin-server/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml index 2665e5561ba..58e8313d178 100644 --- a/zeppelin-interpreter/pom.xml +++ b/zeppelin-interpreter/pom.xml @@ -44,7 +44,7 @@ 3.0.3 1.0 2.12.1 - 2.0-m10 + 2.0.1 2.3 diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index ac5cb2b1256..2371601c5be 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -38,7 +38,7 @@ 2.6.0 2.2.1 1.13 - 2.0-m10 + 2.0.1 1.8 4.1.0 From 6dc80ef0df8567d72d793cf1482e27b41699c618 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Sat, 20 May 2017 07:27:44 +0200 Subject: [PATCH 11/18] revert javax rs ws version change --- zeppelin-interpreter/pom.xml | 4 ++-- zeppelin-server/pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml index 58e8313d178..4ed1cf447f5 100644 --- a/zeppelin-interpreter/pom.xml +++ b/zeppelin-interpreter/pom.xml @@ -44,7 +44,7 @@ 3.0.3 1.0 2.12.1 - 2.0.1 + 2.0-m10 2.3 @@ -89,7 +89,7 @@ javax.ws.rs-api ${ws.rsapi.version} - + org.slf4j slf4j-api diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index 2371601c5be..ac5cb2b1256 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -38,7 +38,7 @@ 2.6.0 2.2.1 1.13 - 2.0.1 + 2.0-m10 1.8 4.1.0 From 62a4c9197f2178d37f43854fede9275b1b5f04a2 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Sat, 20 May 2017 08:01:13 +0200 Subject: [PATCH 12/18] remove unnecessary dependency --- file/pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/file/pom.xml b/file/pom.xml index a987e26fe1a..dab02cddf61 100644 --- a/file/pom.xml +++ b/file/pom.xml @@ -57,12 +57,6 @@ slf4j-log4j12 - - org.glassfish.jersey.core - jersey-common - ${jersey.common.version} - - junit junit From 981dde49c4cbd6729f876bfa1e2594f2cbdc79b2 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Tue, 23 May 2017 17:45:10 +0200 Subject: [PATCH 13/18] Uber HTTP Components to build URI --- file/pom.xml | 6 +++ zeppelin-interpreter/pom.xml | 7 +++ .../org/apache/zeppelin/file/HDFSCommand.java | 48 +++++++++++-------- 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/file/pom.xml b/file/pom.xml index dab02cddf61..3939401d502 100644 --- a/file/pom.xml +++ b/file/pom.xml @@ -48,6 +48,12 @@ + org.glassfish.jersey.core + jersey-common + ${jersey.common.version} + + + org.slf4j slf4j-api diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml index 4ed1cf447f5..3f9279f6581 100644 --- a/zeppelin-interpreter/pom.xml +++ b/zeppelin-interpreter/pom.xml @@ -45,6 +45,7 @@ 1.0 2.12.1 2.0-m10 + 4.3.6 2.3 @@ -226,6 +227,12 @@ guava + + org.apache.httpcomponents + httpclient + ${httpcomponents.client.version} + + junit junit diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSCommand.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSCommand.java index dd886edc1b9..ed58ac306e9 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSCommand.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSCommand.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,6 +18,7 @@ package org.apache.zeppelin.file; +import org.apache.http.client.utils.URIBuilder; import org.slf4j.Logger; import javax.ws.rs.core.UriBuilder; @@ -30,7 +31,6 @@ /** * Definition and HTTP invocation methods for all WebHDFS commands - * */ public class HDFSCommand { @@ -99,8 +99,7 @@ public String checkArgs(Op op, String path, Arg[] args) throws Exception { path == null || (op.minArgs > 0 && (args == null || - args.length != op.minArgs))) - { + args.length != op.minArgs))) { String a = ""; a = (op != null) ? a + op.op + "\n" : a; a = (path != null) ? a + path + "\n" : a; @@ -111,11 +110,11 @@ public String checkArgs(Op op, String path, Arg[] args) throws Exception { } - public String runCommand(Op op, String path, Arg[] args) throws Exception { + public String runCommand(Op op, String path, Arg[] args) throws Exception { return runCommand(op, path, null, args); } - public String runCommand(Op op, String path, byte[] argFile, Arg[] args) throws Exception { + public String runCommand(Op op, String path, byte[] argFile, Arg[] args) throws Exception { // Check arguments String error = checkArgs(op, path, args); if (error != null) { @@ -123,31 +122,41 @@ public String runCommand(Op op, String path, byte[] argFile, Arg[] args) throws return "ERROR: BAD ARGS"; } + // Build URI - UriBuilder builder = UriBuilder - .fromPath(url) - .path(path) - .queryParam("op", op.op); + String finalUrl = url; + if (url.endsWith("/") && path.startsWith("/")) + finalUrl += path.substring(1); + else + finalUrl += path; + + URIBuilder uriBuilder = new URIBuilder(finalUrl) + .addParameter("op", op.op) + .addParameter("user", this.user); if (args != null) { boolean isUserName = false; for (Arg a : args) { - builder = builder.queryParam(a.key, a.value); +// builder = builder.queryParam(a.key, a.value); + uriBuilder.addParameter(a.key, a.value); + //System.out.println("a.key/a.value=" + a.key + "=" + a.value); if ("user.name".equals(a.key)) { isUserName = true; } } if (!isUserName) { - builder = builder.queryParam("user.name", this.user); + //builder = builder.queryParam("user.name", this.user); + uriBuilder.addParameter("user.name", this.user); } + } else { + // builder = builder.queryParam("user.name", this.user); + uriBuilder.addParameter("user.name", this.user); } - else { - builder = builder.queryParam("user.name", this.user); - } - java.net.URI uri = builder.build(); - + java.net.URI uri = uriBuilder.build(); + // System.out.println("resUrl=" + resUrl); // Connect and get response string URL hdfsUrl = uri.toURL(); + // System.out.println("hdfsUrl=" + hdfsUrl.toExternalForm()); HttpURLConnection con = (HttpURLConnection) hdfsUrl.openConnection(); if (op.cmd == HttpType.GET) { @@ -188,7 +197,6 @@ public String runCommand(Op op, String path, byte[] argFile, Arg[] args) throws con.setInstanceFollowRedirects(false); return getReceivedResponse(con, HttpType.DELETE, hdfsUrl); } - return null; } From e76f5b589cc2f1200dc54fe63c24cdf23fb50e3f Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 15 Jun 2017 14:21:08 +0200 Subject: [PATCH 14/18] Rename HDFS to WebHDFS --- docs/storage/storage.md | 8 +++---- ...reter.java => WebHDFSFileInterpreter.java} | 10 ++++---- ...t.java => WebHDFSFileInterpreterTest.java} | 16 ++++++------- .../{HDFSCommand.java => WebHDFSCommand.java} | 4 ++-- zeppelin-server/pom.xml | 22 ++++++++++++++++- ...bookRepo.java => WebHdfsNotebookRepo.java} | 12 +++++----- .../util/{HdfsSite.java => WebHdfsSite.java} | 24 +++++++++---------- 7 files changed, 58 insertions(+), 38 deletions(-) rename file/src/main/java/org/apache/zeppelin/file/{HDFSFileInterpreter.java => WebHDFSFileInterpreter.java} (97%) rename file/src/test/java/org/apache/zeppelin/file/{HDFSFileInterpreterTest.java => WebHDFSFileInterpreterTest.java} (95%) rename zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/{HDFSCommand.java => WebHDFSCommand.java} (98%) rename zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/{HdfsNotebookRepo.java => WebHdfsNotebookRepo.java} (94%) rename zeppelin-zengine/src/main/java/org/apache/zeppelin/util/{HdfsSite.java => WebHdfsSite.java} (89%) diff --git a/docs/storage/storage.md b/docs/storage/storage.md index e0d1ba033cc..0f58f82d413 100644 --- a/docs/storage/storage.md +++ b/docs/storage/storage.md @@ -33,7 +33,7 @@ There are few notebook storage systems available for a use out of the box: * storage using Amazon S3 service - `S3NotebookRepo` * storage using Azure service - `AzureNotebookRepo` * storage using MongoDB - `MongoNotebookRepo` - * storage using HDFS - `HdfsNotebookRepo` + * storage using Web HDFS - `WebHdfsNotebookRepo` Multiple storage systems can be used at the same time by providing a comma-separated list of the class-names in the configuration. By default, only first two of them will be automatically kept in sync by Zeppelin. @@ -248,14 +248,14 @@ Optionally, you can specify Azure folder structure name in the file **zeppelin-s
-## Notebook Storage in Hdfs +## Notebook Storage in Web Hdfs - To enable your notebooks to be stored on HDFS - uncomment the next property in `zeppelin-site.xml` in order to use HdfsNotebookRepo class: + To enable your notebooks to be stored on HDFS - uncomment the next property in `zeppelin-site.xml` in order to use WebHdfsNotebookRepo class: ``` zeppelin.notebook.storage - org.apache.zeppelin.notebook.repo.HdfsNotebookRepo + org.apache.zeppelin.notebook.repo.WebHdfsNotebookRepo notebook persistence layer implementation diff --git a/file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java b/file/src/main/java/org/apache/zeppelin/file/WebHDFSFileInterpreter.java similarity index 97% rename from file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java rename to file/src/main/java/org/apache/zeppelin/file/WebHDFSFileInterpreter.java index 05f488d1ae1..f114c9f7e72 100644 --- a/file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java +++ b/file/src/main/java/org/apache/zeppelin/file/WebHDFSFileInterpreter.java @@ -32,28 +32,28 @@ * HDFS implementation of File interpreter for Zeppelin. * */ -public class HDFSFileInterpreter extends FileInterpreter { +public class WebHDFSFileInterpreter extends FileInterpreter { public static final String HDFS_URL = "hdfs.url"; public static final String HDFS_USER = "hdfs.user"; public static final String HDFS_MAXLENGTH = "hdfs.maxlength"; Exception exceptionOnConnect = null; - HDFSCommand cmd = null; + WebHDFSCommand cmd = null; Gson gson = null; public void prepare() { String userName = getProperty(HDFS_USER); String hdfsUrl = getProperty(HDFS_URL); int i = Integer.parseInt(getProperty(HDFS_MAXLENGTH)); - cmd = new HDFSCommand(hdfsUrl, userName, logger, i); + cmd = new WebHDFSCommand(hdfsUrl, userName, logger, i); gson = new Gson(); } - public HDFSFileInterpreter(Properties property){ + public WebHDFSFileInterpreter(Properties property){ super(property); prepare(); } - + // tests whether we're able to connect to HDFS private void testConnection() { diff --git a/file/src/test/java/org/apache/zeppelin/file/HDFSFileInterpreterTest.java b/file/src/test/java/org/apache/zeppelin/file/WebHDFSFileInterpreterTest.java similarity index 95% rename from file/src/test/java/org/apache/zeppelin/file/HDFSFileInterpreterTest.java rename to file/src/test/java/org/apache/zeppelin/file/WebHDFSFileInterpreterTest.java index 335693f921c..6543860ab93 100644 --- a/file/src/test/java/org/apache/zeppelin/file/HDFSFileInterpreterTest.java +++ b/file/src/test/java/org/apache/zeppelin/file/WebHDFSFileInterpreterTest.java @@ -40,11 +40,11 @@ * Tests Interpreter by running pre-determined commands against mock file system * */ -public class HDFSFileInterpreterTest extends TestCase { +public class WebHDFSFileInterpreterTest extends TestCase { @Test public void test() { - HDFSFileInterpreter t = new MockHDFSFileInterpreter(new Properties()); + WebHDFSFileInterpreter t = new MockWebHDFSFileInterpreter(new Properties()); t.open(); // We have info for /, /user, /tmp, /mr-history/done @@ -164,7 +164,7 @@ void addGetFileStatusData() { mfs.put("/mr-history/done?op=GETFILESTATUS", "{\"FileStatus\":{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16393,\"group\":\"hadoop\",\"length\":0,\"modificationTime\":1441253197480,\"owner\":\"mapred\",\"pathSuffix\":\"\",\"permission\":\"777\",\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}}"); } - public void addMockData(HDFSCommand.Op op) { + public void addMockData(WebHDFSCommand.Op op) { if (op.op.equals("LISTSTATUS")) { addListStatusData(); } else if (op.op.equals("GETFILESTATUS")) { @@ -180,10 +180,10 @@ public String get(String key) { /** * Run commands against mock file system that simulates webhdfs responses */ - class MockHDFSCommand extends HDFSCommand { + class MockWebHDFSCommand extends WebHDFSCommand { MockFileSystem fs = null; - public MockHDFSCommand(String url, String user, Logger logger) { + public MockWebHDFSCommand(String url, String user, Logger logger) { super(url, user, logger, 1000); fs = new MockFileSystem(); fs.addMockData(getFileStatus); @@ -210,16 +210,16 @@ public String runCommand(Op op, String path, Arg[] args) throws Exception { /** * Mock Interpreter - uses Mock HDFS command */ - class MockHDFSFileInterpreter extends HDFSFileInterpreter { + class MockWebHDFSFileInterpreter extends WebHDFSFileInterpreter { @Override public void prepare() { // Run commands against mock File System instead of WebHDFS - cmd = new MockHDFSCommand("", "", logger); + cmd = new MockWebHDFSCommand("", "", logger); gson = new Gson(); } - public MockHDFSFileInterpreter(Properties property) { + public MockWebHDFSFileInterpreter(Properties property) { super(property); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSCommand.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/WebHDFSCommand.java similarity index 98% rename from zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSCommand.java rename to zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/WebHDFSCommand.java index ed58ac306e9..4b21fa41e2b 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSCommand.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/WebHDFSCommand.java @@ -32,7 +32,7 @@ /** * Definition and HTTP invocation methods for all WebHDFS commands */ -public class HDFSCommand { +public class WebHDFSCommand { /** * Type of HTTP request @@ -86,7 +86,7 @@ public Arg(String key, String value) { public Op deleteFile = new Op("DELETE", HttpType.DELETE, 0); public Op renameFile = new Op("RENAME", HttpType.PUT, 0); - public HDFSCommand(String url, String user, Logger logger, int maxLength) { + public WebHDFSCommand(String url, String user, Logger logger, int maxLength) { super(); this.url = url; this.user = user; diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index bf4f52aee75..680f2506c6f 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -39,7 +39,7 @@ 2.6.0 2.2.1 1.13 - 2.0.1 + 2.0-m10 1.8 4.1.0 @@ -114,12 +114,22 @@ javax.annotation javax.annotation-api + + javax.ws.rs + javax.ws.rs-api + org.glassfish.jersey.containers jersey-container-servlet-core ${jersey.version} + + + javax.ws.rs + javax.ws.rs-api + + org.glassfish.jersey.media @@ -130,12 +140,22 @@ com.fasterxml.jackson.core jackson-annotations + + javax.ws.rs + javax.ws.rs-api + org.glassfish.jersey.core jersey-server ${jersey.version} + + + javax.ws.rs + javax.ws.rs-api + + diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/HdfsNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/WebHdfsNotebookRepo.java similarity index 94% rename from zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/HdfsNotebookRepo.java rename to zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/WebHdfsNotebookRepo.java index 7accafaadb0..5885e4c0c6a 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/HdfsNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/WebHdfsNotebookRepo.java @@ -27,7 +27,7 @@ import org.apache.zeppelin.notebook.Paragraph; import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.user.AuthenticationInfo; -import org.apache.zeppelin.util.HdfsSite; +import org.apache.zeppelin.util.WebHdfsSite; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,18 +42,18 @@ * */ -public class HdfsNotebookRepo implements NotebookRepo { - private static Logger logger = LoggerFactory.getLogger(HdfsNotebookRepo.class); - private HdfsSite hdfsSite; +public class WebHdfsNotebookRepo implements NotebookRepo { + private static Logger logger = LoggerFactory.getLogger(WebHdfsNotebookRepo.class); + private WebHdfsSite hdfsSite; private ZeppelinConfiguration conf; private String rootDir; - public HdfsNotebookRepo(ZeppelinConfiguration conf) throws IOException { + public WebHdfsNotebookRepo(ZeppelinConfiguration conf) throws IOException { this.conf = conf; try { rootDir = removeProtocol(conf.getNotebookDir()); - hdfsSite = new HdfsSite(conf); + hdfsSite = new WebHdfsSite(conf); hdfsSite.mkdirs(rootDir); } catch (URISyntaxException e) { throw new IOException(e); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/HdfsSite.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/WebHdfsSite.java similarity index 89% rename from zeppelin-zengine/src/main/java/org/apache/zeppelin/util/HdfsSite.java rename to zeppelin-zengine/src/main/java/org/apache/zeppelin/util/WebHdfsSite.java index daae473db14..8b9250bfa9a 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/HdfsSite.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/WebHdfsSite.java @@ -19,7 +19,7 @@ import com.google.gson.Gson; import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.file.HDFSCommand; +import org.apache.zeppelin.file.WebHDFSCommand; import org.apache.zeppelin.file.HDFSStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,12 +33,12 @@ /** * Class to create / move / delete / write to / read from HDFS filessytem */ -public class HdfsSite { +public class WebHdfsSite { public static final String HDFS_URL = "hdfs.url"; public static final String HDFS_USER = "hdfs.user"; public static final String HDFS_MAXLENGTH = "hdfs.maxlength"; - private static Logger logger = LoggerFactory.getLogger(HdfsSite.class); + private static Logger logger = LoggerFactory.getLogger(WebHdfsSite.class); public static final String HDFS_NOTEBOOK_DIR = "hdfs.notebook.dir"; static final String NOTE_JSON = "note.json"; static final String NOTE_JSON_TEMP = "_note.json"; @@ -48,10 +48,10 @@ public class HdfsSite { String hdfsUrl = null; String hdfsUser = null; int hdfsMaxLength = 0; - HDFSCommand hdfsCmd = null; + WebHDFSCommand hdfsCmd = null; - public HdfsSite(ZeppelinConfiguration conf) throws URISyntaxException { + public WebHdfsSite(ZeppelinConfiguration conf) throws URISyntaxException { this.conf = conf; this.hdfsUrl = conf.getString(HDFS_URL, HDFS_URL, "http://localhost:50070/webhdfs/v1/"); this.hdfsMaxLength = conf.getInt(HDFS_URL, HDFS_MAXLENGTH, 100000); @@ -63,7 +63,7 @@ public HdfsSite(ZeppelinConfiguration conf) throws URISyntaxException { this.enableWebHDFS = false; } else { - this.hdfsCmd = new HDFSCommand(hdfsUrl, hdfsUser, logger, this.hdfsMaxLength); + this.hdfsCmd = new WebHDFSCommand(hdfsUrl, hdfsUser, logger, this.hdfsMaxLength); this.enableWebHDFS = exists("/"); } } @@ -114,8 +114,8 @@ public String[] listFiles(String directory) throws IOException { public void delete(String path) throws IOException { logger.debug("remove : " + path); - HDFSCommand.Arg recursive = this.hdfsCmd.new Arg("recursive", "true"); - HDFSCommand.Arg[] args = {recursive}; + WebHDFSCommand.Arg recursive = this.hdfsCmd.new Arg("recursive", "true"); + WebHDFSCommand.Arg[] args = {recursive}; try { this.hdfsCmd.runCommand(this.hdfsCmd.deleteFile, path, args); @@ -138,8 +138,8 @@ public void mkdirs(String path) throws IOException { public void writeFile(byte[] content, String path) throws IOException { try { - HDFSCommand.Arg dest = this.hdfsCmd.new Arg("overwrite", "true"); - HDFSCommand.Arg[] createArgs = {dest}; + WebHDFSCommand.Arg dest = this.hdfsCmd.new Arg("overwrite", "true"); + WebHDFSCommand.Arg[] createArgs = {dest}; this.hdfsCmd.runCommand(this.hdfsCmd.createWriteFile, path, content, createArgs); } catch (Exception e) { logger.error("Exception: ", e); @@ -149,8 +149,8 @@ public void writeFile(byte[] content, String path) throws IOException { public void rename(String oldPath, String newPath) throws IOException { try { - HDFSCommand.Arg dest = this.hdfsCmd.new Arg("destination", newPath); - HDFSCommand.Arg[] renameArgs = {dest}; + WebHDFSCommand.Arg dest = this.hdfsCmd.new Arg("destination", newPath); + WebHDFSCommand.Arg[] renameArgs = {dest}; this.hdfsCmd.runCommand(this.hdfsCmd.renameFile, oldPath, renameArgs); } catch (Exception e) { logger.error("Exception: ", e); From 92fc7b84ea405fa7e446012d0acefc524a23c67f Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Fri, 30 Jun 2017 09:10:18 +0200 Subject: [PATCH 15/18] remove UriBuilder dependency --- .../src/main/java/org/apache/zeppelin/file/WebHDFSCommand.java | 1 - 1 file changed, 1 deletion(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/WebHDFSCommand.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/WebHDFSCommand.java index 4b21fa41e2b..3d9f7acd087 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/WebHDFSCommand.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/WebHDFSCommand.java @@ -21,7 +21,6 @@ import org.apache.http.client.utils.URIBuilder; import org.slf4j.Logger; -import javax.ws.rs.core.UriBuilder; import java.io.BufferedReader; import java.io.DataOutputStream; import java.io.IOException; From 61fce8c2caa9163e3387f30c3ce5d6646578078d Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Fri, 30 Jun 2017 12:45:01 +0200 Subject: [PATCH 16/18] Rename HDFSFileInterfpreter to WebHDFSFoleInterpreter --- conf/zeppelin-site.xml.template | 2 +- .../main/resources/interpreter-setting.json | 2 +- zeppelin-interpreter/pom.xml | 16 +---------- zeppelin-server/pom.xml | 28 +++---------------- .../zeppelin/conf/ZeppelinConfiguration.java | 2 +- 5 files changed, 8 insertions(+), 42 deletions(-) diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index 584434b9d53..56f7f30b8db 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -271,7 +271,7 @@ zeppelin.interpreters - org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter,org.apache.zeppelin.groovy.GroovyInterpreter + org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.WebHDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter,org.apache.zeppelin.groovy.GroovyInterpreter Comma separated interpreter configurations. First interpreter become a default diff --git a/file/src/main/resources/interpreter-setting.json b/file/src/main/resources/interpreter-setting.json index b4f9199c8d7..557bbf30e90 100644 --- a/file/src/main/resources/interpreter-setting.json +++ b/file/src/main/resources/interpreter-setting.json @@ -2,7 +2,7 @@ { "group": "file", "name": "hdfs", - "className": "org.apache.zeppelin.file.HDFSFileInterpreter", + "className": "org.apache.zeppelin.file.WebHDFSFileInterpreter", "properties": { "hdfs.url": { "envName": null, diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml index 3f9279f6581..fc6a3d53513 100644 --- a/zeppelin-interpreter/pom.xml +++ b/zeppelin-interpreter/pom.xml @@ -44,8 +44,6 @@ 3.0.3 1.0 2.12.1 - 2.0-m10 - 4.3.6 2.3 @@ -85,12 +83,6 @@ commons-lang - - javax.ws.rs - javax.ws.rs-api - ${ws.rsapi.version} - - org.slf4j slf4j-api @@ -227,12 +219,6 @@ guava - - org.apache.httpcomponents - httpclient - ${httpcomponents.client.version} - - junit junit @@ -245,4 +231,4 @@ test - + \ No newline at end of file diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index d9f8f7103b1..00ce4bc51dc 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -39,7 +39,7 @@ 2.6.0 2.2.1 1.13 - 2.0-m10 + 2.0.1 1.8 4.1.0 @@ -114,22 +114,12 @@ javax.annotation javax.annotation-api - - javax.ws.rs - javax.ws.rs-api - org.glassfish.jersey.containers jersey-container-servlet-core ${jersey.version} - - - javax.ws.rs - javax.ws.rs-api - - org.glassfish.jersey.media @@ -140,22 +130,12 @@ com.fasterxml.jackson.core jackson-annotations - - javax.ws.rs - javax.ws.rs-api - org.glassfish.jersey.core jersey-server ${jersey.version} - - - javax.ws.rs - javax.ws.rs-api - - @@ -505,7 +485,7 @@ - ../bin + ../bin @@ -517,10 +497,10 @@ - ../zeppelin-distribution/target/zeppelin-${project.version}/zeppelin-${project.version}/bin + ../zeppelin-distribution/target/zeppelin-${project.version}/zeppelin-${project.version}/bin - + \ No newline at end of file diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index ade1fbd2c37..c577c87833d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -585,7 +585,7 @@ public static enum ConfVars { + "org.apache.zeppelin.livy.LivyPySpark3Interpreter," + "org.apache.zeppelin.livy.LivySparkRInterpreter," + "org.apache.zeppelin.alluxio.AlluxioInterpreter," - + "org.apache.zeppelin.file.HDFSFileInterpreter," + + "org.apache.zeppelin.file.WebHDFSFileInterpreter," + "org.apache.zeppelin.pig.PigInterpreter," + "org.apache.zeppelin.pig.PigQueryInterpreter," + "org.apache.zeppelin.flink.FlinkInterpreter," From 652af86196ef4caaeac11ec7830b3110b4d0fef4 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Fri, 30 Jun 2017 13:58:08 +0200 Subject: [PATCH 17/18] Remove unnecessary comments --- .../main/java/org/apache/zeppelin/file/WebHDFSCommand.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/WebHDFSCommand.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/WebHDFSCommand.java index 3d9f7acd087..42f0377c986 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/WebHDFSCommand.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/WebHDFSCommand.java @@ -136,26 +136,20 @@ public String runCommand(Op op, String path, byte[] argFile, Arg[] args) throws if (args != null) { boolean isUserName = false; for (Arg a : args) { -// builder = builder.queryParam(a.key, a.value); uriBuilder.addParameter(a.key, a.value); - //System.out.println("a.key/a.value=" + a.key + "=" + a.value); if ("user.name".equals(a.key)) { isUserName = true; } } if (!isUserName) { - //builder = builder.queryParam("user.name", this.user); uriBuilder.addParameter("user.name", this.user); } } else { - // builder = builder.queryParam("user.name", this.user); uriBuilder.addParameter("user.name", this.user); } java.net.URI uri = uriBuilder.build(); - // System.out.println("resUrl=" + resUrl); // Connect and get response string URL hdfsUrl = uri.toURL(); - // System.out.println("hdfsUrl=" + hdfsUrl.toExternalForm()); HttpURLConnection con = (HttpURLConnection) hdfsUrl.openConnection(); if (op.cmd == HttpType.GET) { From b8d9d0b2c821314baeef2c3d6005c394dbe42d91 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Fri, 21 Jul 2017 13:14:11 -0700 Subject: [PATCH 18/18] no message --- .../zeppelin/file/WebHDFSFileInterpreter.java | 8 ++++---- .../zeppelin/file/WebHDFSFileInterpreterTest.java | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/file/src/main/java/org/apache/zeppelin/file/WebHDFSFileInterpreter.java b/file/src/main/java/org/apache/zeppelin/file/WebHDFSFileInterpreter.java index 244101c9bda..cc61985ca0e 100644 --- a/file/src/main/java/org/apache/zeppelin/file/WebHDFSFileInterpreter.java +++ b/file/src/main/java/org/apache/zeppelin/file/WebHDFSFileInterpreter.java @@ -32,24 +32,24 @@ * HDFS implementation of File interpreter for Zeppelin. * */ -public class HDFSFileInterpreter extends FileInterpreter { +public class WebHDFSFileInterpreter extends FileInterpreter { static final String HDFS_URL = "hdfs.url"; static final String HDFS_USER = "hdfs.user"; static final String HDFS_MAXLENGTH = "hdfs.maxlength"; Exception exceptionOnConnect = null; - HDFSCommand cmd = null; + WebHDFSCommand cmd = null; Gson gson = null; public void prepare() { String userName = getProperty(HDFS_USER); String hdfsUrl = getProperty(HDFS_URL); int i = Integer.parseInt(getProperty(HDFS_MAXLENGTH)); - cmd = new HDFSCommand(hdfsUrl, userName, logger, i); + cmd = new WebHDFSCommand(hdfsUrl, userName, logger, i); gson = new Gson(); } - public HDFSFileInterpreter(Properties property){ + public WebHDFSFileInterpreter(Properties property){ super(property); prepare(); } diff --git a/file/src/test/java/org/apache/zeppelin/file/WebHDFSFileInterpreterTest.java b/file/src/test/java/org/apache/zeppelin/file/WebHDFSFileInterpreterTest.java index adc9bd6b55f..619523a427d 100644 --- a/file/src/test/java/org/apache/zeppelin/file/WebHDFSFileInterpreterTest.java +++ b/file/src/test/java/org/apache/zeppelin/file/WebHDFSFileInterpreterTest.java @@ -40,12 +40,12 @@ * Tests Interpreter by running pre-determined commands against mock file system * */ -public class HDFSFileInterpreterTest extends TestCase { +public class WebHDFSFileInterpreterTest extends TestCase { @Test public void testMaxLength() { - HDFSFileInterpreter t = new MockHDFSFileInterpreter(new Properties()); + WebHDFSFileInterpreter t = new MockHDFSFileInterpreter(new Properties()); t.open(); InterpreterResult result = t.interpret("ls -l /", null); String lineSeparator = "\n"; @@ -56,7 +56,7 @@ public void testMaxLength() { Properties properties = new Properties(); final int maxLength = fileStatusLength - 2; properties.setProperty("hdfs.maxlength", String.valueOf(maxLength)); - HDFSFileInterpreter t1 = new MockHDFSFileInterpreter(properties); + WebHDFSFileInterpreter t1 = new MockHDFSFileInterpreter(properties); t1.open(); InterpreterResult result1 = t1.interpret("ls -l /", null); assertEquals(result1.message().get(0).getData().split(lineSeparator).length, maxLength); @@ -65,7 +65,7 @@ public void testMaxLength() { @Test public void test() { - HDFSFileInterpreter t = new MockHDFSFileInterpreter(new Properties()); + WebHDFSFileInterpreter t = new MockHDFSFileInterpreter(new Properties()); t.open(); // We have info for /, /user, /tmp, /mr-history/done @@ -186,7 +186,7 @@ void addGetFileStatusData() { mfs.put("/mr-history/done?op=GETFILESTATUS", "{\"FileStatus\":{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16393,\"group\":\"hadoop\",\"length\":0,\"modificationTime\":1441253197480,\"owner\":\"mapred\",\"pathSuffix\":\"\",\"permission\":\"777\",\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}}"); } - public void addMockData(HDFSCommand.Op op) { + public void addMockData(WebHDFSCommand.Op op) { if (op.op.equals("LISTSTATUS")) { addListStatusData(); } else if (op.op.equals("GETFILESTATUS")) { @@ -202,7 +202,7 @@ public String get(String key) { /** * Run commands against mock file system that simulates webhdfs responses */ - class MockHDFSCommand extends HDFSCommand { + class MockHDFSCommand extends WebHDFSCommand { MockFileSystem fs = null; public MockHDFSCommand(String url, String user, Logger logger, int maxLength) { @@ -236,7 +236,7 @@ public String runCommand(Op op, String path, Arg[] args) throws Exception { /** * Mock Interpreter - uses Mock HDFS command */ - class MockHDFSFileInterpreter extends HDFSFileInterpreter { + class MockHDFSFileInterpreter extends WebHDFSFileInterpreter { @Override public void prepare() {