Notebook: (Web) HDFS as a backend storage (Read & Write Mode)

conf/zeppelin-site.xml.template

@@ -271,7 +271,7 @@
 
 <property>
   <name>zeppelin.interpreters</name>
-  <value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter,org.apache.zeppelin.groovy.GroovyInterpreter</value>
+  <value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.WebHDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter,org.apache.zeppelin.groovy.GroovyInterpreter</value>
   <description>Comma separated interpreter configurations. First interpreter become a default</description>
 </property>
 

docs/setup/storage/storage.md

@@ -33,6 +33,7 @@ There are few notebook storage systems available for a use out of the box:
   * storage using Amazon S3 service - `S3NotebookRepo`
   * storage using Azure service - `AzureNotebookRepo`
   * storage using MongoDB - `MongoNotebookRepo`
+  * storage using Web HDFS - `WebHdfsNotebookRepo`
 
 Multiple storage systems can be used at the same time by providing a comma-separated list of the class-names in the configuration.
 By default, only first two of them will be automatically kept in sync by Zeppelin.
@@ -246,7 +247,51 @@ Optionally, you can specify Azure folder structure name in the file **zeppelin-s
 ```
 
 </br>
+
+## Notebook Storage in Web Hdfs <a name="HDFS"></a>
+
+To enable your notebooks to be stored on HDFS - uncomment the next property in `zeppelin-site.xml` in order to use WebHdfsNotebookRepo class:
+
+```
+<property>
+<name>zeppelin.notebook.storage</name>
+<value>org.apache.zeppelin.notebook.repo.WebHdfsNotebookRepo</value>
+<description>notebook persistence layer implementation</description>
+</property>
+
+<property>
+<name>hdfs.url</name>
+<value>http://localhost:50070/webhdfs/v1/</value>
+<description>HDFS url</description>
+</property>
+
+<property>
+<name>hdfs.user</name>
+<value>hdfs</value>
+<description>HDFS user</description>
+</property>
+
+<property>
+<name>hdfs.maxlength</name>
+<value>1000</value>
+<description>Maximum number of lines of results fetched</description>
+</property>
+
+```
+
+and replace the notebook directory property below by an absolute HDFS location as follows :
+```
+<property>
+<name>zeppelin.notebook.dir</name>
+<value>/tmp/notebook</value>
+<description>path or URI for notebook persist</description>
+</property>
+```
+
+</br>
+
 ## Notebook Storage in ZeppelinHub  <a name="ZeppelinHub"></a>
+>>>>>>> 1c23f2138834bba3a08b365c59d861687c289589:docs/setup/storage/storage.md
 
 ZeppelinHub storage layer allows out of the box connection of Zeppelin instance with your ZeppelinHub account. First of all, you need to either comment out the following  property in **zeppelin-site.xml**:
 

file/pom.xml

@@ -33,7 +33,6 @@
 
   <properties>
     <!--library versions-->
-    <ws.rsapi.version>2.0</ws.rsapi.version>
     <jersey.common.version>2.22.2</jersey.common.version>
 
     <!--plugin versions-->
@@ -49,12 +48,12 @@
     </dependency>
 
     <dependency>
-      <groupId>javax.ws.rs</groupId>
-      <artifactId>javax.ws.rs-api</artifactId>
-      <version>${ws.rsapi.version}</version>
+      <groupId>org.glassfish.jersey.core</groupId>
+      <artifactId>jersey-common</artifactId>
+      <version>${jersey.common.version}</version>
     </dependency>
 
-    <dependency>
+  <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
     </dependency>
@@ -64,12 +63,6 @@
       <artifactId>slf4j-log4j12</artifactId>
     </dependency>
 
-    <dependency>
-      <groupId>org.glassfish.jersey.core</groupId>
-      <artifactId>jersey-common</artifactId>
-      <version>${jersey.common.version}</version>
-    </dependency>
-
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>

file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java → file/src/main/java/org/apache/zeppelin/file/WebHDFSFileInterpreter.java

@@ -32,24 +32,24 @@
  * HDFS implementation of File interpreter for Zeppelin.
  *
  */
-public class HDFSFileInterpreter extends FileInterpreter {
+public class WebHDFSFileInterpreter extends FileInterpreter {
   static final String HDFS_URL = "hdfs.url";
   static final String HDFS_USER = "hdfs.user";
   static final String HDFS_MAXLENGTH = "hdfs.maxlength";
 
   Exception exceptionOnConnect = null;
-  HDFSCommand cmd = null;
+  WebHDFSCommand cmd = null;
   Gson gson = null;
 
   public void prepare() {
     String userName = getProperty(HDFS_USER);
     String hdfsUrl = getProperty(HDFS_URL);
     int i = Integer.parseInt(getProperty(HDFS_MAXLENGTH));
-    cmd = new HDFSCommand(hdfsUrl, userName, logger, i);
+    cmd = new WebHDFSCommand(hdfsUrl, userName, logger, i);
     gson = new Gson();
   }
 
-  public HDFSFileInterpreter(Properties property){
+  public WebHDFSFileInterpreter(Properties property){
     super(property);
     prepare();
   }

file/src/main/resources/interpreter-setting.json

@@ -2,7 +2,7 @@
   {
     "group": "file",
     "name": "hdfs",
-    "className": "org.apache.zeppelin.file.HDFSFileInterpreter",
+    "className": "org.apache.zeppelin.file.WebHDFSFileInterpreter",
     "properties": {
       "hdfs.url": {
         "envName": null,

file/src/test/java/org/apache/zeppelin/file/HDFSFileInterpreterTest.java → file/src/test/java/org/apache/zeppelin/file/WebHDFSFileInterpreterTest.java

@@ -40,12 +40,12 @@
  * Tests Interpreter by running pre-determined commands against mock file system
  *
  */
-public class HDFSFileInterpreterTest extends TestCase {
+public class WebHDFSFileInterpreterTest extends TestCase {
 
     @Test
     public void testMaxLength() {
 
-      HDFSFileInterpreter t = new MockHDFSFileInterpreter(new Properties());
+      WebHDFSFileInterpreter t = new MockHDFSFileInterpreter(new Properties());
       t.open();
       InterpreterResult result = t.interpret("ls -l /", null);
       String lineSeparator = "\n";
@@ -56,7 +56,7 @@ public void testMaxLength() {
       Properties properties = new Properties();
       final int maxLength = fileStatusLength - 2;
       properties.setProperty("hdfs.maxlength", String.valueOf(maxLength));
-      HDFSFileInterpreter t1 = new MockHDFSFileInterpreter(properties);
+      WebHDFSFileInterpreter t1 = new MockHDFSFileInterpreter(properties);
       t1.open();
       InterpreterResult result1 = t1.interpret("ls -l /", null);
       assertEquals(result1.message().get(0).getData().split(lineSeparator).length, maxLength);
@@ -65,7 +65,7 @@ public void testMaxLength() {
 
     @Test
     public void test() {
-      HDFSFileInterpreter t = new MockHDFSFileInterpreter(new Properties());
+      WebHDFSFileInterpreter t = new MockHDFSFileInterpreter(new Properties());
       t.open();
 
       // We have info for /, /user, /tmp, /mr-history/done
@@ -186,7 +186,7 @@ void addGetFileStatusData() {
       mfs.put("/mr-history/done?op=GETFILESTATUS",
           "{\"FileStatus\":{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16393,\"group\":\"hadoop\",\"length\":0,\"modificationTime\":1441253197480,\"owner\":\"mapred\",\"pathSuffix\":\"\",\"permission\":\"777\",\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}}");
     }
-    public void addMockData(HDFSCommand.Op op) {
+    public void addMockData(WebHDFSCommand.Op op) {
       if (op.op.equals("LISTSTATUS")) {
         addListStatusData();
       } else if (op.op.equals("GETFILESTATUS")) {
@@ -202,7 +202,7 @@ public String get(String key) {
   /**
    * Run commands against mock file system that simulates webhdfs responses
    */
-  class MockHDFSCommand extends HDFSCommand {
+  class MockHDFSCommand extends WebHDFSCommand {
     MockFileSystem fs = null;
 
     public MockHDFSCommand(String url, String user, Logger logger, int maxLength) {
@@ -236,7 +236,7 @@ public String runCommand(Op op, String path, Arg[] args) throws Exception {
   /**
    * Mock Interpreter - uses Mock HDFS command
    */
-  class MockHDFSFileInterpreter extends HDFSFileInterpreter {
+  class MockHDFSFileInterpreter extends WebHDFSFileInterpreter {
 
     @Override
     public void prepare() {

zeppelin-interpreter/pom.xml

@@ -231,4 +231,4 @@
       <scope>test</scope>
     </dependency>
   </dependencies>
-</project>
+</project>

zeppelin-interpreter/src/main/java/org/apache/zeppelin/file/HDFSStatus.java

@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.file;
+
+/**
+ * Class that hosts HDFS file statuses
+ */
+public class HDFSStatus {
+  /**
+   * Status of one file
+   *
+   * matches returned JSON
+   */
+  public class OneFileStatus {
+    public long accessTime;
+    public int blockSize;
+    public int childrenNum;
+    public int fileId;
+    public String group;
+    public long length;
+    public long modificationTime;
+    public String owner;
+    public String pathSuffix;
+    public String permission;
+    public int replication;
+    public int storagePolicy;
+    public String type;
+    public String toString() {
+      StringBuilder sb = new StringBuilder();
+      sb.append("\nAccessTime = ").append(accessTime);
+      sb.append("\nBlockSize = ").append(blockSize);
+      sb.append("\nChildrenNum = ").append(childrenNum);
+      sb.append("\nFileId = ").append(fileId);
+      sb.append("\nGroup = ").append(group);
+      sb.append("\nLength = ").append(length);
+      sb.append("\nModificationTime = ").append(modificationTime);
+      sb.append("\nOwner = ").append(owner);
+      sb.append("\nPathSuffix = ").append(pathSuffix);
+      sb.append("\nPermission = ").append(permission);
+      sb.append("\nReplication = ").append(replication);
+      sb.append("\nStoragePolicy = ").append(storagePolicy);
+      sb.append("\nType = ").append(type);
+      return sb.toString();
+    }
+  }
+
+  /**
+   * Status of one file
+   *
+   * matches returned JSON
+   */
+  public class SingleFileStatus {
+    public OneFileStatus FileStatus;
+  }
+
+  /**
+   * Status of all files in a directory
+   *
+   * matches returned JSON
+   */
+  public class MultiFileStatus {
+    public OneFileStatus[] FileStatus;
+  }
+
+  /**
+   * Status of all files in a directory
+   *
+   * matches returned JSON
+   */
+  public class AllFileStatus {
+    public MultiFileStatus FileStatuses;
+  }
+
+
+}
+
+