Added ipfsnotebookrepo [ Zeppelin-683 ]

conf/zeppelin-site.xml.template

@@ -158,6 +158,21 @@
 </property>
 -->
 
+<!-- For saving and sharing notebooks with IPFS -->
+<!--
+<property>
+  <name>zeppelin.notebook.storage</name>
+  <value>org.apache.zeppelin.notebook.repo.ipfs.IPFSNotebookRepo</value>
+  <description>notebook persistence layer implementation for sharing notebooks</description>
+</property>
+
+<property>
+  <name>zeppelin.notebook.ipfs.apiServer</name>
+  <value>http://localhost:5001/api/v0/</value>
+  <description>ipfs api Server Address</description>
+</property>
+-->
+
 <property>
   <name>zeppelin.notebook.storage</name>
   <value>org.apache.zeppelin.notebook.repo.VFSNotebookRepo</value>

docs/storage/storage.md

@@ -238,4 +238,39 @@ export ZEPPELINHUB_API_TOKEN = ZeppelinHub token
 export ZEPPELINHUB_API_ADDRESS = address of ZeppelinHub service (e.g. https://www.zeppelinhub.com)
 ```
 
-You can get more information on generating `token` and using authentication on the corresponding [help page](http://help.zeppelinhub.com/zeppelin_integration/#add-a-new-zeppelin-instance-and-generate-a-token).
+You can get more information on generating `token` and using authentication on the corresponding [help page](http://help.zeppelinhub.com/zeppelin_integration/#add-a-new-zeppelin-instance-and-generate-a-token).
+
+</br>
+## Storage in IPFS  <a name="IPFS"></a>
+
+Using IpfsNotebookRepo you can use Ipfs to save Note revisions and retrieve a particular revision.
+
+Comment out the following in **zeppelin-site.xml**
+
+Make sure you enter the correct ipfs apiServer.
+```
+<property>
+        <name>zeppelin.notebook.storage</name>
+        <value>org.apache.zeppelin.notebook.repo.ipfs.IPFSNotebookRepo</value>
+        <description>notebook persistence layer implementation</description>
+    </property>
+
+    <property>
+        <name>zeppelin.notebook.ipfs.apiServer</name>
+        <value>http://localhost:5001/api/v0/</value>
+        <description>ipfs api Server Multiaddress</description>
+    </property>
+```
+and comment
+
+```
+<property>
+  <name>zeppelin.notebook.storage</name>
+  <value>org.apache.zeppelin.notebook.repo.VFSNotebookRepo</value>
+  <description>notebook persistence layer implementation</description>
+</property>
+```
+In the `notebook` directory is a file `ipfsnotehashes.json` which contains the ipfs hash for each
+ note revision which is commited, it is saved in your local '.ipfs' directory. You can share
+ these ipfs hash with others. Notebook can also be imported by entering this hash.
+

zeppelin-distribution/src/bin_license/LICENSE

@@ -17,6 +17,7 @@ The following components are provided under Apache License.
     (Apache 2.0) Apache Commons Exec (commons-exec:commons-exec:1.3 - http://commons.apache.org/exec/)
     (Apache 2.0) Http Components (org.apache.httpcomponents:httpcore:4.3.3 - https://github.com/apache/httpclient)
     (Apache 2.0) Http Components (org.apache.httpcomponents:httpclient:4.3.6 - https://github.com/apache/httpclient)
+    (Apache 2.0) Http Components (org.apache.httpcomponents:httpmime:4.3.6 - https://github.com/apache/httpclient)
     (Apache 2.0) Apache Commons Lang (org.apache.commons:commons-lang:2.5 - http://commons.apache.org/proper/commons-lang/)
     (Apache 2.0) Apache Commons Lang 3 (org.apache.commons:commons-lang3:3.4 - http://commons.apache.org/proper/commons-lang/)
     (Apache 2.0) Apache Commons Math 3 (org.apache.commons:commons-math3:3.4.1 - http://commons.apache.org/proper/commons-math/)

zeppelin-server/pom.xml

@@ -319,6 +319,10 @@
           <groupId>org.eclipse.jetty.websocket</groupId>
           <artifactId>websocket-client</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.httpcomponents</groupId>
+          <artifactId>httpmime</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 

zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java

@@ -188,6 +188,9 @@ public void onMessage(NotebookSocket conn, String msg) {
           case IMPORT_NOTE:
             importNote(conn, userAndRoles, notebook, messagereceived);
             break;
+          case IMPORT_NOTE_URL:
+            importNoteFromUrl(conn, userAndRoles, notebook, messagereceived);
+            break;
           case COMMIT_PARAGRAPH:
             updateParagraph(conn, userAndRoles, notebook, messagereceived);
             break;
@@ -728,6 +731,33 @@ protected Note importNote(NotebookSocket conn, HashSet<String> userAndRoles,
     return note;
   }
 
+  protected Note importNoteFromUrl(NotebookSocket conn, HashSet<String> userAndRoles,
+                                   Notebook notebook, Message fromMessage) throws IOException {
+    Note note = null;
+    if (fromMessage != null) {
+      String type = (String) fromMessage.get("type");
+      if (type.equals("ipfs")) {
+        String name = (String) fromMessage.get("name");
+        Map<String, Object> options = ((Map<String, Object>) fromMessage.get("options"));
+        String urlHash = (String) options.get("hash");
+        AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
+        note = notebook.importNoteFromBackend(urlHash, name, subject);
+        Message message = new Message(OP.IMPORT_NOTE_STATUS);
+        message.put("type", "ipfs");
+        message.put("options", options);
+        if (note != null) {
+          note.persist(subject);
+          message.put("importStatus", "success");
+        } else {
+          message.put("importStatus", "failure");
+        }
+        conn.send(serializeMessage(message));
+        broadcastNoteList(subject);
+      }
+    }
+    return note;
+  }
+
   private void removeParagraph(NotebookSocket conn, HashSet<String> userAndRoles,
                                Notebook notebook, Message fromMessage) throws IOException {
     final String paragraphId = (String) fromMessage.get("id");

zeppelin-web/src/components/noteName-import/note-import-dialog.html

@@ -56,6 +56,7 @@ <h4 class="modal-title">Import new note</h4>
             <label for="noteImportUrl">URL</label>
             <input placeholder="Note name" type="text" class="form-control" id="noteImportUrl"
                    ng-model="note.importUrl" />
+            <label class="checkbox-inline"><input type="checkbox"  ng-model="note.isOtherUrl" />Ipfs Hash</label>
           </div>
 
         </div>

zeppelin-web/src/components/noteName-import/notenameImport.controller.js

@@ -14,11 +14,12 @@
 
 'use strict';
 
-angular.module('zeppelinWebApp').controller('NoteImportCtrl', function($scope, $timeout, websocketMsgSrv) {
+angular.module('zeppelinWebApp').controller('NoteImportCtrl', function($scope, $timeout, ngToast, websocketMsgSrv) {
   var vm = this;
   $scope.note = {};
   $scope.note.step1 = true;
   $scope.note.step2 = false;
+  $scope.note.isOtherUrl = false;
 
   vm.resetFlags = function() {
     $scope.note = {};
@@ -65,12 +66,16 @@ angular.module('zeppelinWebApp').controller('NoteImportCtrl', function($scope, $
   vm.importNote = function() {
     $scope.note.errorText = '';
     if ($scope.note.importUrl) {
-      jQuery.getJSON($scope.note.importUrl, function(result) {
-        vm.processImportJson(result);
-      }).fail(function() {
-        $scope.note.errorText = 'Unable to Fetch URL';
-        $scope.$apply();
-      });
+      if (!$scope.note.isOtherUrl) {
+        jQuery.getJSON($scope.note.importUrl, function(result) {
+          vm.processImportJson(result);
+        }).fail(function() {
+          $scope.note.errorText = 'Unable to Fetch URL';
+          $scope.$apply();
+        });
+      } else {
+        websocketMsgSrv.importNoteFromBackend($scope.note.importUrl, $scope.note.noteImportName, 'ipfs');
+      }
     } else {
       $scope.note.errorText = 'Enter URL';
       $scope.$apply();
@@ -110,4 +115,23 @@ angular.module('zeppelinWebApp').controller('NoteImportCtrl', function($scope, $
     vm.resetFlags();
     angular.element('#noteImportModal').modal('hide');
   });
+
+  $scope.$on('importNoteResult', function(event, data) {
+    if (data.type === 'ipfs') {
+      var ngToastConf = {
+        content: 'Failed to import Note with hash ' + data.options.hash,
+        verticalPosition: 'bottom',
+        horizontalPosition: 'center',
+        timeout: '5000'
+      };
+
+      if (data.importStatus === 'success') {
+        ngToastConf.content = 'Successfully imported Note with hash ' + data.options.hash,
+        ngToast.success(ngToastConf);
+      } else {
+        ngToast.danger(ngToastConf);
+      }
+    }
+  });
+
 });

zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js

@@ -60,6 +60,8 @@ angular.module('zeppelinWebApp').factory('websocketEvents',
       $location.path('/notebook/' + data.note.id);
     } else if (op === 'NOTES_INFO') {
       $rootScope.$broadcast('setNoteMenu', data.notes);
+    } else if (op === 'IMPORT_NOTE_STATUS') {
+      $rootScope.$broadcast('importNoteResult', data);
     } else if (op === 'LIST_NOTEBOOK_JOBS') {
       $rootScope.$broadcast('setNotebookJobs', data.notebookJobs);
     } else if (op === 'LIST_UPDATE_NOTEBOOK_JOBS') {

zeppelin-web/src/components/websocketEvents/websocketMsg.service.js

@@ -151,6 +151,19 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope,
       });
     },
 
+    importNoteFromBackend: function(hashUrl, name, type) {
+      websocketEvents.sendNewEvent({
+        op: 'IMPORT_NOTE_URL',
+        data: {
+          type: type,
+          name: name,
+          options: {
+            hash: hashUrl
+          }
+        }
+      });
+    },
+
     checkpointNotebook: function(noteId, commitMessage) {
       websocketEvents.sendNewEvent({
         op: 'CHECKPOINT_NOTEBOOK',

zeppelin-zengine/pom.xml

@@ -249,6 +249,11 @@
       <scope>test</scope>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpmime</artifactId>
+      <version>4.3.6</version>
+    </dependency>
   </dependencies>
 
   <build>

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java

@@ -182,6 +182,39 @@ public String exportNote(String noteId) throws IOException, IllegalArgumentExcep
     return gson.toJson(note);
   }
 
+  /**
+   *
+   * @param url - the ipfs url or hash to get note from
+   * @param subject
+   * @return note
+   * @throws IOException
+   */
+  public Note importNoteFromBackend(String url, String noteName, AuthenticationInfo subject)
+      throws IOException {
+    Note newNote = null;
+    Note oldNote = notebookRepo.getNoteFromUrl(url, subject);
+    if (oldNote != null) {
+      try {
+        newNote = createNote(subject);
+        if (noteName != null) {
+          newNote.setName(noteName);
+        } else {
+          newNote.setName(oldNote.getName());
+        }
+        List<Paragraph> paragraphs = oldNote.getParagraphs();
+        for (Paragraph p : paragraphs) {
+          newNote.addCloneParagraph(p);
+        }
+        newNote.persist(subject);
+      } catch (IOException e) {
+        logger.error("Importing note from " + url + " failed", e);
+        throw e;
+      }
+    }
+
+    return newNote;
+  }
+
   /**
    * import JSON as a new note.
    *

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java

@@ -227,4 +227,10 @@ public List<Revision> revisionHistory(String noteId, AuthenticationInfo subject)
     // Auto-generated method stub
     return null;
   }
+
+  @Override
+  public Note getNoteFromUrl(String url, AuthenticationInfo subject) throws IOException{
+    // Auto-generated method stub
+    return null;
+  }
 }

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepo.java

@@ -100,6 +100,15 @@ public interface NotebookRepo {
    */
   @ZeppelinApi public List<Revision> revisionHistory(String noteId, AuthenticationInfo subject);
 
+  /**
+   * Download a note from a given URL or Hash
+   * @param url
+   * @param subject
+   * @return note in JSON String
+   */
+  @ZeppelinApi public Note getNoteFromUrl(String url, AuthenticationInfo subject)
+      throws IOException;
+
   /**
    * Represents the 'Revision' a point in life of the notebook
    */
@@ -112,6 +121,23 @@ public Revision(String revId, String message, int time) {
     public String id;
     public String message;
     public int time;
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (!(o instanceof Revision)) {
+        return false;
+      }
+      Revision revision = (Revision) o;
+      return id.equals(revision.id);
+    }
+
+    @Override
+    public int hashCode() {
+      return id.hashCode();
+    }
   }
 
 }

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java

@@ -410,4 +410,15 @@ public List<Revision> revisionHistory(String noteId, AuthenticationInfo subject)
     }
     return revisions;
   }
+
+  @Override
+  public Note getNoteFromUrl(String url, AuthenticationInfo subject) {
+    Note newNote = null;
+    try {
+      newNote = getRepo(0).getNoteFromUrl(url, subject);
+    } catch (IOException e) {
+      LOG.error("Failed to download note with given url", e);
+    }
+    return newNote;
+  }
 }

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java

@@ -270,4 +270,10 @@ public List<Revision> revisionHistory(String noteId, AuthenticationInfo subject)
     // Auto-generated method stub
     return null;
   }
+
+  @Override
+  public Note getNoteFromUrl(String url, AuthenticationInfo subject) throws IOException {
+    // Auto-generated method stub
+    return null;
+  }
 }

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java

@@ -285,4 +285,10 @@ public List<Revision> revisionHistory(String noteId, AuthenticationInfo subject)
     return null;
   }
 
+  @Override
+  public Note getNoteFromUrl(String url, AuthenticationInfo subject) throws IOException {
+    // Auto-generated method stub
+    return null;
+  }
+
 }