[ZEPPELIN-1706] Make spark Ui accesible in various interpreter modes

.travis.yml

@@ -84,7 +84,7 @@ before_install:
   - source ~/.environ
 
 install:
-  - mvn $BUILD_FLAG $PROFILE -B
+  - mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=warn $BUILD_FLAG $PROFILE -B
 
 before_script:
   - travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java

@@ -27,6 +27,7 @@
 import org.apache.zeppelin.interpreter.InterpreterGroup;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.RemoteZeppelinServerResource;
+import org.apache.zeppelin.interpreter.InterpreterOption;
 import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent;
 import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType;
 import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
@@ -235,10 +236,10 @@ public void run() {
           Map<String, String> metaInfos = gson.fromJson(event.getData(),
               new TypeToken<Map<String, String>>() {
               }.getType());
-          String id = interpreterGroup.getId();
-          int indexOfColon = id.indexOf(":");
-          String settingId = id.substring(0, indexOfColon);
-          listener.onMetaInfosReceived(settingId, metaInfos);
+          String intpGroupId = interpreterGroup.getId();
+          int indexOfColon = intpGroupId.indexOf(":");
+          String settingId = intpGroupId.substring(0, indexOfColon);
+          listener.onMetaInfosReceived(settingId, intpGroupId, metaInfos);
         }
         logger.debug("Event from remote process {}", event.getType());
       } catch (Exception e) {

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessListener.java

@@ -28,7 +28,8 @@ public interface RemoteInterpreterProcessListener {
   public void onOutputUpdated(
       String noteId, String paragraphId, int index, InterpreterResult.Type type, String output);
   public void onOutputClear(String noteId, String paragraphId);
-  public void onMetaInfosReceived(String settingId, Map<String, String> metaInfos);
+  public void onMetaInfosReceived(String settingId, String intpGrpId,
+      Map<String, String> metaInfos);
   public void onRemoteRunParagraph(String noteId, String ParagraphID) throws Exception;
   public void onGetParagraphRunners(
       String noteId, String paragraphId, RemoteWorksEventListener callback);

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java

@@ -168,7 +168,8 @@ public void onOutputClear(String noteId, String paragraphId) {
   }
 
   @Override
-  public void onMetaInfosReceived(String settingId, Map<String, String> metaInfos) {
+  public void onMetaInfosReceived(String settingId, String intpGrpId,
+                                           Map<String, String> metaInfos) {
 
   }
 

zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java

@@ -310,7 +310,8 @@ public void onOutputClear(String noteId, String paragraphId) {
   }
 
   @Override
-  public void onMetaInfosReceived(String settingId, Map<String, String> metaInfos) {
+  public void onMetaInfosReceived(String settingId, String intpGrpId,
+                                          Map<String, String> metaInfos) {
 
   }
 

zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java

@@ -23,15 +23,14 @@
 import java.util.Map;
 import java.util.Properties;
 
-import javax.servlet.http.HttpServletRequest;
 import javax.ws.rs.DELETE;
 import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.PUT;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
-import javax.ws.rs.core.Context;
+import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.Response.Status;
 
@@ -47,6 +46,7 @@
 import org.apache.zeppelin.interpreter.InterpreterException;
 import org.apache.zeppelin.interpreter.InterpreterFactory;
 import org.apache.zeppelin.interpreter.InterpreterSetting;
+import org.apache.zeppelin.rest.message.MetaInfosInterpreterRequest;
 import org.apache.zeppelin.rest.message.NewInterpreterSettingRequest;
 import org.apache.zeppelin.rest.message.UpdateInterpreterSettingRequest;
 import org.apache.zeppelin.server.JsonResponse;
@@ -245,15 +245,16 @@ public Response addRepository(String message) {
    */
   @GET
   @Path("getmetainfos/{settingId}")
-  public Response getMetaInfo(@Context HttpServletRequest req,
-                              @PathParam("settingId") String settingId) {
-    String propName = req.getParameter("propName");
+  public Response getMetaInfo(String message,
+       @PathParam("settingId") String settingId, @QueryParam("noteId") String noteId,
+       @QueryParam("subject")String subject, @QueryParam("property") String propName) {
+
     if (propName == null) {
       return new JsonResponse<>(Status.BAD_REQUEST).build();
     }
     String propValue = null;
     InterpreterSetting interpreterSetting = interpreterFactory.get(settingId);
-    Map<String, String> infos = interpreterSetting.getInfos();
+    Map<String, String> infos = interpreterSetting.getInfos(subject, noteId);
     if (infos != null) {
       propValue = infos.get(propName);
     }

zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/MetaInfosInterpreterRequest.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.rest.message;
+
+/**
+ * Meta infos  rest api request message
+ */
+public class MetaInfosInterpreterRequest {
+  String noteId;
+  String subject;
+  String property;
+
+  public MetaInfosInterpreterRequest() {
+
+  }
+
+  public String getNoteId() {
+    return noteId;
+  }
+
+  public String getSubject() {
+    return subject;
+  }
+
+  public String getProperty() {
+    return property;
+  }
+}

zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java

@@ -1931,10 +1931,11 @@ private void getInterpreterSettings(NotebookSocket conn, AuthenticationInfo subj
   }
 
   @Override
-  public void onMetaInfosReceived(String settingId, Map<String, String> metaInfos) {
+  public void onMetaInfosReceived(String settingId, String intpGrpId,
+                                          Map<String, String> metaInfos) {
     InterpreterSetting interpreterSetting = notebook().getInterpreterFactory()
         .get(settingId);
-    interpreterSetting.setInfos(metaInfos);
+    interpreterSetting.setInfos(intpGrpId, metaInfos);
   }
 
   private void switchConnectionToWatcher(NotebookSocket conn, Message messagereceived)

zeppelin-server/src/main/java/org/apache/zeppelin/types/InterpreterSettingsList.java

@@ -29,12 +29,14 @@ public class InterpreterSettingsList {
   private String name;
   private boolean selected;
   private List<InterpreterInfo> interpreters;
+  private String group;
 
   public InterpreterSettingsList(String id, String name,
-      List<InterpreterInfo> interpreters, boolean selected) {
+      List<InterpreterInfo> interpreters, boolean selected, String group) {
     this.id = id;
     this.name = name;
     this.interpreters = interpreters;
     this.selected = selected;
+    this.group = group;
   }
 }

zeppelin-server/src/main/java/org/apache/zeppelin/utils/InterpreterBindingUtils.java

@@ -34,7 +34,7 @@ public static List<InterpreterSettingsList> getInterpreterBindings(Notebook note
         notebook.getBindedInterpreterSettings(noteId);
     for (InterpreterSetting setting : selectedSettings) {
       settingList.add(new InterpreterSettingsList(setting.getId(), setting.getName(),
-          setting.getInterpreterInfos(), true));
+          setting.getInterpreterInfos(), true, setting.getGroup()));
     }
 
     List<InterpreterSetting> availableSettings = notebook.getInterpreterFactory().get();
@@ -49,7 +49,7 @@ public static List<InterpreterSettingsList> getInterpreterBindings(Notebook note
 
       if (!selected) {
         settingList.add(new InterpreterSettingsList(setting.getId(), setting.getName(),
-            setting.getInterpreterInfos(), false));
+            setting.getInterpreterInfos(), false, setting.getGroup()));
       }
     }
 

zeppelin-web/package.json

@@ -7,8 +7,7 @@
     "postinstall": "node_modules/.bin/bower install --silent",
     "build": "./node_modules/.bin/grunt build",
     "start": "./node_modules/.bin/grunt serve",
-    "test": "./node_modules/.bin/grunt test",
-    "pretest": "./node_modules/.bin/npm install karma-phantomjs-launcher"
+    "test": "./node_modules/.bin/grunt test"
   },
   "dependencies": {
     "grunt-angular-templates": "^0.5.7",
@@ -48,6 +47,7 @@
     "karma": "~1.3.0",
     "karma-coverage": "^1.1.1",
     "karma-jasmine": "~1.0.2",
+    "karma-phantomjs-launcher": "~1.0.2",
     "load-grunt-tasks": "^0.4.0",
     "time-grunt": "^0.3.1"
   },

zeppelin-web/pom.xml

@@ -108,8 +108,8 @@
               <goal>install-node-and-npm</goal>
             </goals>
             <configuration>
-              <nodeVersion>${node.version}</nodeVersion>
-              <npmVersion>${npm.version}</npmVersion>
+              <nodeVersion>v6.9.1</nodeVersion>
+              <npmVersion>3.10.8</npmVersion>
             </configuration>
           </execution>
 

zeppelin-web/src/app/interpreter/interpreter.controller.js

@@ -700,7 +700,12 @@
     };
 
     $scope.showSparkUI = function(settingId) {
-      $http.get(baseUrlSrv.getRestApiBase() + '/interpreter/getmetainfos/' + settingId + '?propName=url')
+      var userName;
+      if ($rootScope.ticket) {
+        userName = $rootScope.ticket.principal;
+      }
+      $http.get(baseUrlSrv.getRestApiBase() + '/interpreter/getmetainfos/' + settingId + '?subject=' +
+         userName + '&property=url')
         .success(function(data, status, headers, config) {
           var url = data.body.url;
           if (!url) {

zeppelin-web/src/app/interpreter/interpreter.html

@@ -130,7 +130,8 @@ <h3 class="interpreter-title">{{setting.name}}
         <span style="float:right" ng-show="!valueform.$visible" >
           <button class="btn btn-default btn-xs"
                   ng-click="showSparkUI(setting.id)"
-                  ng-show="setting.group == 'spark'">
+                  ng-show="setting.group == 'spark' && (setting.option.perNote == 'scoped' ||
+                  setting.option.perNote == 'shared')">
             <span class="fa fa-external-link"></span> spark ui</button>
           <button class="btn btn-default btn-xs"
                   ng-click="valueform.$show();

zeppelin-web/src/app/notebook/notebook.controller.js

@@ -606,8 +606,13 @@
         message: 'Do you want to restart ' + interpeter.name + ' interpreter?',
         callback: function(result) {
           if (result) {
+            var userName;
+            if ($rootScope.ticket) {
+              userName = $rootScope.ticket.principal;
+            }
             var payload  = {
-              'noteId': $scope.note.id
+              'noteId': $scope.note.id,
+              'subject': userName
             };
 
             thisConfirm.$modalFooter.find('button').addClass('disabled');
@@ -633,6 +638,27 @@
       });
     };
 
+    $scope.getSparkUrl = function(interpeter) {
+      var userName;
+      if ($rootScope.ticket) {
+        userName = $rootScope.ticket.principal;
+      }
+      $http.get(baseUrlSrv.getRestApiBase() + '/interpreter/getmetainfos/' + interpeter.id + '?noteId=' +
+         $scope.note.id + '&subject=' + userName + '&property=url')
+        .success(function(data, status, headers, config) {
+          var url = data.body.url;
+          if (!url) {
+            BootstrapDialog.alert({
+              message: 'No spark application running'
+            });
+            return;
+          }
+          window.open(url, '_blank');
+        }).error(function(data, status, headers, config) {
+         console.log('Error %o %o', status, data.message);
+       });
+    };
+
     $scope.savePermissions = function() {
       convertPermissionsToArray();
       $http.put(baseUrlSrv.getRestApiBase() + '/notebook/' + $scope.note.id + '/permissions',

zeppelin-web/src/app/notebook/notebook.html

@@ -37,6 +37,12 @@ <h5>Interpreter binding</h5>
                ng-class="{'inactivelink': !item.selected}"
                tooltip="Restart">
               <span class="glyphicon glyphicon-refresh btn-md"></span>
+            </a>&nbsp
+             <a ng-click="getSparkUrl(item)"
+               ng-class="{'inactivelink': !item.selected}"
+               ng-show="item.group == 'spark'"
+               tooltip="Spark UI">
+              <span class="glyphicon glyphicon-share btn-md"></span>
             </a>&nbsp
             <div as-sortable-item-handle
                  ng-click="item.selected = !item.selected"

zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java

@@ -1001,8 +1001,6 @@ public void restart(String id) {
       // Check if dependency in specified path is changed
       // If it did, overwrite old dependency jar with new one
       if (intpSetting != null) {
-        //clean up metaInfos
-        intpSetting.setInfos(null);
         copyDependenciesFromLocalPath(intpSetting);
 
         stopJobAllInterpreter(intpSetting);

zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java

@@ -44,7 +44,7 @@ public class InterpreterSetting {
   private String name;
   // always be null in case of InterpreterSettingRef
   private String group;
-  private transient Map<String, String> infos;
+  private transient Map<String, Map<String, String>> IntpGrpKeyToinfosMap;
 
   /**
    * properties can be either Properties or Map<String, InterpreterProperty>
@@ -115,7 +115,7 @@ public String getName() {
     return name;
   }
 
-  String getGroup() {
+  public String getGroup() {
     return group;
   }
 
@@ -177,6 +177,8 @@ void closeAndRemoveInterpreterGroup(String noteId) {
     }
 
     if (groupToRemove != null) {
+      String intpGrpId = groupToRemove.getId();
+      setInfos(intpGrpId, null);
       groupToRemove.close();
       groupToRemove.destroy();
     }
@@ -281,11 +283,27 @@ public void setErrorReason(String errorReason) {
     this.errorReason = errorReason;
   }
 
-  public void setInfos(Map<String, String> infos) {
-    this.infos = infos;
+  public void setInfos(String intpGrpKey, Map<String, String> infos) {
+    if (IntpGrpKeyToinfosMap == null) {
+      IntpGrpKeyToinfosMap = new HashMap<>();
+    }
+    if (infos == null) {
+      IntpGrpKeyToinfosMap.remove(intpGrpKey);
+    } else if (!infos.isEmpty()) {
+      IntpGrpKeyToinfosMap.put(intpGrpKey, infos);
+    }
   }
 
-  public Map<String, String> getInfos() {
+  public Map<String, String> getInfos(String intpGrpId) {
+    Map<String, String> infos = null;
+    if (IntpGrpKeyToinfosMap != null) {
+      infos = IntpGrpKeyToinfosMap.get(intpGrpId);
+    }
     return infos;
   }
+
+  public Map<String, String> getInfos(String user, String noteId) {
+    String interpreterProcessKey = getInterpreterProcessKey(user, noteId);
+    return getInfos(getId() + ":" + interpreterProcessKey);
+  }
 }