Notebook serving front-end

bin/interpreter.cmd

@@ -29,6 +29,7 @@ if /I "%~1"=="-d" (
 if /I "%~1"=="-p" set PORT=%~2
 if /I "%~1"=="-c" set CALLBACK_HOST=%~2
 if /I "%~1"=="-l" set LOCAL_INTERPRETER_REPO=%~2
+if /I "%~1"=="-s" set REST_API_SERVER_PORT=%~2
 shift
 goto loop
 :cont
@@ -128,11 +129,11 @@ if not defined ZEPPELIN_CLASSPATH_OVERRIDES (
 if defined SPARK_SUBMIT (
     set JAVA_INTP_OPTS=%JAVA_INTP_OPTS% -Dzeppelin.log.file='%ZEPPELIN_LOGFILE%'
 
-    "%SPARK_SUBMIT%" --class %ZEPPELIN_SERVER% --jars %CLASSPATH% --driver-java-options "!JAVA_INTP_OPTS!" %SPARK_SUBMIT_OPTIONS% "%SPARK_APP_JAR%" "%CALLBACK_HOST%" %PORT%
+    "%SPARK_SUBMIT%" --class %ZEPPELIN_SERVER% --jars %CLASSPATH% --driver-java-options "!JAVA_INTP_OPTS!" %SPARK_SUBMIT_OPTIONS% "%SPARK_APP_JAR%" "%CALLBACK_HOST%" %PORT% %REST_API_SERVER_PORT%
 ) else (
     set JAVA_INTP_OPTS=%JAVA_INTP_OPTS% -Dzeppelin.log.file="%ZEPPELIN_LOGFILE%"
 
-    "%ZEPPELIN_RUNNER%" !JAVA_INTP_OPTS! %ZEPPELIN_INTP_MEM% -cp %ZEPPELIN_CLASSPATH_OVERRIDES%;%CLASSPATH% %ZEPPELIN_SERVER% "%CALLBACK_HOST%" %PORT%
+    "%ZEPPELIN_RUNNER%" !JAVA_INTP_OPTS! %ZEPPELIN_INTP_MEM% -cp %ZEPPELIN_CLASSPATH_OVERRIDES%;%CLASSPATH% %ZEPPELIN_SERVER% "%CALLBACK_HOST%" %PORT% %REST_API_SERVER_PORT%
 )
 
 exit /b

bin/interpreter.sh

@@ -21,10 +21,10 @@ bin=$(dirname "${BASH_SOURCE-$0}")
 bin=$(cd "${bin}">/dev/null; pwd)
 
 function usage() {
-    echo "usage) $0 -p <port> -r <intp_port> -d <interpreter dir to load> -l <local interpreter repo dir to load> -g <interpreter group name>"
+    echo "usage) $0 -p <port> -r <intp_port> -d <interpreter dir to load> -l <local interpreter repo dir to load> -g <interpreter group name> -s <rest api server port>"
 }
 
-while getopts "hc:p:r:i:d:l:v:u:g:" o; do
+while getopts "hc:p:r:i:d:l:v:u:g:s:" o; do
     case ${o} in
         h)
             usage
@@ -58,6 +58,9 @@ while getopts "hc:p:r:i:d:l:v:u:g:" o; do
         g)
             INTERPRETER_SETTING_NAME=${OPTARG}
             ;;
+        s)
+            REST_API_SERVER_PORT=${OPTARG}
+            ;;
         esac
 done
 
@@ -234,9 +237,9 @@ if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]]; then
 fi
 
 if [[ -n "${SPARK_SUBMIT}" ]]; then
-    INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${ZEPPELIN_SPARK_CONF} ${SPARK_APP_JAR} ${CALLBACK_HOST} ${PORT} ${INTP_GROUP_ID} ${INTP_PORT}`
+    INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${ZEPPELIN_SPARK_CONF} ${SPARK_APP_JAR} ${CALLBACK_HOST} ${PORT} ${INTP_GROUP_ID} ${INTP_PORT} ${REST_API_SERVER_PORT}`
 else
-    INTERPRETER_RUN_COMMAND+=' '` echo ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH} ${ZEPPELIN_SERVER} ${CALLBACK_HOST} ${PORT} ${INTP_GROUP_ID} ${INTP_PORT}`
+    INTERPRETER_RUN_COMMAND+=' '` echo ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH} ${ZEPPELIN_SERVER} ${CALLBACK_HOST} ${PORT} ${INTP_GROUP_ID} ${INTP_PORT} ${REST_API_SERVER_PORT}`
 fi
 
 

bin/zeppelin.sh

@@ -19,20 +19,26 @@
 # Run Zeppelin 
 #
 
-USAGE="Usage: bin/zeppelin.sh [--config <conf-dir>]"
+USAGE="Usage: bin/zeppelin.sh [--config <conf-dir>] [--run <noteId>]"
 
-if [[ "$1" == "--config" ]]; then
-  shift
-  conf_dir="$1"
-  if [[ ! -d "${conf_dir}" ]]; then
-    echo "ERROR : ${conf_dir} is not a directory"
-    echo ${USAGE}
-    exit 1
-  else
-    export ZEPPELIN_CONF_DIR="${conf_dir}"
-  fi
-  shift
-fi
+POSITIONAL=()
+while [[ $# -gt 0 ]]
+do
+  key="$1"
+  case $key in
+    --config)
+    ZEPPELIN_CONF_DIR="$2"
+    shift # past argument
+    shift # past value
+    ;;
+    --run)
+    ZEPPELIN_NOTEBOOK_RUN="$2"
+    shift # past argument
+    shift # past value
+    ;;
+  esac
+done
+set -- "${POSITIONAL[@]}" # restore positional parameters
 
 bin=$(dirname "${BASH_SOURCE-$0}")
 bin=$(cd "${bin}">/dev/null; pwd)

k8s/background/zeppelin-background-server.yaml

@@ -0,0 +1,161 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{zeppelin.k8s.background.name}}-conf
+data:
+  # 'serviceDomain' is a Domain name to use for accessing Zeppelin UI.
+  # Should point IP address of 'zeppelin-server' service.
+  #
+  # Wildcard subdomain need to be point the same IP address to access service inside of Pod (such as SparkUI).
+  # i.e. if service domain is 'local.zeppelin-project.org', DNS configuration should make 'local.zeppelin-project.org' and '*.local.zeppelin-project.org' point the same address.
+  #
+  # Default value is 'local.zeppelin-project.org' while it points 127.0.0.1 and `kubectl port-forward zeppelin-server` will give localhost to connects.
+  # If you have your ingress controller configured to connect to `zeppelin-server` service and have a domain name for it (with wildcard subdomain point the same address), you can replace serviceDomain field with your own domain.
+  serviceDomain: local.zeppelin-project.org:8080
+  sparkContainerImage: spark:2.4.0
+  redisAddr: zeppelin-serving-metric-redis.default.svc.cluster.local # metric storage
+---
+apiVersion: {{zeppelin.k8s.background.resource.apiversion}}
+kind: {{zeppelin.k8s.background.resource.type}}
+metadata:
+  namespace: {{zeppelin.k8s.background.namespace}}
+  name: {{zeppelin.k8s.background.name}}
+  labels:
+    taskType: {{zeppelin.k8s.background.type}}
+spec:
+{% if zeppelin.k8s.background.resource.type == "Deployment" %}
+  selector:
+    matchLabels:
+      app: {{zeppelin.k8s.background.name}}
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: {{zeppelin.k8s.background.name}}
+    spec:
+{% endif %}
+{% if zeppelin.k8s.background.resource.type == "Job" %}
+  backoffLimit: 1
+  activeDeadlineSeconds: 3600
+  template:
+    metadata:
+      labels:
+        app: {{zeppelin.k8s.background.name}}
+    spec:
+      restartPolicy: Never
+{% endif %}
+      automountServiceAccountToken: true
+      containers:
+      - name: zeppelin-server
+        image: apache/zeppelin:0.9.0-SNAPSHOT
+        command: ["sh", "-c", "$(ZEPPELIN_HOME)/bin/zeppelin.sh"]
+        lifecycle:
+          preStop:
+            exec:
+              # SIGTERM triggers a quick exit; gracefully terminate instead
+              command: ["sh", "-c", "ps -ef | grep org.apache.zeppelin.server.ZeppelinServer | grep -v grep | awk '{print $2}' | xargs kill"]
+        env:
+        - name: ZEPPELIN_K8S_CONTAINER_IMAGE
+          value: apache/zeppelin:0.9.0-SNAPSHOT
+        - name: ZEPPELIN_HOME
+          value: /zeppelin
+        - name: ZEPPELIN_SERVER_RPC_PORTRANGE
+          value: 12320:12320
+        - name: POD_UID
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.uid
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.name
+        - name: SERVICE_NAME
+          value: {{zeppelin.k8s.background.name}}
+        - name: ZEPPELIN_K8S_SPARK_CONTAINER_IMAGE
+          valueFrom:
+            configMapKeyRef:
+              name: {{zeppelin.k8s.background.name}}-conf
+              key: sparkContainerImage
+        - name: SERVICE_DOMAIN
+          valueFrom:
+            configMapKeyRef:
+              name: {{zeppelin.k8s.background.name}}-conf
+              key: serviceDomain
+        - name: ZEPPELIN_INTERPRETER_METRIC_REDIS_ADDR
+          value: "{{zeppelin.k8s.serving.metric.redis.addr}}"
+        - name: ZEPPELIN_NOTEBOOK_DIR
+          value: "{{zeppelin.k8s.background.notebook.dir}}"
+        - name: ZEPPELIN_NOTEBOOK_RUN_ID
+          value: "{{zeppelin.k8s.background.noteId}}"
+        - name: ZEPPELIN_NOTEBOOK_RUN_REV
+          value: "{{zeppelin.k8s.background.revId}}"
+        - name: ZEPPELIN_NOTEBOOK_RUN_SERVICE_CONTEXT
+          value: "{{zeppelin.k8s.background.serviceContext}}"
+        - name: ZEPPELIN_NOTEBOOK_RUN_AUTOSHUTDOWN
+          value: "{{zeppelin.k8s.background.autoshutdown}}"
+        - name: MASTER   # default value of master property for spark interpreter.
+          value: "k8s://https://kubernetes.default.svc"
+        - name: ZEPPELIN_BACKGROUND_TYPE
+          value: "{{zeppelin.k8s.background.type}}"
+        volumeMounts:
+          - name: zeppelin-task-context-volume
+            mountPath: /zeppelin/task
+      volumes:
+      - name: zeppelin-task-context-volume
+        persistentVolumeClaim:
+          claimName: task-context-volume-claim
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: {{zeppelin.k8s.background.name}}    # keep Service name the same to Pod name.
+spec:
+  ports:
+    - name: http
+      port: 80
+    - name: rpc            # port name is referenced in the code. So it shouldn't be changed.
+      port: 12320
+  selector:
+    app: {{zeppelin.k8s.background.name}}
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{zeppelin.k8s.background.name}}-role
+rules:
+- apiGroups: [""]
+  resources: ["pods", "services"]
+  verbs: ["create", "get", "update", "patch", "list", "delete", "watch"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["roles", "rolebindings"]
+  verbs: ["bind", "create", "get", "update", "patch", "list", "delete", "watch"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{zeppelin.k8s.background.name}}-role-binding
+subjects:
+- kind: ServiceAccount
+  name: default
+roleRef:
+  kind: Role
+  name: {{zeppelin.k8s.background.name}}-role
+  apiGroup: rbac.authorization.k8s.io

k8s/bin/update_routing_table.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import sys, os, json, time, shutil
+from subprocess import check_output
+
+SERVING_PORT=8090
+TMP_FILE_PATH="/tmp/z_api_route_location"
+
+def genLocation(namespace, serviceName, noteId, revId, endpoint):
+  location = ""
+  # print("serviceName=" + serviceName + ", noteId=" + noteId + ", revId=" + revId + ", endpoint=" + endpoint)
+  location += "location /serving/{}/{}/{} {{".format(noteId, revId, endpoint)
+  location += "\n  resolver 127.0.0.1:53 ipv6=off;"
+  location += "\n  rewrite ^/serving/[^/]*/[^/]*/(.*) /$1 break;"
+  location += "\n  proxy_pass http://{}.{}.svc.cluster.local:{};".format(serviceName, namespace, SERVING_PORT)
+  location += "\n  proxy_set_header Host $host;"
+  location += "\n  proxy_http_version 1.1;"
+  location += "\n  proxy_set_header Upgrade $http_upgrade;"
+  location += "\n  proxy_set_header Connection $connection_upgrade;"
+  location += "\n}\n"
+  return location
+
+def main(locationWritePath):
+  # get all Services has 'noteId' in label key.
+  out = check_output(["kubectl", "get", "services", "-l", "noteId", "-o", "json"])
+  services = json.loads(out)
+
+  locations = []
+
+  for service in services['items']:
+    labels = service["metadata"]["labels"]
+    noteId = labels["noteId"]
+    revId = labels["revId"]
+    serviceName = service["metadata"]["name"]
+    namespace = service["metadata"]["namespace"]
+
+    # iterate all endpoints
+    for key in labels:
+      if key.startswith("endpoint-"):
+        endpoint = labels[key]
+        locations.append(genLocation(namespace, serviceName, noteId, revId, endpoint))
+
+  # write generated nginx location block to file
+  locationConf = "\n".join(locations)
+
+  locationFile = open(TMP_FILE_PATH, "w")
+  locationFile.write(locationConf)
+  locationFile.close()
+
+  shutil.copyfile(TMP_FILE_PATH, locationWritePath)
+
+if __name__== "__main__":
+  intervalSec = float(sys.argv[1])
+  locationWritePath = sys.argv[2]
+  while True:
+    main(locationWritePath)
+    time.sleep(intervalSec)

k8s/interpreter/100-interpreter-spec.yaml

@@ -52,7 +52,7 @@ spec:
     env:
   {% for key, value in zeppelin.k8s.envs.items() %}
     - name: {{key}}
-      value: {{value}}
+      value: "{{value}}"
   {% endfor %}
   {% if zeppelin.k8s.interpreter.group.name == "spark" %}
     volumeMounts: