diff --git a/bin/interpreter.cmd b/bin/interpreter.cmd index 8877c45409f..6ae2737716b 100644 --- a/bin/interpreter.cmd +++ b/bin/interpreter.cmd @@ -29,6 +29,7 @@ if /I "%~1"=="-d" ( if /I "%~1"=="-p" set PORT=%~2 if /I "%~1"=="-c" set CALLBACK_HOST=%~2 if /I "%~1"=="-l" set LOCAL_INTERPRETER_REPO=%~2 +if /I "%~1"=="-s" set REST_API_SERVER_PORT=%~2 shift goto loop :cont @@ -128,11 +129,11 @@ if not defined ZEPPELIN_CLASSPATH_OVERRIDES ( if defined SPARK_SUBMIT ( set JAVA_INTP_OPTS=%JAVA_INTP_OPTS% -Dzeppelin.log.file='%ZEPPELIN_LOGFILE%' - "%SPARK_SUBMIT%" --class %ZEPPELIN_SERVER% --jars %CLASSPATH% --driver-java-options "!JAVA_INTP_OPTS!" %SPARK_SUBMIT_OPTIONS% "%SPARK_APP_JAR%" "%CALLBACK_HOST%" %PORT% + "%SPARK_SUBMIT%" --class %ZEPPELIN_SERVER% --jars %CLASSPATH% --driver-java-options "!JAVA_INTP_OPTS!" %SPARK_SUBMIT_OPTIONS% "%SPARK_APP_JAR%" "%CALLBACK_HOST%" %PORT% %REST_API_SERVER_PORT% ) else ( set JAVA_INTP_OPTS=%JAVA_INTP_OPTS% -Dzeppelin.log.file="%ZEPPELIN_LOGFILE%" - "%ZEPPELIN_RUNNER%" !JAVA_INTP_OPTS! %ZEPPELIN_INTP_MEM% -cp %ZEPPELIN_CLASSPATH_OVERRIDES%;%CLASSPATH% %ZEPPELIN_SERVER% "%CALLBACK_HOST%" %PORT% + "%ZEPPELIN_RUNNER%" !JAVA_INTP_OPTS! %ZEPPELIN_INTP_MEM% -cp %ZEPPELIN_CLASSPATH_OVERRIDES%;%CLASSPATH% %ZEPPELIN_SERVER% "%CALLBACK_HOST%" %PORT% %REST_API_SERVER_PORT% ) exit /b diff --git a/bin/interpreter.sh b/bin/interpreter.sh index 0bbfd0c0c8e..8ccd0d04220 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -21,10 +21,10 @@ bin=$(dirname "${BASH_SOURCE-$0}") bin=$(cd "${bin}">/dev/null; pwd) function usage() { - echo "usage) $0 -p -r -d -l -g " + echo "usage) $0 -p -r -d -l -g -s " } -while getopts "hc:p:r:i:d:l:v:u:g:" o; do +while getopts "hc:p:r:i:d:l:v:u:g:s:" o; do case ${o} in h) usage @@ -58,6 +58,9 @@ while getopts "hc:p:r:i:d:l:v:u:g:" o; do g) INTERPRETER_SETTING_NAME=${OPTARG} ;; + s) + REST_API_SERVER_PORT=${OPTARG} + ;; esac done @@ -234,9 +237,9 @@ if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]]; then fi if [[ -n "${SPARK_SUBMIT}" ]]; then - INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${ZEPPELIN_SPARK_CONF} ${SPARK_APP_JAR} ${CALLBACK_HOST} ${PORT} ${INTP_GROUP_ID} ${INTP_PORT}` + INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${ZEPPELIN_SPARK_CONF} ${SPARK_APP_JAR} ${CALLBACK_HOST} ${PORT} ${INTP_GROUP_ID} ${INTP_PORT} ${REST_API_SERVER_PORT}` else - INTERPRETER_RUN_COMMAND+=' '` echo ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH} ${ZEPPELIN_SERVER} ${CALLBACK_HOST} ${PORT} ${INTP_GROUP_ID} ${INTP_PORT}` + INTERPRETER_RUN_COMMAND+=' '` echo ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH} ${ZEPPELIN_SERVER} ${CALLBACK_HOST} ${PORT} ${INTP_GROUP_ID} ${INTP_PORT} ${REST_API_SERVER_PORT}` fi diff --git a/bin/zeppelin.sh b/bin/zeppelin.sh index a13f9db977d..7cac192ef30 100755 --- a/bin/zeppelin.sh +++ b/bin/zeppelin.sh @@ -19,20 +19,26 @@ # Run Zeppelin # -USAGE="Usage: bin/zeppelin.sh [--config ]" +USAGE="Usage: bin/zeppelin.sh [--config ] [--run ]" -if [[ "$1" == "--config" ]]; then - shift - conf_dir="$1" - if [[ ! -d "${conf_dir}" ]]; then - echo "ERROR : ${conf_dir} is not a directory" - echo ${USAGE} - exit 1 - else - export ZEPPELIN_CONF_DIR="${conf_dir}" - fi - shift -fi +POSITIONAL=() +while [[ $# -gt 0 ]] +do + key="$1" + case $key in + --config) + ZEPPELIN_CONF_DIR="$2" + shift # past argument + shift # past value + ;; + --run) + ZEPPELIN_NOTEBOOK_RUN="$2" + shift # past argument + shift # past value + ;; + esac +done +set -- "${POSITIONAL[@]}" # restore positional parameters bin=$(dirname "${BASH_SOURCE-$0}") bin=$(cd "${bin}">/dev/null; pwd) diff --git a/k8s/background/zeppelin-background-server.yaml b/k8s/background/zeppelin-background-server.yaml new file mode 100644 index 00000000000..40bc9925ee6 --- /dev/null +++ b/k8s/background/zeppelin-background-server.yaml @@ -0,0 +1,161 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{zeppelin.k8s.background.name}}-conf +data: + # 'serviceDomain' is a Domain name to use for accessing Zeppelin UI. + # Should point IP address of 'zeppelin-server' service. + # + # Wildcard subdomain need to be point the same IP address to access service inside of Pod (such as SparkUI). + # i.e. if service domain is 'local.zeppelin-project.org', DNS configuration should make 'local.zeppelin-project.org' and '*.local.zeppelin-project.org' point the same address. + # + # Default value is 'local.zeppelin-project.org' while it points 127.0.0.1 and `kubectl port-forward zeppelin-server` will give localhost to connects. + # If you have your ingress controller configured to connect to `zeppelin-server` service and have a domain name for it (with wildcard subdomain point the same address), you can replace serviceDomain field with your own domain. + serviceDomain: local.zeppelin-project.org:8080 + sparkContainerImage: spark:2.4.0 + redisAddr: zeppelin-serving-metric-redis.default.svc.cluster.local # metric storage +--- +apiVersion: {{zeppelin.k8s.background.resource.apiversion}} +kind: {{zeppelin.k8s.background.resource.type}} +metadata: + namespace: {{zeppelin.k8s.background.namespace}} + name: {{zeppelin.k8s.background.name}} + labels: + taskType: {{zeppelin.k8s.background.type}} +spec: +{% if zeppelin.k8s.background.resource.type == "Deployment" %} + selector: + matchLabels: + app: {{zeppelin.k8s.background.name}} + replicas: 1 + template: + metadata: + labels: + app: {{zeppelin.k8s.background.name}} + spec: +{% endif %} +{% if zeppelin.k8s.background.resource.type == "Job" %} + backoffLimit: 1 + activeDeadlineSeconds: 3600 + template: + metadata: + labels: + app: {{zeppelin.k8s.background.name}} + spec: + restartPolicy: Never +{% endif %} + automountServiceAccountToken: true + containers: + - name: zeppelin-server + image: apache/zeppelin:0.9.0-SNAPSHOT + command: ["sh", "-c", "$(ZEPPELIN_HOME)/bin/zeppelin.sh"] + lifecycle: + preStop: + exec: + # SIGTERM triggers a quick exit; gracefully terminate instead + command: ["sh", "-c", "ps -ef | grep org.apache.zeppelin.server.ZeppelinServer | grep -v grep | awk '{print $2}' | xargs kill"] + env: + - name: ZEPPELIN_K8S_CONTAINER_IMAGE + value: apache/zeppelin:0.9.0-SNAPSHOT + - name: ZEPPELIN_HOME + value: /zeppelin + - name: ZEPPELIN_SERVER_RPC_PORTRANGE + value: 12320:12320 + - name: POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: SERVICE_NAME + value: {{zeppelin.k8s.background.name}} + - name: ZEPPELIN_K8S_SPARK_CONTAINER_IMAGE + valueFrom: + configMapKeyRef: + name: {{zeppelin.k8s.background.name}}-conf + key: sparkContainerImage + - name: SERVICE_DOMAIN + valueFrom: + configMapKeyRef: + name: {{zeppelin.k8s.background.name}}-conf + key: serviceDomain + - name: ZEPPELIN_INTERPRETER_METRIC_REDIS_ADDR + value: "{{zeppelin.k8s.serving.metric.redis.addr}}" + - name: ZEPPELIN_NOTEBOOK_DIR + value: "{{zeppelin.k8s.background.notebook.dir}}" + - name: ZEPPELIN_NOTEBOOK_RUN_ID + value: "{{zeppelin.k8s.background.noteId}}" + - name: ZEPPELIN_NOTEBOOK_RUN_REV + value: "{{zeppelin.k8s.background.revId}}" + - name: ZEPPELIN_NOTEBOOK_RUN_SERVICE_CONTEXT + value: "{{zeppelin.k8s.background.serviceContext}}" + - name: ZEPPELIN_NOTEBOOK_RUN_AUTOSHUTDOWN + value: "{{zeppelin.k8s.background.autoshutdown}}" + - name: MASTER # default value of master property for spark interpreter. + value: "k8s://https://kubernetes.default.svc" + - name: ZEPPELIN_BACKGROUND_TYPE + value: "{{zeppelin.k8s.background.type}}" + volumeMounts: + - name: zeppelin-task-context-volume + mountPath: /zeppelin/task + volumes: + - name: zeppelin-task-context-volume + persistentVolumeClaim: + claimName: task-context-volume-claim +--- +kind: Service +apiVersion: v1 +metadata: + name: {{zeppelin.k8s.background.name}} # keep Service name the same to Pod name. +spec: + ports: + - name: http + port: 80 + - name: rpc # port name is referenced in the code. So it shouldn't be changed. + port: 12320 + selector: + app: {{zeppelin.k8s.background.name}} +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{zeppelin.k8s.background.name}}-role +rules: +- apiGroups: [""] + resources: ["pods", "services"] + verbs: ["create", "get", "update", "patch", "list", "delete", "watch"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings"] + verbs: ["bind", "create", "get", "update", "patch", "list", "delete", "watch"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{zeppelin.k8s.background.name}}-role-binding +subjects: +- kind: ServiceAccount + name: default +roleRef: + kind: Role + name: {{zeppelin.k8s.background.name}}-role + apiGroup: rbac.authorization.k8s.io diff --git a/k8s/bin/update_routing_table.py b/k8s/bin/update_routing_table.py new file mode 100755 index 00000000000..93ffef472fb --- /dev/null +++ b/k8s/bin/update_routing_table.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import sys, os, json, time, shutil +from subprocess import check_output + +SERVING_PORT=8090 +TMP_FILE_PATH="/tmp/z_api_route_location" + +def genLocation(namespace, serviceName, noteId, revId, endpoint): + location = "" + # print("serviceName=" + serviceName + ", noteId=" + noteId + ", revId=" + revId + ", endpoint=" + endpoint) + location += "location /serving/{}/{}/{} {{".format(noteId, revId, endpoint) + location += "\n resolver 127.0.0.1:53 ipv6=off;" + location += "\n rewrite ^/serving/[^/]*/[^/]*/(.*) /$1 break;" + location += "\n proxy_pass http://{}.{}.svc.cluster.local:{};".format(serviceName, namespace, SERVING_PORT) + location += "\n proxy_set_header Host $host;" + location += "\n proxy_http_version 1.1;" + location += "\n proxy_set_header Upgrade $http_upgrade;" + location += "\n proxy_set_header Connection $connection_upgrade;" + location += "\n}\n" + return location + +def main(locationWritePath): + # get all Services has 'noteId' in label key. + out = check_output(["kubectl", "get", "services", "-l", "noteId", "-o", "json"]) + services = json.loads(out) + + locations = [] + + for service in services['items']: + labels = service["metadata"]["labels"] + noteId = labels["noteId"] + revId = labels["revId"] + serviceName = service["metadata"]["name"] + namespace = service["metadata"]["namespace"] + + # iterate all endpoints + for key in labels: + if key.startswith("endpoint-"): + endpoint = labels[key] + locations.append(genLocation(namespace, serviceName, noteId, revId, endpoint)) + + # write generated nginx location block to file + locationConf = "\n".join(locations) + + locationFile = open(TMP_FILE_PATH, "w") + locationFile.write(locationConf) + locationFile.close() + + shutil.copyfile(TMP_FILE_PATH, locationWritePath) + +if __name__== "__main__": + intervalSec = float(sys.argv[1]) + locationWritePath = sys.argv[2] + while True: + main(locationWritePath) + time.sleep(intervalSec) diff --git a/k8s/interpreter/100-interpreter-spec.yaml b/k8s/interpreter/100-interpreter-spec.yaml index c857ff2cb80..b814cf52e50 100644 --- a/k8s/interpreter/100-interpreter-spec.yaml +++ b/k8s/interpreter/100-interpreter-spec.yaml @@ -52,7 +52,7 @@ spec: env: {% for key, value in zeppelin.k8s.envs.items() %} - name: {{key}} - value: {{value}} + value: "{{value}}" {% endfor %} {% if zeppelin.k8s.interpreter.group.name == "spark" %} volumeMounts: diff --git a/k8s/serving-api-router/serving-api-router.yaml b/k8s/serving-api-router/serving-api-router.yaml new file mode 100644 index 00000000000..93a922199a5 --- /dev/null +++ b/k8s/serving-api-router/serving-api-router.yaml @@ -0,0 +1,233 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: zeppelin-serving-api-router-conf +data: + serviceDomain: local.zeppelin-project.org:8080 + nginx_apply_routing_table_and_reload.sh: | + #!/bin/sh + nginx & + while : + do + echo "" > /tmp/nginx.conf + sed -n '/SERVING_ROUTING_TABLE/!p;//q' /tmp/conf/nginx.conf >> /tmp/nginx.conf + if [ -f "/routing_table/location" ]; then + cat /routing_table/location >> /tmp/nginx.conf + fi + sed -e '1,/SERVING_ROUTING_TABLE/ d' /tmp/conf/nginx.conf >> /tmp/nginx.conf + sed -i -e "s/SERVICE_DOMAIN/$(cat /tmp/conf/serviceDomain)/g" /tmp/nginx.conf; + sed -i -e "s/NAMESPACE/$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)/g" /tmp/nginx.conf; + + diff /tmp/nginx.conf /etc/nginx/nginx.conf + if [ $? -ne 0 ]; then + mv /tmp/nginx.conf /etc/nginx/nginx.conf + nginx -s reload + fi + nginx -s reload + sleep 3 + done + + nginx.conf: | + daemon off; + worker_processes auto; + events { + worker_connections 1024; + } + http { + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + # first server block will be default. Proxy zeppelin server. + server { + listen 80; + + SERVING_ROUTING_TABLE + + location / { + resolver 127.0.0.1:53 ipv6=off; + proxy_pass http://zeppelin-server.NAMESPACE.svc.cluster.local:8080; + proxy_set_header Host $host; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_redirect http://localhost $scheme://SERVICE_DOMAIN; + } + } + + # match request domain [port]-[service].[serviceDomain] + # proxy extra service such as spark-ui + server { + listen 80; + server_name "~(?[0-9]+)-(?[^.]*)\.(.*)"; + location / { + resolver 127.0.0.1:53 ipv6=off; + proxy_pass http://$svc_name.NAMESPACE.svc.cluster.local:$svc_port; + proxy_set_header Host $host; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_redirect http://localhost $scheme://SERVICE_DOMAIN; + + # redirect rule for spark ui. 302 redirect response misses port number of service domain + proxy_redirect ~(http:[/]+[0-9]+[-][^-]+[-][^.]+)[^/]+(\/jobs.*) $1.SERVICE_DOMAIN$2; + } + } + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: zeppelin-serving-api-router +spec: + selector: + matchLabels: + app: zeppelin-serving-api-router + replicas: 1 + template: + metadata: + labels: + app: zeppelin-serving-api-router + spec: + automountServiceAccountToken: true + containers: + - name: serving-api-router + image: nginx:1.14.0 + command: ["/bin/sh", "-c"] + args: + - sh /tmp/conf/nginx_apply_routing_table_and_reload.sh + volumeMounts: + - name: zeppelin-server-conf-volume + mountPath: /tmp/conf + - name: routing-table-volume + mountPath: /routing_table + env: + - name: TRIGGER + value: "11" + lifecycle: + preStop: + exec: + # SIGTERM triggers a quick exit; gracefully terminate instead + command: ["/usr/sbin/nginx", "-s", "quit"] + - name: dnsmasq # nginx requires dns resolver for dynamic dns resolution + image: "janeczku/go-dnsmasq:release-1.0.5" + args: + - --listen + - "127.0.0.1:53" + - --default-resolver + - --append-search-domains + - --hostsfile=/etc/hosts + - --verbose + - name: routing-table-generator + image: apache/zeppelin:0.9.0-SNAPSHOT + command: ["/bin/bash"] + args: + - "-c" + - "/zeppelin/k8s/bin/update_routing_table.py 5 /routing_table/location" + volumeMounts: + - name: routing-table-volume + mountPath: /routing_table + volumes: + - name: zeppelin-server-conf-volume + configMap: + name: zeppelin-serving-api-router-conf + items: + - key: nginx.conf + path: nginx.conf + - key: serviceDomain + path: serviceDomain + - key: nginx_apply_routing_table_and_reload.sh + path: nginx_apply_routing_table_and_reload.sh + - name: routing-table-volume + emptyDir: {} +--- +kind: Service +apiVersion: v1 +metadata: + name: zeppelin-serving-api-router +spec: + ports: + - name: http + port: 80 + selector: + app: zeppelin-serving-api-router +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: zeppelin-serving-metric-redis-conf +data: + redis.conf: | + save "" + maxmemory-policy volatile-lru + appendonly no + protected-mode no + bind 0.0.0.0 + port 6379 + dir ./ +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: zeppelin-serving-metric-redis +spec: + replicas: 1 + selector: + matchLabels: + app: zeppelin-serving-metric-redis + template: + metadata: + labels: + app: zeppelin-serving-metric-redis + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: redis + image: redis:3.2.0-alpine + command: + - redis-server + args: + - /etc/redis/redis.conf + ports: + - containerPort: 6379 + volumeMounts: + - name: redis-conf-volume + mountPath: /etc/redis + volumes: + - name: redis-conf-volume + configMap: + name: zeppelin-serving-metric-redis-conf + items: + - key: redis.conf + path: redis.conf +--- +kind: Service +apiVersion: v1 +metadata: + name: zeppelin-serving-metric-redis +spec: + selector: + app: zeppelin-serving-metric-redis + type: ClusterIP + ports: + - name: redis-service-port + protocol: TCP + port: 6379 + targetPort: 6379 diff --git a/k8s/zeppelin-server.yaml b/k8s/zeppelin-server.yaml index dbee39c8671..0c3afd8c4e3 100644 --- a/k8s/zeppelin-server.yaml +++ b/k8s/zeppelin-server.yaml @@ -29,6 +29,7 @@ data: # If you have your ingress controller configured to connect to `zeppelin-server` service and have a domain name for it (with wildcard subdomain point the same address), you can replace serviceDomain field with your own domain. serviceDomain: local.zeppelin-project.org:8080 sparkContainerImage: spark:2.4.0 + redisAddr: zeppelin-serving-metric-redis.default.svc.cluster.local:6379 # metric storage nginx.conf: | daemon off; worker_processes auto; @@ -108,6 +109,11 @@ spec: fieldRef: apiVersion: v1 fieldPath: metadata.name + - name: SERVICE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name - name: ZEPPELIN_K8S_SPARK_CONTAINER_IMAGE valueFrom: configMapKeyRef: @@ -118,9 +124,16 @@ spec: configMapKeyRef: name: zeppelin-server-conf key: serviceDomain + - name: ZEPPELIN_INTERPRETER_METRIC_REDIS_ADDR + valueFrom: + configMapKeyRef: + name: zeppelin-server-conf + key: redisAddr - name: MASTER # default value of master property for spark interpreter. value: k8s://https://kubernetes.default.svc - # volumeMounts: + volumeMounts: + - name: zeppelin-task-context-volume + mountPath: /zeppelin/task # - name: zeppelin-server-notebook-volume # configure this to persist notebook # mountPath: /zeppelin/notebook # - name: zeppelin-server-conf # configure this to persist Zeppelin configuration @@ -162,6 +175,9 @@ spec: path: nginx.conf - key: serviceDomain path: serviceDomain + - name: zeppelin-task-context-volume + persistentVolumeClaim: + claimName: task-context-volume-claim --- kind: Service apiVersion: v1 @@ -171,6 +187,8 @@ spec: ports: - name: http port: 80 + - name: zeppelin + port: 8080 - name: rpc # port name is referenced in the code. So it shouldn't be changed. port: 12320 selector: @@ -182,7 +200,13 @@ metadata: name: zeppelin-server-role rules: - apiGroups: [""] - resources: ["pods", "services"] + resources: ["pods", "services", "configmaps"] + verbs: ["create", "get", "update", "patch", "list", "delete", "watch"] +- apiGroups: ["extensions", "apps"] + resources: ["deployments"] + verbs: ["create", "get", "update", "patch", "list", "delete", "watch"] +- apiGroups: ["batch", "extensions"] + resources: ["jobs"] verbs: ["create", "get", "update", "patch", "list", "delete", "watch"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings"] diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonRestApiHandler.java b/python/src/main/java/org/apache/zeppelin/python/PythonRestApiHandler.java new file mode 100644 index 00000000000..1fa90369656 --- /dev/null +++ b/python/src/main/java/org/apache/zeppelin/python/PythonRestApiHandler.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.python; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Map; +import java.util.concurrent.ConcurrentLinkedQueue; +import org.apache.commons.io.IOUtils; +import org.apache.zeppelin.serving.RestApiHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * PythonRestApiHandler. + */ +public class PythonRestApiHandler extends RestApiHandler { + private static Logger LOGGER = LoggerFactory.getLogger(PythonRestApiHandler.class); + + private final String endpoint; + private final ConcurrentLinkedQueue restApiReqResQueue; + private Gson gson = new Gson(); + + public PythonRestApiHandler( + String endpoint, + ConcurrentLinkedQueue restApiReqResQueue) { + this.endpoint = endpoint; + this.restApiReqResQueue = restApiReqResQueue; + } + + @Override + protected void handle(HttpServletRequest request, HttpServletResponse response) + throws IOException { + BufferedReader reader = request.getReader(); + Object requestObject; + String contentType = request.getContentType(); + try { + if (contentType != null && contentType.startsWith("application/json")) { + requestObject = gson.fromJson(reader, new TypeToken() {}.getType()); + } else { + requestObject = IOUtils.toString(reader); + } + } catch (Exception e) { + LOGGER.error("Bad request", e); + response.setStatus(HttpServletResponse.SC_BAD_REQUEST); + return; + } + + PythonRestApiRequestResponseMessage message = new PythonRestApiRequestResponseMessage( + endpoint, + requestObject, + request, + response); + + synchronized (restApiReqResQueue) { + restApiReqResQueue.add(message); + restApiReqResQueue.notifyAll(); + } + + synchronized (message) { + long start = System.currentTimeMillis(); + while (!message.isResponseSet() && System.currentTimeMillis() - start <= 60 * 1000) { + try { + message.wait(); + } catch (InterruptedException e) { + continue; + } + } + } + + if (!message.isResponseSet()) { + response.setStatus(HttpServletResponse.SC_REQUEST_TIMEOUT); + } else { + response.setStatus(HttpServletResponse.SC_OK); + Object headers = message.getResponseHeader(); + if (headers != null && headers instanceof Map) { + ((Map) headers).forEach((k, v) -> response.setHeader(k.toString(), v.toString())); + } + + PrintWriter responseWriter = response.getWriter(); + gson.toJson(message.getResponseBody(), responseWriter); + } + } +} diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonRestApiRequestResponseMessage.java b/python/src/main/java/org/apache/zeppelin/python/PythonRestApiRequestResponseMessage.java new file mode 100644 index 00000000000..ebfba74b12e --- /dev/null +++ b/python/src/main/java/org/apache/zeppelin/python/PythonRestApiRequestResponseMessage.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.python; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * This object will be used for pass serving rest api request request between jvm and python + */ +public class PythonRestApiRequestResponseMessage { + private String endpoint; + private Object requestBody; + private Object responseBody; + private Object responseHeader; + private HttpServletRequest httpRequest; + private HttpServletResponse httpResponse; + private boolean isResponseSet = false; + + public PythonRestApiRequestResponseMessage( + String endpoint, + Object requestBody, + HttpServletRequest httpRequest, + HttpServletResponse httpResponse) { + this.endpoint = endpoint; + this.requestBody = requestBody; + this.httpRequest = httpRequest; + this.httpResponse = httpResponse; + } + + public String getEndpoint() { + return endpoint; + } + + public Object getRequestBody() { + return requestBody; + } + + public Object getResponseBody() { + return responseBody; + } + + public void setResponseBody(Object responseBody) { + this.responseBody = responseBody; + isResponseSet = true; + synchronized (this) { + this.notify(); + } + } + + public HttpServletRequest gethttpRequest() { + return httpRequest; + } + + public HttpServletResponse getHttpResponse() { + return httpResponse; + } + + public boolean isResponseSet() { + return isResponseSet; + } + + public void setResponseHeader(Object responseHeader) { + this.responseHeader = responseHeader; + } + + public Object getResponseHeader() { + return responseHeader; + } +} diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonZeppelinContext.java b/python/src/main/java/org/apache/zeppelin/python/PythonZeppelinContext.java index 32d717e9f71..65afff9cd2d 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonZeppelinContext.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonZeppelinContext.java @@ -17,6 +17,7 @@ package org.apache.zeppelin.python; +import java.util.concurrent.ConcurrentLinkedQueue; import org.apache.zeppelin.interpreter.BaseZeppelinContext; import org.apache.zeppelin.interpreter.InterpreterHookRegistry; @@ -27,6 +28,8 @@ * ZeppelinContext for Python */ public class PythonZeppelinContext extends BaseZeppelinContext { + ConcurrentLinkedQueue restApiReqResQueue + = new ConcurrentLinkedQueue(); public PythonZeppelinContext(InterpreterHookRegistry hooks, int maxResult) { super(hooks, maxResult); @@ -46,4 +49,31 @@ public List getSupportedClasses() { public String showData(Object obj, int maxResult) { return null; } + + /** + * method will be invoked by python. + */ + public void addRestApiHandler(String endpoint) { + PythonRestApiHandler handler = new PythonRestApiHandler(endpoint, restApiReqResQueue); + super.addRestApi(endpoint, handler); + } + + /** + * method will be invoked by python + * @return + */ + public PythonRestApiRequestResponseMessage getNextApiRequestFromQueue() { + PythonRestApiRequestResponseMessage message = restApiReqResQueue.poll(); + if (message == null) { + synchronized (restApiReqResQueue) { + try { + restApiReqResQueue.wait(1000); + } catch (InterruptedException e) { + // nothing to do + } + } + message = restApiReqResQueue.poll(); + } + return message; + } } diff --git a/python/src/main/resources/grpc/python/zeppelin_python.py b/python/src/main/resources/grpc/python/zeppelin_python.py index 873989ba2d0..df01eccf626 100644 --- a/python/src/main/resources/grpc/python/zeppelin_python.py +++ b/python/src/main/resources/grpc/python/zeppelin_python.py @@ -15,7 +15,7 @@ # limitations under the License. # -from py4j.java_gateway import java_import, JavaGateway, GatewayClient +from py4j.java_gateway import java_import, JavaGateway, GatewayClient, CallbackServerParameters import os # start JVM gateway @@ -23,10 +23,14 @@ from py4j.java_gateway import GatewayParameters gateway_secret = os.environ["PY4J_GATEWAY_SECRET"] gateway = JavaGateway(gateway_parameters=GatewayParameters(address="${JVM_GATEWAY_ADDRESS}", - port=${JVM_GATEWAY_PORT}, auth_token=gateway_secret, auto_convert=True)) + port=${JVM_GATEWAY_PORT}, auth_token=gateway_secret, auto_convert=True), + start_callback_server=True, + callback_server_parameters=CallbackServerParameters()) java_import(gateway.jvm, "org.apache.zeppelin.display.Input") intp = gateway.entry_point else: - gateway = JavaGateway(GatewayClient(address="${JVM_GATEWAY_ADDRESS}", port=${JVM_GATEWAY_PORT}), auto_convert=True) + gateway = JavaGateway(GatewayClient(address="${JVM_GATEWAY_ADDRESS}", port=${JVM_GATEWAY_PORT}), auto_convert=True, + start_callback_server=True, + callback_server_parameters=CallbackServerParameters()) java_import(gateway.jvm, "org.apache.zeppelin.display.Input") intp = gateway.entry_point diff --git a/python/src/main/resources/python/zeppelin_context.py b/python/src/main/resources/python/zeppelin_context.py index dc97c141c15..f57bfa1d817 100644 --- a/python/src/main/resources/python/zeppelin_context.py +++ b/python/src/main/resources/python/zeppelin_context.py @@ -18,6 +18,7 @@ import os, sys import warnings import base64 +import threading from io import BytesIO @@ -26,6 +27,7 @@ except ImportError: from io import StringIO + class PyZeppelinContext(object): """ A context impl that uses Py4j to communicate to JVM """ @@ -38,6 +40,9 @@ def __init__(self, z, gateway): self.max_result = z.getMaxResult() self._displayhook = lambda *args: None self._setup_matplotlib() + self._apiHandlers = {} + t = threading.Thread(target=self._handleApiRequestThread) + t.start() # By implementing special methods it makes operating on it more Pythonic def __setitem__(self, key, item): @@ -88,6 +93,31 @@ def checkbox(self, name, options, defaultChecked=[]): def noteCheckbox(self, name, options, defaultChecked=[]): return self.z.noteCheckbox(name, self.getDefaultChecked(defaultChecked), self.getParamOptions(options)) + def addRestApi(self, name, fn): + self._apiHandlers[name] = fn + return self.z.addRestApiHandler(name) + + def _handleApiRequestThread(self): + while True: + msg = self.z.getNextApiRequestFromQueue() + if msg == None: + continue + + endpoint = msg.getEndpoint() + request = msg.getRequestBody() + fn = self._apiHandlers[endpoint] + + try: + ret = fn(request) + if isinstance(ret, tuple): + msg.setResponseHeader(ret[1]) + msg.setResponseBody(ret[0]) + else: + msg.setResponseBody(ret) + except: + err = sys.exc_info()[0] + msg.setResponseBody(str(err)) + def registerHook(self, event, cmd, replName=None): if replName is None: self.z.registerHook(event, cmd) diff --git a/python/src/main/resources/python/zeppelin_python.py b/python/src/main/resources/python/zeppelin_python.py index db224e4bc5b..5819bab55e9 100644 --- a/python/src/main/resources/python/zeppelin_python.py +++ b/python/src/main/resources/python/zeppelin_python.py @@ -17,7 +17,7 @@ import os, sys, traceback, json, re -from py4j.java_gateway import java_import, JavaGateway, GatewayClient +from py4j.java_gateway import java_import, JavaGateway, GatewayClient, CallbackServerParameters from py4j.protocol import Py4JJavaError import ast @@ -84,9 +84,13 @@ def getCompletion(self, text_value): from py4j.java_gateway import GatewayParameters gateway_secret = os.environ["PY4J_GATEWAY_SECRET"] gateway = JavaGateway(gateway_parameters=GatewayParameters( - address=host, port=port, auth_token=gateway_secret, auto_convert=True)) + address=host, port=port, auth_token=gateway_secret, auto_convert=True), + start_callback_server=True, + callback_server_parameters=CallbackServerParameters()) else: - gateway = JavaGateway(GatewayClient(address=host, port=port), auto_convert=True) + gateway = JavaGateway(GatewayClient(address=host, port=port), auto_convert=True, + start_callback_server=True, + callback_server_parameters=CallbackServerParameters()) intp = gateway.entry_point _zcUserQueryNameSpace = {} diff --git a/python/src/test/java/org/apache/zeppelin/python/BasePythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/BasePythonInterpreterTest.java index 6e8bbc95634..ac128a08d60 100644 --- a/python/src/test/java/org/apache/zeppelin/python/BasePythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/BasePythonInterpreterTest.java @@ -31,6 +31,7 @@ import org.apache.zeppelin.interpreter.InterpreterResultMessage; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventClient; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; +import org.apache.zeppelin.serving.RestApiServer; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -356,6 +357,7 @@ protected InterpreterContext getInterpreterContext() { .setParagraphId("paragraphId") .setInterpreterOut(new InterpreterOutput(null)) .setIntpEventClient(mock(RemoteInterpreterEventClient.class)) + .setRestApiServer(RestApiServer.singleton()) .build(); } } diff --git a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java index 236e08cd93c..67fb9f15a14 100644 --- a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java @@ -18,6 +18,10 @@ package org.apache.zeppelin.python; import net.jodah.concurrentunit.Waiter; +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; +import org.apache.commons.httpclient.methods.PutMethod; +import org.apache.commons.httpclient.methods.StringRequestEntity; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; @@ -27,6 +31,8 @@ import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.InterpreterResultMessage; import org.apache.zeppelin.interpreter.LazyOpenInterpreter; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; +import org.apache.zeppelin.serving.RestApiServer; import org.junit.Test; import java.io.IOException; @@ -39,8 +45,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; - public class IPythonInterpreterTest extends BasePythonInterpreterTest { + private HttpClient client; protected Properties initIntpProperties() { Properties properties = new Properties(); @@ -63,6 +69,7 @@ protected void startInterpreter(Properties properties) throws InterpreterExcepti public void setUp() throws InterpreterException { Properties properties = initIntpProperties(); startInterpreter(properties); + client = new HttpClient(new MultiThreadedHttpConnectionManager()); } @Override @@ -326,4 +333,130 @@ public void testIPythonFailToLaunch() throws InterpreterException { } } + @Test + public void testAddRestApiJsonRequest() + throws InterpreterException, InterruptedException, IOException { + // given + int port = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); + RestApiServer.setPort(port); + InterpreterContext context = getInterpreterContext(); + + // when + InterpreterResult result = interpreter.interpret( + "def stringlen(d):\n return len(d[\"input\"])\nz.addRestApi(\"len\", stringlen)\n", + context); + waitForResult(result, Code.SUCCESS); + + // then + PutMethod put = new PutMethod(String.format("http://localhost:%d/%s", + RestApiServer.getPort(), + "len")); + put.setRequestEntity(new StringRequestEntity( + "{\"input\": \"abc\"}", + "application/json", + "utf8")); + + int code = client.executeMethod(put); + assertEquals(200, code); + assertEquals("3", put.getResponseBodyAsString()); + + put.releaseConnection(); + } + + @Test + public void testAddRestApiJsonRequestJsonReturn() + throws InterpreterException, InterruptedException, IOException { + // given + int port = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); + RestApiServer.setPort(port); + InterpreterContext context = getInterpreterContext(); + + // when + InterpreterResult result = interpreter.interpret( + "def stringlen(d):\n return {\"len\": len(d[\"input\"])}\n" + + "z.addRestApi(\"len\", stringlen)\n", + context); + waitForResult(result, Code.SUCCESS); + + // then + PutMethod put = new PutMethod(String.format("http://localhost:%d/%s", + RestApiServer.getPort(), "len")); + put.setRequestEntity(new StringRequestEntity( + "{\"input\": \"abc\"}", + "application/json", + "utf8")); + + int code = client.executeMethod(put); + assertEquals(200, code); + assertEquals("{\"len\":3}", put.getResponseBodyAsString()); + + put.releaseConnection(); + } + + @Test + public void testAddRestApiStringRequest() + throws InterpreterException, InterruptedException, IOException { + // given + int port = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); + RestApiServer.setPort(port); + InterpreterContext context = getInterpreterContext(); + + // when + InterpreterResult result = interpreter.interpret( + "def stringlen(d):\n return len(d)\nz.addRestApi(\"len\", stringlen)\n", context); + waitForResult(result, Code.SUCCESS); + + // then + PutMethod put = new PutMethod(String.format("http://localhost:%d/%s", + RestApiServer.getPort(), "len")); + put.setRequestEntity(new StringRequestEntity("abc", "text/plain", "utf8")); + + int code = client.executeMethod(put); + assertEquals(200, code); + assertEquals("3", put.getResponseBodyAsString()); + + put.releaseConnection(); + } + + @Test + public void testCustomResponseHeader() + throws InterpreterException, InterruptedException, IOException { + // given + int port = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); + RestApiServer.setPort(port); + InterpreterContext context = getInterpreterContext(); + + // when + InterpreterResult result = interpreter.interpret( + "def stringlen(d):\n return len(d), {'z-metric-c1': len(d)}\n" + + "z.addRestApi(\"len\", stringlen)\n", context); + waitForResult(result, Code.SUCCESS); + + // then + PutMethod put = new PutMethod(String.format("http://localhost:%d/%s", + RestApiServer.getPort(), "len")); + put.setRequestEntity(new StringRequestEntity("abc", "text/plain", "utf8")); + + int code = client.executeMethod(put); + assertEquals(200, code); + assertEquals("3", put.getResponseBodyAsString()); + assertEquals("3", put.getResponseHeader("z-metric-c1").getValue()); + + put.releaseConnection(); + } + + private void waitForResult(InterpreterResult result, InterpreterResult.Code code) { + long start = System.currentTimeMillis(); + while (result.code() != code) { + try { + Thread.sleep(300); + } catch (InterruptedException e) { + continue; + } + + if (System.currentTimeMillis() - start > 30 * 1000) { + throw new RuntimeException("Result expected " + code); + } + } + } } diff --git a/scripts/docker/zeppelin/bin/Dockerfile b/scripts/docker/zeppelin/bin/Dockerfile index b05df71aea7..7b36fa20c5f 100644 --- a/scripts/docker/zeppelin/bin/Dockerfile +++ b/scripts/docker/zeppelin/bin/Dockerfile @@ -96,11 +96,14 @@ RUN echo "$LOG_TAG Cleanup" && \ apt-get autoclean && \ apt-get clean -RUN echo "$LOG_TAG Download Zeppelin binary" && \ - wget -O /tmp/zeppelin-${Z_VERSION}-bin-all.tgz http://archive.apache.org/dist/zeppelin/zeppelin-${Z_VERSION}/zeppelin-${Z_VERSION}-bin-all.tgz && \ - tar -zxvf /tmp/zeppelin-${Z_VERSION}-bin-all.tgz && \ - rm -rf /tmp/zeppelin-${Z_VERSION}-bin-all.tgz && \ - mv /zeppelin-${Z_VERSION}-bin-all ${Z_HOME} +#RUN echo "$LOG_TAG Download Zeppelin binary" && \ +# wget -O /tmp/zeppelin-${Z_VERSION}-bin-all.tgz http://archive.apache.org/dist/zeppelin/zeppelin-${Z_VERSION}/zeppelin-${Z_VERSION}-bin-all.tgz && \ +# tar -zxvf /tmp/zeppelin-${Z_VERSION}-bin-all.tgz && \ +# rm -rf /tmp/zeppelin-${Z_VERSION}-bin-all.tgz && \ +# mv /zeppelin-${Z_VERSION}-bin-all ${Z_HOME} + +ADD zeppelin-${Z_VERSION}.tar.gz / +RUN ln -s /zeppelin-${Z_VERSION} /zeppelin COPY log4j.properties ${Z_HOME}/conf/ diff --git a/spark/interpreter/src/main/scala/org/apache/zeppelin/spark/SparkZeppelinContext.scala b/spark/interpreter/src/main/scala/org/apache/zeppelin/spark/SparkZeppelinContext.scala index e80c1527462..c175af30d27 100644 --- a/spark/interpreter/src/main/scala/org/apache/zeppelin/spark/SparkZeppelinContext.scala +++ b/spark/interpreter/src/main/scala/org/apache/zeppelin/spark/SparkZeppelinContext.scala @@ -24,6 +24,7 @@ import org.apache.zeppelin.annotation.ZeppelinApi import org.apache.zeppelin.display.AngularObjectWatcher import org.apache.zeppelin.display.ui.OptionInput.ParamOption import org.apache.zeppelin.interpreter.{BaseZeppelinContext, InterpreterContext, InterpreterHookRegistry} +import org.apache.zeppelin.serving.JsonApiHandler import scala.collection.{JavaConversions, Seq} @@ -129,6 +130,14 @@ class SparkZeppelinContext(val sc: SparkContext, angularWatch(name, null, func) } + @ZeppelinApi def addRestApi[T, R](name: String, + func: (T) => R): Unit = { + val handler = new JsonApiHandler[T]() { + override def handle(t: T): AnyRef = func; + }; + interpreterContext.addRestApi(name, handler) + } + private def angularWatch(name: String, noteId: String, func: (AnyRef, AnyRef) => Unit): Unit = { val w = new AngularObjectWatcher(getInterpreterContext) { override def watch(oldObject: Any, newObject: AnyRef, context: InterpreterContext): Unit = { diff --git a/zeppelin-interpreter-api/pom.xml b/zeppelin-interpreter-api/pom.xml index 10baa9289bb..281ebaa6ee8 100644 --- a/zeppelin-interpreter-api/pom.xml +++ b/zeppelin-interpreter-api/pom.xml @@ -138,6 +138,14 @@ io ${shaded.dependency.prefix}.io + + org.eclipse + ${shaded.dependency.prefix}.org.eclipse + + + org.ow2 + ${shaded.dependency.prefix}.org.ow2 + diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml index bf624ceda38..81752c75706 100644 --- a/zeppelin-interpreter/pom.xml +++ b/zeppelin-interpreter/pom.xml @@ -151,6 +151,31 @@ slf4j-log4j12 + + + org.eclipse.jetty + jetty-server + ${jetty.version} + + + + org.eclipse.jetty + jetty-servlet + ${jetty.version} + + + + redis.clients + jedis + 3.0.1 + + + org.apache.commons + commons-pool2 + + + + org.apache.maven @@ -278,6 +303,13 @@ ${jline.version} + + com.orange.redis-embedded + embedded-redis + 0.6 + test + + junit junit @@ -289,6 +321,11 @@ mockito-all test + + org.apache.flink + flink-table_2.11 + RELEASE + diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index a9aff467033..b5359cd97cf 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -353,6 +353,22 @@ public String getNotebookDir() { return getString(ConfVars.ZEPPELIN_NOTEBOOK_DIR); } + public String getNotebookRunId() { + return getString(ConfVars.ZEPPELIN_NOTEBOOK_RUN_ID); + } + + public String getNotebookRunRev() { + return getString(ConfVars.ZEPPELIN_NOTEBOOK_RUN_REV); + } + + public String getNotebookRunServiceContext() { + return getString(ConfVars.ZEPPELIN_NOTEBOOK_RUN_SERVICE_CONTEXT); + } + + public boolean getNotebookRunAutoShutdown() { + return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_RUN_AUTOSHUTDOWN); + } + public String getPluginsDir() { return getRelativeDir(getString(ConfVars.ZEPPELIN_PLUGINS_DIR)); } @@ -519,6 +535,14 @@ public String getInterpreterPortRange() { return getString(ConfVars.ZEPPELIN_INTERPRETER_RPC_PORTRANGE); } + public int getInterpreterRestApiServerPort() { + return getInt(ConfVars.ZEPPELIN_INTERPRETER_RESTAPI_PORT); + } + + public String getInterpreterMetricRedisAddr() { + return getString(ConfVars.ZEPPELIN_INTERPRETER_METRIC_REDIS_ADDR); + } + public boolean isWindowsPath(String path){ return path.matches("^[A-Za-z]:\\\\.*"); } @@ -703,6 +727,14 @@ public String getK8sTemplatesDir() { return getRelativeDir(ConfVars.ZEPPELIN_K8S_TEMPLATE_DIR); } + public String getK8sServingContextDir() { + return getRelativeDir(ConfVars.ZEPPELIN_K8S_SERVING_CONTEXT_DIR); + } + + public String getK8sTestContextDir() { + return getRelativeDir(ConfVars.ZEPPELIN_K8S_TEST_CONTEXT_DIR); + } + public Map dumpConfigurations(Predicate predicate) { Map properties = new HashMap<>(); @@ -766,6 +798,12 @@ public enum ConfVars { ZEPPELIN_INTERPRETER_OUTPUT_LIMIT("zeppelin.interpreter.output.limit", 1024 * 100), ZEPPELIN_ENCODING("zeppelin.encoding", "UTF-8"), ZEPPELIN_NOTEBOOK_DIR("zeppelin.notebook.dir", "notebook"), + + ZEPPELIN_NOTEBOOK_RUN_ID("zeppelin.notebook.run.id", null), // run particular note id on zeppelin start + ZEPPELIN_NOTEBOOK_RUN_REV("zeppelin.notebook.run.rev", null), // revision id for ZEPPELIN_NOTEBOOK_RUN_ID. + ZEPPELIN_NOTEBOOK_RUN_SERVICE_CONTEXT("zeppelin.notebook.run.servicecontext", null), // base64 encoded serialized service context to be used ZEPPELIN_NOTEBOOK_RUN_ID. + ZEPPELIN_NOTEBOOK_RUN_AUTOSHUTDOWN("zeppelin.notebook.run.autoshutdown", true), // after specified note (ZEPPELIN_NOTEBOOK_RUN_ID) run, shutdown zeppelin server + ZEPPELIN_RECOVERY_DIR("zeppelin.recovery.dir", "recovery"), ZEPPELIN_RECOVERY_STORAGE_CLASS("zeppelin.recovery.storage.class", "org.apache.zeppelin.interpreter.recovery.NullRecoveryStorage"), @@ -839,7 +877,8 @@ public enum ConfVars { ZEPPELIN_SERVER_RPC_PORTRANGE("zeppelin.server.rpc.portRange", ":"), ZEPPELIN_INTERPRETER_RPC_PORTRANGE("zeppelin.interpreter.rpc.portRange", ":"), - + ZEPPELIN_INTERPRETER_RESTAPI_PORT("zeppelin.interpreter.restapiserver.port", 0), + ZEPPELIN_INTERPRETER_METRIC_REDIS_ADDR("zeppelin.interpreter.metric.redis.addr", null), // serving metric strage. host:port ZEPPELIN_INTERPRETER_LIFECYCLE_MANAGER_CLASS("zeppelin.interpreter.lifecyclemanager.class", "org.apache.zeppelin.interpreter.lifecycle.NullLifecycleManager"), ZEPPELIN_INTERPRETER_LIFECYCLE_MANAGER_TIMEOUT_CHECK_INTERVAL( @@ -863,6 +902,9 @@ public enum ConfVars { ZEPPELIN_K8S_SPARK_CONTAINER_IMAGE("zeppelin.k8s.spark.container.image", "apache/spark:latest"), ZEPPELIN_K8S_TEMPLATE_DIR("zeppelin.k8s.template.dir", "k8s"), + ZEPPELIN_K8S_SERVING_CONTEXT_DIR("zeppelin.k8s.serving.context.dir", "task/serving"), + ZEPPELIN_K8S_TEST_CONTEXT_DIR("zeppelin.k8s.test.context.dir", "task/test"), + ZEPPELIN_NOTEBOOK_GIT_REMOTE_URL("zeppelin.notebook.git.remote.url", ""), ZEPPELIN_NOTEBOOK_GIT_REMOTE_USERNAME("zeppelin.notebook.git.remote.username", "token"), ZEPPELIN_NOTEBOOK_GIT_REMOTE_ACCESS_TOKEN("zeppelin.notebook.git.remote.access-token", ""), diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/BaseZeppelinContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/BaseZeppelinContext.java index abf2e0ae8c1..ae722bbb994 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/BaseZeppelinContext.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/BaseZeppelinContext.java @@ -28,6 +28,7 @@ import org.apache.zeppelin.resource.Resource; import org.apache.zeppelin.resource.ResourcePool; import org.apache.zeppelin.resource.ResourceSet; +import org.apache.zeppelin.serving.RestApiHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -851,4 +852,9 @@ public ResourceSet getAll() { ResourcePool resourcePool = interpreterContext.getResourcePool(); return resourcePool.getAll(); } + + @ZeppelinApi + public void addRestApi(String endpoint, RestApiHandler handler) { + interpreterContext.addRestApi(endpoint, handler); + } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Constants.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Constants.java index 87748fffabc..90148dc8afc 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Constants.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Constants.java @@ -39,6 +39,8 @@ public class Constants { public static final Map TIME_SUFFIXES; + public static final int ZEPPELIN_INTERPRETER_RESTAPI_DEFAULT_PORT = 8090; + static { TIME_SUFFIXES = new HashMap<>(); TIME_SUFFIXES.put("us", TimeUnit.MICROSECONDS); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java index 4e0a8df9f0c..9c54af60cb2 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java @@ -21,6 +21,8 @@ import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventClient; import org.apache.zeppelin.resource.ResourcePool; +import org.apache.zeppelin.serving.RestApiHandler; +import org.apache.zeppelin.serving.RestApiServer; import org.apache.zeppelin.user.AuthenticationInfo; import java.util.HashMap; @@ -62,6 +64,7 @@ public static void remove() { private Map progressMap; private Map localProperties = new HashMap<>(); private RemoteInterpreterEventClient intpEventClient; + private RestApiServer restApiServer; /** * Builder class for InterpreterContext @@ -158,6 +161,11 @@ public Builder setLocalProperties(Map localProperties) { return this; } + public Builder setRestApiServer(RestApiServer restApiServer) { + context.restApiServer = restApiServer; + return this; + } + public InterpreterContext build() { InterpreterContext.set(context); return context; @@ -261,6 +269,11 @@ public InterpreterOutput out() { return out; } + public void addRestApi(String endpoint, RestApiHandler handler) { + restApiServer.addEndpoint(endpoint, handler); + this.intpEventClient.addRestApi(noteId, endpoint); + } + /** * Set progress of paragraph manually * @param n integer from 0 to 100 diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java index 5ac1c0a8dcd..27e47b56eec 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java @@ -17,6 +17,8 @@ package org.apache.zeppelin.interpreter.remote; import com.google.gson.Gson; +import java.net.InetAddress; +import java.net.UnknownHostException; import org.apache.thrift.TException; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistryListener; @@ -30,6 +32,7 @@ import org.apache.zeppelin.interpreter.thrift.OutputUpdateEvent; import org.apache.zeppelin.interpreter.thrift.ParagraphInfo; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventService; +import org.apache.zeppelin.interpreter.thrift.RestApiInfo; import org.apache.zeppelin.interpreter.thrift.RunParagraphsEvent; import org.apache.zeppelin.interpreter.thrift.ServiceException; import org.apache.zeppelin.resource.RemoteResource; @@ -37,6 +40,7 @@ import org.apache.zeppelin.resource.ResourceId; import org.apache.zeppelin.resource.ResourcePoolConnector; import org.apache.zeppelin.resource.ResourceSet; +import org.apache.zeppelin.serving.RestApiServer; import org.apache.zeppelin.user.AuthenticationInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -277,6 +281,25 @@ public synchronized void onParaInfosReceived(Map infos) { } } + public synchronized void addRestApi(String noteId, String endpointName) { + int port = RestApiServer.getPort(); + try { + String hostname = InetAddress.getLocalHost().getHostName(); + RestApiInfo apiInfo = new RestApiInfo( + intpGroupId, + noteId, + endpointName, + hostname, + port + ); + intpEventServiceClient.addRestApi(apiInfo); + } catch (TException e) { + LOGGER.warn("Fail to add rest api endpoint", e); + } catch (UnknownHostException e) { + LOGGER.error("Can't get host name", e); + } + } + @Override public synchronized void onAdd(String interpreterGroupId, AngularObject object) { try { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index 1d4d2310b2e..f5fef1f4de0 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -8,7 +8,7 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + * Unless required by app[licable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -17,6 +17,7 @@ package org.apache.zeppelin.interpreter.remote; +import com.google.common.annotations.VisibleForTesting; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import org.apache.commons.lang.exception.ExceptionUtils; @@ -67,6 +68,8 @@ import org.apache.zeppelin.resource.ResourcePool; import org.apache.zeppelin.resource.ResourceSet; import org.apache.zeppelin.resource.WellKnownResourceName; +import org.apache.zeppelin.serving.RedisMetricStorage; +import org.apache.zeppelin.serving.RestApiServer; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.scheduler.JobListener; @@ -132,18 +135,22 @@ public class RemoteInterpreterServer extends Thread private boolean isTest; + private RestApiServer restApiServer; + public RemoteInterpreterServer(String intpEventServerHost, int intpEventServerPort, String interpreterGroupId, - String portRange) - throws IOException, TTransportException { - this(intpEventServerHost, intpEventServerPort, portRange, interpreterGroupId, false); + String portRange, + int restApiServerPort) + throws IOException, TTransportException { + this(intpEventServerHost, intpEventServerPort, portRange, interpreterGroupId, restApiServerPort,false); } public RemoteInterpreterServer(String intpEventServerHost, int intpEventServerPort, String portRange, String interpreterGroupId, + int restApiServerPort, boolean isTest) throws TTransportException, IOException { logger.info("Starting remote interpreter server on port {}, intpEventServerAddress: {}:{}", port, @@ -178,6 +185,23 @@ public RemoteInterpreterServer(String intpEventServerHost, server = new TThreadPoolServer( new TThreadPoolServer.Args(serverTransport).processor(processor)); remoteWorksResponsePool = Collections.synchronizedMap(new HashMap()); + + if (restApiServerPort <= 0) { + restApiServerPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); + } + // initialize restApiServer for serving + RestApiServer.setPort(restApiServerPort); + restApiServer = RestApiServer.singleton(); + + // serving metric storage + if (System.getenv("SERVICE_NAME") != null) { + try { + restApiServer.addMetricStorage(new RedisMetricStorage()); + } catch (Exception e) { + logger.info("Redis metric storage is not initialized"); + logger.debug("Error", e); + } + } } @Override @@ -273,6 +297,7 @@ public static void main(String[] args) throws TTransportException, InterruptedException, IOException { String zeppelinServerHost = null; int port = Constants.ZEPPELIN_INTERPRETER_DEFAUlT_PORT; + int restApiServerPort = Constants.ZEPPELIN_INTERPRETER_RESTAPI_DEFAULT_PORT; String portRange = ":"; String interpreterGroupId = null; if (args.length > 0) { @@ -282,9 +307,13 @@ public static void main(String[] args) if (args.length > 3) { portRange = args[3]; } + + if (args.length > 4) { + restApiServerPort = Integer.parseInt(args[4]); + } } RemoteInterpreterServer remoteInterpreterServer = - new RemoteInterpreterServer(zeppelinServerHost, port, interpreterGroupId, portRange); + new RemoteInterpreterServer(zeppelinServerHost, port, interpreterGroupId, portRange, restApiServerPort); remoteInterpreterServer.start(); // add signal handler @@ -349,7 +378,7 @@ public void createInterpreter(String interpreterGroupId, String sessionId, Strin } } - protected InterpreterGroup getInterpreterGroup() { + public InterpreterGroup getInterpreterGroup() { return interpreterGroup; } @@ -361,6 +390,11 @@ protected RemoteInterpreterEventClient getIntpEventClient() { return intpEventClient; } + @VisibleForTesting + public void setIntpEventClient(RemoteInterpreterEventClient intpEventClient) { + this.intpEventClient = intpEventClient; + } + private void setSystemProperty(Properties properties) { for (Object key : properties.keySet()) { if (!RemoteInterpreterUtils.isEnvString((String) key)) { @@ -749,6 +783,7 @@ private InterpreterContext convert(RemoteInterpreterContext ric, InterpreterOutp .setInterpreterOut(output) .setIntpEventClient(intpEventClient) .setProgressMap(progressMap) + .setRestApiServer(restApiServer) .build(); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/AngularObjectId.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/AngularObjectId.java index 24e9fd9df37..9c99dd991ba 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/AngularObjectId.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/AngularObjectId.java @@ -16,7 +16,7 @@ * limitations under the License. */ /** - * Autogenerated by Thrift Compiler (0.9.2) + * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2019-3-4") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2019-04-19") public class AngularObjectId implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("AngularObjectId"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/AppOutputAppendEvent.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/AppOutputAppendEvent.java index bac58e9c8c7..4076dc430ce 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/AppOutputAppendEvent.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/AppOutputAppendEvent.java @@ -16,7 +16,7 @@ * limitations under the License. */ /** - * Autogenerated by Thrift Compiler (0.9.2) + * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2019-3-4") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2019-04-19") public class AppOutputAppendEvent implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("AppOutputAppendEvent"); @@ -392,7 +392,7 @@ public Object getFieldValue(_Fields field) { return getAppId(); case INDEX: - return Integer.valueOf(getIndex()); + return getIndex(); case DATA: return getData(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/AppOutputUpdateEvent.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/AppOutputUpdateEvent.java index 03ff738d188..d67fa6304c1 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/AppOutputUpdateEvent.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/AppOutputUpdateEvent.java @@ -16,7 +16,7 @@ * limitations under the License. */ /** - * Autogenerated by Thrift Compiler (0.9.2) + * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2019-3-4") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2019-04-19") public class AppOutputUpdateEvent implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("AppOutputUpdateEvent"); @@ -437,7 +437,7 @@ public Object getFieldValue(_Fields field) { return getAppId(); case INDEX: - return Integer.valueOf(getIndex()); + return getIndex(); case TYPE: return getType(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/AppStatusUpdateEvent.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/AppStatusUpdateEvent.java index ad09d5ddcb8..1d53d626ad7 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/AppStatusUpdateEvent.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/AppStatusUpdateEvent.java @@ -16,7 +16,7 @@ * limitations under the License. */ /** - * Autogenerated by Thrift Compiler (0.9.2) + * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2019-3-4") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2019-04-19") public class AppStatusUpdateEvent implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("AppStatusUpdateEvent"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/InterpreterCompletion.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/InterpreterCompletion.java index d19d1ac6178..fca760ff20a 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/InterpreterCompletion.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/InterpreterCompletion.java @@ -16,7 +16,7 @@ * limitations under the License. */ /** - * Autogenerated by Thrift Compiler (0.9.2) + * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2019-3-4") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2019-04-19") public class InterpreterCompletion implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("InterpreterCompletion"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/OutputAppendEvent.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/OutputAppendEvent.java index 69abc082b33..92363e2e0d6 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/OutputAppendEvent.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/OutputAppendEvent.java @@ -16,7 +16,7 @@ * limitations under the License. */ /** - * Autogenerated by Thrift Compiler (0.9.2) + * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2019-3-4") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2019-04-19") public class OutputAppendEvent implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("OutputAppendEvent"); @@ -389,7 +389,7 @@ public Object getFieldValue(_Fields field) { return getParagraphId(); case INDEX: - return Integer.valueOf(getIndex()); + return getIndex(); case DATA: return getData(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/OutputUpdateAllEvent.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/OutputUpdateAllEvent.java index ac32e6e5248..f6f127c5cca 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/OutputUpdateAllEvent.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/OutputUpdateAllEvent.java @@ -16,7 +16,7 @@ * limitations under the License. */ /** - * Autogenerated by Thrift Compiler (0.9.2) + * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2019-3-4") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2019-04-19") public class OutputUpdateAllEvent implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("OutputUpdateAllEvent"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/OutputUpdateEvent.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/OutputUpdateEvent.java index 0e226ba8405..01b2082e05d 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/OutputUpdateEvent.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/OutputUpdateEvent.java @@ -16,7 +16,7 @@ * limitations under the License. */ /** - * Autogenerated by Thrift Compiler (0.9.2) + * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2019-3-4") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2019-04-19") public class OutputUpdateEvent implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("OutputUpdateEvent"); @@ -434,7 +434,7 @@ public Object getFieldValue(_Fields field) { return getParagraphId(); case INDEX: - return Integer.valueOf(getIndex()); + return getIndex(); case TYPE: return getType(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/ParagraphInfo.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/ParagraphInfo.java index bd50d99c0c7..b3a65e30f29 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/ParagraphInfo.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/ParagraphInfo.java @@ -16,7 +16,7 @@ * limitations under the License. */ /** - * Autogenerated by Thrift Compiler (0.9.2) + * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2019-3-4") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2019-04-19") public class ParagraphInfo implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ParagraphInfo"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RegisterInfo.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RegisterInfo.java index 46fb9b01072..0dea8ae9d1c 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RegisterInfo.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RegisterInfo.java @@ -16,7 +16,7 @@ * limitations under the License. */ /** - * Autogenerated by Thrift Compiler (0.9.2) + * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2019-3-4") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2019-04-19") public class RegisterInfo implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RegisterInfo"); @@ -296,7 +296,7 @@ public Object getFieldValue(_Fields field) { return getHost(); case PORT: - return Integer.valueOf(getPort()); + return getPort(); case INTERPRETER_GROUP_ID: return getInterpreterGroupId(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteApplicationResult.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteApplicationResult.java index 3cbbb3d4c9e..d00f6afaad0 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteApplicationResult.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteApplicationResult.java @@ -16,7 +16,7 @@ * limitations under the License. */ /** - * Autogenerated by Thrift Compiler (0.9.2) + * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2019-3-4") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2019-04-19") public class RemoteApplicationResult implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteApplicationResult"); @@ -248,7 +248,7 @@ public void setFieldValue(_Fields field, Object value) { public Object getFieldValue(_Fields field) { switch (field) { case SUCCESS: - return Boolean.valueOf(isSuccess()); + return isSuccess(); case MSG: return getMsg(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java index 4ff489690f8..7a3b2297bf5 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java @@ -16,7 +16,7 @@ * limitations under the License. */ /** - * Autogenerated by Thrift Compiler (0.9.2) + * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2019-3-4") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2019-04-19") public class RemoteInterpreterContext implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterContext"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java index 0fa6465197e..2ca7e2456aa 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java @@ -16,7 +16,7 @@ * limitations under the License. */ /** - * Autogenerated by Thrift Compiler (0.9.2) + * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2019-3-4") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2019-04-19") public class RemoteInterpreterEvent implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterEvent"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventService.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventService.java index 4bbc8f75538..cc0c69f6da1 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventService.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventService.java @@ -16,7 +16,7 @@ * limitations under the License. */ /** - * Autogenerated by Thrift Compiler (0.9.2) + * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2019-3-4") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2019-04-19") public class RemoteInterpreterEventService { public interface Iface { @@ -88,6 +88,8 @@ public interface Iface { public List getParagraphList(String user, String noteId) throws ServiceException, org.apache.thrift.TException; + public void addRestApi(RestApiInfo restApiInfo) throws org.apache.thrift.TException; + } public interface AsyncIface { @@ -124,6 +126,8 @@ public interface AsyncIface { public void getParagraphList(String user, String noteId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + public void addRestApi(RestApiInfo restApiInfo, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + } public static class Client extends org.apache.thrift.TServiceClient implements Iface { @@ -489,6 +493,26 @@ public List recv_getParagraphList() throws ServiceException, org. throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getParagraphList failed: unknown result"); } + public void addRestApi(RestApiInfo restApiInfo) throws org.apache.thrift.TException + { + send_addRestApi(restApiInfo); + recv_addRestApi(); + } + + public void send_addRestApi(RestApiInfo restApiInfo) throws org.apache.thrift.TException + { + addRestApi_args args = new addRestApi_args(); + args.setRestApiInfo(restApiInfo); + sendBase("addRestApi", args); + } + + public void recv_addRestApi() throws org.apache.thrift.TException + { + addRestApi_result result = new addRestApi_result(); + receiveBase(result, "addRestApi"); + return; + } + } public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface { public static class Factory implements org.apache.thrift.async.TAsyncClientFactory { @@ -1043,6 +1067,38 @@ public List getResult() throws ServiceException, org.apache.thrif } } + public void addRestApi(RestApiInfo restApiInfo, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + addRestApi_call method_call = new addRestApi_call(restApiInfo, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class addRestApi_call extends org.apache.thrift.async.TAsyncMethodCall { + private RestApiInfo restApiInfo; + public addRestApi_call(RestApiInfo restApiInfo, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.restApiInfo = restApiInfo; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("addRestApi", org.apache.thrift.protocol.TMessageType.CALL, 0)); + addRestApi_args args = new addRestApi_args(); + args.setRestApiInfo(restApiInfo); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_addRestApi(); + } + } + } public static class Processor extends org.apache.thrift.TBaseProcessor implements org.apache.thrift.TProcessor { @@ -1072,6 +1128,7 @@ protected Processor(I iface, Map extends org.apache.thrift.ProcessFunction { + public addRestApi() { + super("addRestApi"); + } + + public addRestApi_args getEmptyArgsInstance() { + return new addRestApi_args(); + } + + protected boolean isOneway() { + return false; + } + + public addRestApi_result getResult(I iface, addRestApi_args args) throws org.apache.thrift.TException { + addRestApi_result result = new addRestApi_result(); + iface.addRestApi(args.restApiInfo); + return result; + } + } + } public static class AsyncProcessor extends org.apache.thrift.TBaseAsyncProcessor { @@ -1428,6 +1505,7 @@ protected AsyncProcessor(I iface, Map extends org.apache.thrift.AsyncProcessFunction { + public addRestApi() { + super("addRestApi"); + } + + public addRestApi_args getEmptyArgsInstance() { + return new addRestApi_args(); + } + + public AsyncMethodCallback getResultHandler(final AsyncFrameBuffer fb, final int seqid) { + final org.apache.thrift.AsyncProcessFunction fcall = this; + return new AsyncMethodCallback() { + public void onComplete(Void o) { + addRestApi_result result = new addRestApi_result(); + try { + fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid); + return; + } catch (Exception e) { + LOGGER.error("Exception writing to internal frame buffer", e); + } + fb.close(); + } + public void onError(Exception e) { + byte msgType = org.apache.thrift.protocol.TMessageType.REPLY; + org.apache.thrift.TBase msg; + addRestApi_result result = new addRestApi_result(); + { + msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; + msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage()); + } + try { + fcall.sendResponse(fb,msg,msgType,seqid); + return; + } catch (Exception ex) { + LOGGER.error("Exception writing to internal frame buffer", ex); + } + fb.close(); + } + }; + } + + protected boolean isOneway() { + return false; + } + + public void start(I iface, addRestApi_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws TException { + iface.addRestApi(args.restApiInfo,resultHandler); + } + } + } public static class registerInterpreterProcess_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { @@ -13549,4 +13677,618 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getParagraphList_res } + public static class addRestApi_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("addRestApi_args"); + + private static final org.apache.thrift.protocol.TField REST_API_INFO_FIELD_DESC = new org.apache.thrift.protocol.TField("restApiInfo", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new addRestApi_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new addRestApi_argsTupleSchemeFactory()); + } + + public RestApiInfo restApiInfo; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + REST_API_INFO((short)1, "restApiInfo"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // REST_API_INFO + return REST_API_INFO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.REST_API_INFO, new org.apache.thrift.meta_data.FieldMetaData("restApiInfo", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, RestApiInfo.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(addRestApi_args.class, metaDataMap); + } + + public addRestApi_args() { + } + + public addRestApi_args( + RestApiInfo restApiInfo) + { + this(); + this.restApiInfo = restApiInfo; + } + + /** + * Performs a deep copy on other. + */ + public addRestApi_args(addRestApi_args other) { + if (other.isSetRestApiInfo()) { + this.restApiInfo = new RestApiInfo(other.restApiInfo); + } + } + + public addRestApi_args deepCopy() { + return new addRestApi_args(this); + } + + @Override + public void clear() { + this.restApiInfo = null; + } + + public RestApiInfo getRestApiInfo() { + return this.restApiInfo; + } + + public addRestApi_args setRestApiInfo(RestApiInfo restApiInfo) { + this.restApiInfo = restApiInfo; + return this; + } + + public void unsetRestApiInfo() { + this.restApiInfo = null; + } + + /** Returns true if field restApiInfo is set (has been assigned a value) and false otherwise */ + public boolean isSetRestApiInfo() { + return this.restApiInfo != null; + } + + public void setRestApiInfoIsSet(boolean value) { + if (!value) { + this.restApiInfo = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case REST_API_INFO: + if (value == null) { + unsetRestApiInfo(); + } else { + setRestApiInfo((RestApiInfo)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case REST_API_INFO: + return getRestApiInfo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case REST_API_INFO: + return isSetRestApiInfo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof addRestApi_args) + return this.equals((addRestApi_args)that); + return false; + } + + public boolean equals(addRestApi_args that) { + if (that == null) + return false; + + boolean this_present_restApiInfo = true && this.isSetRestApiInfo(); + boolean that_present_restApiInfo = true && that.isSetRestApiInfo(); + if (this_present_restApiInfo || that_present_restApiInfo) { + if (!(this_present_restApiInfo && that_present_restApiInfo)) + return false; + if (!this.restApiInfo.equals(that.restApiInfo)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_restApiInfo = true && (isSetRestApiInfo()); + list.add(present_restApiInfo); + if (present_restApiInfo) + list.add(restApiInfo); + + return list.hashCode(); + } + + @Override + public int compareTo(addRestApi_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetRestApiInfo()).compareTo(other.isSetRestApiInfo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRestApiInfo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.restApiInfo, other.restApiInfo); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("addRestApi_args("); + boolean first = true; + + sb.append("restApiInfo:"); + if (this.restApiInfo == null) { + sb.append("null"); + } else { + sb.append(this.restApiInfo); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + if (restApiInfo != null) { + restApiInfo.validate(); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class addRestApi_argsStandardSchemeFactory implements SchemeFactory { + public addRestApi_argsStandardScheme getScheme() { + return new addRestApi_argsStandardScheme(); + } + } + + private static class addRestApi_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, addRestApi_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // REST_API_INFO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.restApiInfo = new RestApiInfo(); + struct.restApiInfo.read(iprot); + struct.setRestApiInfoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, addRestApi_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.restApiInfo != null) { + oprot.writeFieldBegin(REST_API_INFO_FIELD_DESC); + struct.restApiInfo.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class addRestApi_argsTupleSchemeFactory implements SchemeFactory { + public addRestApi_argsTupleScheme getScheme() { + return new addRestApi_argsTupleScheme(); + } + } + + private static class addRestApi_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, addRestApi_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetRestApiInfo()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetRestApiInfo()) { + struct.restApiInfo.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, addRestApi_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.restApiInfo = new RestApiInfo(); + struct.restApiInfo.read(iprot); + struct.setRestApiInfoIsSet(true); + } + } + } + + } + + public static class addRestApi_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("addRestApi_result"); + + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new addRestApi_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new addRestApi_resultTupleSchemeFactory()); + } + + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { +; + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(addRestApi_result.class, metaDataMap); + } + + public addRestApi_result() { + } + + /** + * Performs a deep copy on other. + */ + public addRestApi_result(addRestApi_result other) { + } + + public addRestApi_result deepCopy() { + return new addRestApi_result(this); + } + + @Override + public void clear() { + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof addRestApi_result) + return this.equals((addRestApi_result)that); + return false; + } + + public boolean equals(addRestApi_result that) { + if (that == null) + return false; + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + return list.hashCode(); + } + + @Override + public int compareTo(addRestApi_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("addRestApi_result("); + boolean first = true; + + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class addRestApi_resultStandardSchemeFactory implements SchemeFactory { + public addRestApi_resultStandardScheme getScheme() { + return new addRestApi_resultStandardScheme(); + } + } + + private static class addRestApi_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, addRestApi_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, addRestApi_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class addRestApi_resultTupleSchemeFactory implements SchemeFactory { + public addRestApi_resultTupleScheme getScheme() { + return new addRestApi_resultTupleScheme(); + } + } + + private static class addRestApi_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, addRestApi_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, addRestApi_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + } + } + + } + } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java index 9e5a0b49f07..05e118141b6 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java @@ -16,7 +16,7 @@ * limitations under the License. */ /** - * Autogenerated by Thrift Compiler (0.9.2) + * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java index 0f9aeca1dd7..574edfbc0c8 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java @@ -16,7 +16,7 @@ * limitations under the License. */ /** - * Autogenerated by Thrift Compiler (0.9.2) + * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2019-3-4") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2019-04-19") public class RemoteInterpreterResult implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterResult"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResultMessage.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResultMessage.java index 5809e4fde78..c55a97f63f9 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResultMessage.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResultMessage.java @@ -16,7 +16,7 @@ * limitations under the License. */ /** - * Autogenerated by Thrift Compiler (0.9.2) + * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2019-3-4") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2019-04-19") public class RemoteInterpreterResultMessage implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterResultMessage"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java index 396fc9441b9..914ad86c51e 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java @@ -16,7 +16,7 @@ * limitations under the License. */ /** - * Autogenerated by Thrift Compiler (0.9.2) + * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2019-3-4") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2019-04-19") public class RemoteInterpreterService { public interface Iface { @@ -8121,7 +8121,7 @@ public void setFieldValue(_Fields field, Object value) { public Object getFieldValue(_Fields field) { switch (field) { case SUCCESS: - return Integer.valueOf(getSuccess()); + return getSuccess(); } throw new IllegalStateException(); @@ -9498,7 +9498,7 @@ public Object getFieldValue(_Fields field) { return getBuf(); case CURSOR: - return Integer.valueOf(getCursor()); + return getCursor(); case INTERPRETER_CONTEXT: return getInterpreterContext(); @@ -14002,7 +14002,7 @@ public void setFieldValue(_Fields field, Object value) { public Object getFieldValue(_Fields field) { switch (field) { case SUCCESS: - return Boolean.valueOf(isSuccess()); + return isSuccess(); } throw new IllegalStateException(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RestApiInfo.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RestApiInfo.java new file mode 100644 index 00000000000..fdb3e00f792 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RestApiInfo.java @@ -0,0 +1,833 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Autogenerated by Thrift Compiler (0.9.3) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.zeppelin.interpreter.thrift; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import org.apache.thrift.async.AsyncMethodCallback; +import org.apache.thrift.server.AbstractNonblockingServer.*; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import javax.annotation.Generated; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2019-04-19") +public class RestApiInfo implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RestApiInfo"); + + private static final org.apache.thrift.protocol.TField INTP_GROUP_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("intpGroupId", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField NOTE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("noteId", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField ENDPOINT_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("endpointName", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField HOSTNAME_FIELD_DESC = new org.apache.thrift.protocol.TField("hostname", org.apache.thrift.protocol.TType.STRING, (short)4); + private static final org.apache.thrift.protocol.TField SERVER_PORT_FIELD_DESC = new org.apache.thrift.protocol.TField("serverPort", org.apache.thrift.protocol.TType.I32, (short)5); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new RestApiInfoStandardSchemeFactory()); + schemes.put(TupleScheme.class, new RestApiInfoTupleSchemeFactory()); + } + + public String intpGroupId; // required + public String noteId; // required + public String endpointName; // required + public String hostname; // required + public int serverPort; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + INTP_GROUP_ID((short)1, "intpGroupId"), + NOTE_ID((short)2, "noteId"), + ENDPOINT_NAME((short)3, "endpointName"), + HOSTNAME((short)4, "hostname"), + SERVER_PORT((short)5, "serverPort"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // INTP_GROUP_ID + return INTP_GROUP_ID; + case 2: // NOTE_ID + return NOTE_ID; + case 3: // ENDPOINT_NAME + return ENDPOINT_NAME; + case 4: // HOSTNAME + return HOSTNAME; + case 5: // SERVER_PORT + return SERVER_PORT; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __SERVERPORT_ISSET_ID = 0; + private byte __isset_bitfield = 0; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.INTP_GROUP_ID, new org.apache.thrift.meta_data.FieldMetaData("intpGroupId", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.NOTE_ID, new org.apache.thrift.meta_data.FieldMetaData("noteId", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.ENDPOINT_NAME, new org.apache.thrift.meta_data.FieldMetaData("endpointName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.HOSTNAME, new org.apache.thrift.meta_data.FieldMetaData("hostname", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.SERVER_PORT, new org.apache.thrift.meta_data.FieldMetaData("serverPort", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(RestApiInfo.class, metaDataMap); + } + + public RestApiInfo() { + } + + public RestApiInfo( + String intpGroupId, + String noteId, + String endpointName, + String hostname, + int serverPort) + { + this(); + this.intpGroupId = intpGroupId; + this.noteId = noteId; + this.endpointName = endpointName; + this.hostname = hostname; + this.serverPort = serverPort; + setServerPortIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public RestApiInfo(RestApiInfo other) { + __isset_bitfield = other.__isset_bitfield; + if (other.isSetIntpGroupId()) { + this.intpGroupId = other.intpGroupId; + } + if (other.isSetNoteId()) { + this.noteId = other.noteId; + } + if (other.isSetEndpointName()) { + this.endpointName = other.endpointName; + } + if (other.isSetHostname()) { + this.hostname = other.hostname; + } + this.serverPort = other.serverPort; + } + + public RestApiInfo deepCopy() { + return new RestApiInfo(this); + } + + @Override + public void clear() { + this.intpGroupId = null; + this.noteId = null; + this.endpointName = null; + this.hostname = null; + setServerPortIsSet(false); + this.serverPort = 0; + } + + public String getIntpGroupId() { + return this.intpGroupId; + } + + public RestApiInfo setIntpGroupId(String intpGroupId) { + this.intpGroupId = intpGroupId; + return this; + } + + public void unsetIntpGroupId() { + this.intpGroupId = null; + } + + /** Returns true if field intpGroupId is set (has been assigned a value) and false otherwise */ + public boolean isSetIntpGroupId() { + return this.intpGroupId != null; + } + + public void setIntpGroupIdIsSet(boolean value) { + if (!value) { + this.intpGroupId = null; + } + } + + public String getNoteId() { + return this.noteId; + } + + public RestApiInfo setNoteId(String noteId) { + this.noteId = noteId; + return this; + } + + public void unsetNoteId() { + this.noteId = null; + } + + /** Returns true if field noteId is set (has been assigned a value) and false otherwise */ + public boolean isSetNoteId() { + return this.noteId != null; + } + + public void setNoteIdIsSet(boolean value) { + if (!value) { + this.noteId = null; + } + } + + public String getEndpointName() { + return this.endpointName; + } + + public RestApiInfo setEndpointName(String endpointName) { + this.endpointName = endpointName; + return this; + } + + public void unsetEndpointName() { + this.endpointName = null; + } + + /** Returns true if field endpointName is set (has been assigned a value) and false otherwise */ + public boolean isSetEndpointName() { + return this.endpointName != null; + } + + public void setEndpointNameIsSet(boolean value) { + if (!value) { + this.endpointName = null; + } + } + + public String getHostname() { + return this.hostname; + } + + public RestApiInfo setHostname(String hostname) { + this.hostname = hostname; + return this; + } + + public void unsetHostname() { + this.hostname = null; + } + + /** Returns true if field hostname is set (has been assigned a value) and false otherwise */ + public boolean isSetHostname() { + return this.hostname != null; + } + + public void setHostnameIsSet(boolean value) { + if (!value) { + this.hostname = null; + } + } + + public int getServerPort() { + return this.serverPort; + } + + public RestApiInfo setServerPort(int serverPort) { + this.serverPort = serverPort; + setServerPortIsSet(true); + return this; + } + + public void unsetServerPort() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __SERVERPORT_ISSET_ID); + } + + /** Returns true if field serverPort is set (has been assigned a value) and false otherwise */ + public boolean isSetServerPort() { + return EncodingUtils.testBit(__isset_bitfield, __SERVERPORT_ISSET_ID); + } + + public void setServerPortIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __SERVERPORT_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case INTP_GROUP_ID: + if (value == null) { + unsetIntpGroupId(); + } else { + setIntpGroupId((String)value); + } + break; + + case NOTE_ID: + if (value == null) { + unsetNoteId(); + } else { + setNoteId((String)value); + } + break; + + case ENDPOINT_NAME: + if (value == null) { + unsetEndpointName(); + } else { + setEndpointName((String)value); + } + break; + + case HOSTNAME: + if (value == null) { + unsetHostname(); + } else { + setHostname((String)value); + } + break; + + case SERVER_PORT: + if (value == null) { + unsetServerPort(); + } else { + setServerPort((Integer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case INTP_GROUP_ID: + return getIntpGroupId(); + + case NOTE_ID: + return getNoteId(); + + case ENDPOINT_NAME: + return getEndpointName(); + + case HOSTNAME: + return getHostname(); + + case SERVER_PORT: + return getServerPort(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case INTP_GROUP_ID: + return isSetIntpGroupId(); + case NOTE_ID: + return isSetNoteId(); + case ENDPOINT_NAME: + return isSetEndpointName(); + case HOSTNAME: + return isSetHostname(); + case SERVER_PORT: + return isSetServerPort(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof RestApiInfo) + return this.equals((RestApiInfo)that); + return false; + } + + public boolean equals(RestApiInfo that) { + if (that == null) + return false; + + boolean this_present_intpGroupId = true && this.isSetIntpGroupId(); + boolean that_present_intpGroupId = true && that.isSetIntpGroupId(); + if (this_present_intpGroupId || that_present_intpGroupId) { + if (!(this_present_intpGroupId && that_present_intpGroupId)) + return false; + if (!this.intpGroupId.equals(that.intpGroupId)) + return false; + } + + boolean this_present_noteId = true && this.isSetNoteId(); + boolean that_present_noteId = true && that.isSetNoteId(); + if (this_present_noteId || that_present_noteId) { + if (!(this_present_noteId && that_present_noteId)) + return false; + if (!this.noteId.equals(that.noteId)) + return false; + } + + boolean this_present_endpointName = true && this.isSetEndpointName(); + boolean that_present_endpointName = true && that.isSetEndpointName(); + if (this_present_endpointName || that_present_endpointName) { + if (!(this_present_endpointName && that_present_endpointName)) + return false; + if (!this.endpointName.equals(that.endpointName)) + return false; + } + + boolean this_present_hostname = true && this.isSetHostname(); + boolean that_present_hostname = true && that.isSetHostname(); + if (this_present_hostname || that_present_hostname) { + if (!(this_present_hostname && that_present_hostname)) + return false; + if (!this.hostname.equals(that.hostname)) + return false; + } + + boolean this_present_serverPort = true; + boolean that_present_serverPort = true; + if (this_present_serverPort || that_present_serverPort) { + if (!(this_present_serverPort && that_present_serverPort)) + return false; + if (this.serverPort != that.serverPort) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_intpGroupId = true && (isSetIntpGroupId()); + list.add(present_intpGroupId); + if (present_intpGroupId) + list.add(intpGroupId); + + boolean present_noteId = true && (isSetNoteId()); + list.add(present_noteId); + if (present_noteId) + list.add(noteId); + + boolean present_endpointName = true && (isSetEndpointName()); + list.add(present_endpointName); + if (present_endpointName) + list.add(endpointName); + + boolean present_hostname = true && (isSetHostname()); + list.add(present_hostname); + if (present_hostname) + list.add(hostname); + + boolean present_serverPort = true; + list.add(present_serverPort); + if (present_serverPort) + list.add(serverPort); + + return list.hashCode(); + } + + @Override + public int compareTo(RestApiInfo other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetIntpGroupId()).compareTo(other.isSetIntpGroupId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIntpGroupId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.intpGroupId, other.intpGroupId); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetNoteId()).compareTo(other.isSetNoteId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetNoteId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.noteId, other.noteId); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetEndpointName()).compareTo(other.isSetEndpointName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetEndpointName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.endpointName, other.endpointName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetHostname()).compareTo(other.isSetHostname()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetHostname()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.hostname, other.hostname); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetServerPort()).compareTo(other.isSetServerPort()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetServerPort()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.serverPort, other.serverPort); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("RestApiInfo("); + boolean first = true; + + sb.append("intpGroupId:"); + if (this.intpGroupId == null) { + sb.append("null"); + } else { + sb.append(this.intpGroupId); + } + first = false; + if (!first) sb.append(", "); + sb.append("noteId:"); + if (this.noteId == null) { + sb.append("null"); + } else { + sb.append(this.noteId); + } + first = false; + if (!first) sb.append(", "); + sb.append("endpointName:"); + if (this.endpointName == null) { + sb.append("null"); + } else { + sb.append(this.endpointName); + } + first = false; + if (!first) sb.append(", "); + sb.append("hostname:"); + if (this.hostname == null) { + sb.append("null"); + } else { + sb.append(this.hostname); + } + first = false; + if (!first) sb.append(", "); + sb.append("serverPort:"); + sb.append(this.serverPort); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class RestApiInfoStandardSchemeFactory implements SchemeFactory { + public RestApiInfoStandardScheme getScheme() { + return new RestApiInfoStandardScheme(); + } + } + + private static class RestApiInfoStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, RestApiInfo struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // INTP_GROUP_ID + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.intpGroupId = iprot.readString(); + struct.setIntpGroupIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // NOTE_ID + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.noteId = iprot.readString(); + struct.setNoteIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // ENDPOINT_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.endpointName = iprot.readString(); + struct.setEndpointNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // HOSTNAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.hostname = iprot.readString(); + struct.setHostnameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // SERVER_PORT + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.serverPort = iprot.readI32(); + struct.setServerPortIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, RestApiInfo struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.intpGroupId != null) { + oprot.writeFieldBegin(INTP_GROUP_ID_FIELD_DESC); + oprot.writeString(struct.intpGroupId); + oprot.writeFieldEnd(); + } + if (struct.noteId != null) { + oprot.writeFieldBegin(NOTE_ID_FIELD_DESC); + oprot.writeString(struct.noteId); + oprot.writeFieldEnd(); + } + if (struct.endpointName != null) { + oprot.writeFieldBegin(ENDPOINT_NAME_FIELD_DESC); + oprot.writeString(struct.endpointName); + oprot.writeFieldEnd(); + } + if (struct.hostname != null) { + oprot.writeFieldBegin(HOSTNAME_FIELD_DESC); + oprot.writeString(struct.hostname); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(SERVER_PORT_FIELD_DESC); + oprot.writeI32(struct.serverPort); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class RestApiInfoTupleSchemeFactory implements SchemeFactory { + public RestApiInfoTupleScheme getScheme() { + return new RestApiInfoTupleScheme(); + } + } + + private static class RestApiInfoTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, RestApiInfo struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIntpGroupId()) { + optionals.set(0); + } + if (struct.isSetNoteId()) { + optionals.set(1); + } + if (struct.isSetEndpointName()) { + optionals.set(2); + } + if (struct.isSetHostname()) { + optionals.set(3); + } + if (struct.isSetServerPort()) { + optionals.set(4); + } + oprot.writeBitSet(optionals, 5); + if (struct.isSetIntpGroupId()) { + oprot.writeString(struct.intpGroupId); + } + if (struct.isSetNoteId()) { + oprot.writeString(struct.noteId); + } + if (struct.isSetEndpointName()) { + oprot.writeString(struct.endpointName); + } + if (struct.isSetHostname()) { + oprot.writeString(struct.hostname); + } + if (struct.isSetServerPort()) { + oprot.writeI32(struct.serverPort); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, RestApiInfo struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(5); + if (incoming.get(0)) { + struct.intpGroupId = iprot.readString(); + struct.setIntpGroupIdIsSet(true); + } + if (incoming.get(1)) { + struct.noteId = iprot.readString(); + struct.setNoteIdIsSet(true); + } + if (incoming.get(2)) { + struct.endpointName = iprot.readString(); + struct.setEndpointNameIsSet(true); + } + if (incoming.get(3)) { + struct.hostname = iprot.readString(); + struct.setHostnameIsSet(true); + } + if (incoming.get(4)) { + struct.serverPort = iprot.readI32(); + struct.setServerPortIsSet(true); + } + } + } + +} + diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RunParagraphsEvent.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RunParagraphsEvent.java index 3c0da8a2060..95222def51f 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RunParagraphsEvent.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RunParagraphsEvent.java @@ -16,7 +16,7 @@ * limitations under the License. */ /** - * Autogenerated by Thrift Compiler (0.9.2) + * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2019-3-4") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2019-04-19") public class RunParagraphsEvent implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RunParagraphsEvent"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/ServiceException.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/ServiceException.java index fe6b593f36d..477c73972da 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/ServiceException.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/ServiceException.java @@ -16,7 +16,7 @@ * limitations under the License. */ /** - * Autogenerated by Thrift Compiler (0.9.2) + * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2019-3-4") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2019-04-19") public class ServiceException extends TException implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ServiceException"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/serving/JsonApiHandler.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/serving/JsonApiHandler.java new file mode 100644 index 00000000000..df2860337bd --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/serving/JsonApiHandler.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.serving; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.PrintWriter; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +public abstract class JsonApiHandler extends RestApiHandler { + Gson gson = new Gson(); + + @Override + protected void handle(HttpServletRequest request, HttpServletResponse response) throws IOException { + BufferedReader reader = request.getReader(); + T param = gson.fromJson(reader, new TypeToken() {}.getType()); + + Object ret = handle(param); + + response.setStatus(HttpServletResponse.SC_OK); + PrintWriter responseWriter = response.getWriter(); + gson.toJson(ret, responseWriter); + } + + public abstract Object handle(T t); +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/serving/Metric.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/serving/Metric.java new file mode 100644 index 00000000000..32aff401bc8 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/serving/Metric.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.serving; + +public class Metric { + private final long count; + private final double sum; + + public Metric(long count, double sum) { + this.count = count; + this.sum = sum; + } + + public long getCount() { + return count; + } + + public double getSum() { + return sum; + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/serving/MetricStorage.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/serving/MetricStorage.java new file mode 100644 index 00000000000..8da9f9b7114 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/serving/MetricStorage.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.serving; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +public interface MetricStorage { + /** + * Add a metric data + */ + Metric add(Date date, String endpoint, String key, double n); + + /** + * Get series of (aggregated) metric data + */ + List> get(Date from, Date to, String noteId, String revId, String endpoint); +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/serving/RedisMetricStorage.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/serving/RedisMetricStorage.java new file mode 100644 index 00000000000..757ffb6bcf3 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/serving/RedisMetricStorage.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.serving; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.exceptions.JedisConnectionException; + +public class RedisMetricStorage implements MetricStorage { + public static final int DEFAULT_METRIC_EXPIRE_SEC = 60 * 30; + private static final String META_FIELD_COUNT_POSTFIX = "__--zmcnt"; + + private Jedis redis; + private final String host; + private final int port; + + private final String noteId; + private final String revId; + private final SimpleDateFormat dateFormat; + private final int metricExpireSec; + + public RedisMetricStorage() { + // serviceName is "--" + String serviceName = System.getenv("SERVICE_NAME"); + String[] names = serviceName.split("-"); + noteId = names[1]; + revId = names[2]; + + metricExpireSec = DEFAULT_METRIC_EXPIRE_SEC; + + String[] hostPort = getRedisAddr().split(":"); + this.host = hostPort[0]; + this.port = Integer.parseInt(hostPort[1]); + redis = new Jedis(host, port); + dateFormat = new SimpleDateFormat("yyyy-MM-dd_HH:mm"); + } + + @VisibleForTesting + public RedisMetricStorage( + String redisAddr, + String noteId, + String revId, + int metricExpireSec + ) { + this.noteId = noteId; + this.revId = revId; + this.metricExpireSec = metricExpireSec; + + String[] hostPort = redisAddr.split(":"); + this.host = hostPort[0]; + this.port = Integer.parseInt(hostPort[1]); + redis = new Jedis(host, port); + dateFormat = new SimpleDateFormat("yyyy-MM-dd_HH:mm"); + } + + private Jedis redis() { + if (redis == null) { + synchronized (this) { + if (redis == null) { + redis = new Jedis(host, port); + } + } + } + + return redis; + } + + private void reset() { + redis = null; + } + + String getRedisAddr() { + String addr = System.getenv( + ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_METRIC_REDIS_ADDR.name()); + return addr; + } + + String redisKey(Date date, String endpoint) { + return redisKey(date, noteId, revId, endpoint); + } + + String redisKey(Date date, String noteId, String revId, String endpoint) { + long ts = date.getTime()/1000; + ts -= ts % 60; + String dateString = Long.toString(ts); + return String.format("%s.%s.%s.%s", noteId, revId, endpoint, dateString).toLowerCase(); + } + + private void setExpire(Date updateDate, String key) { + try { + redis().expireAt(key, (updateDate.getTime() / 1000) + metricExpireSec); + } catch (JedisConnectionException e) { + reset(); + throw e; + } + } + + @Override + public Metric add(Date date, String endpoint, String field, double n) { + String key = redisKey(date, endpoint); + Double r; + long c = 0; + try { + r = redis().hincrByFloat(key, field, n); + c = redis().hincrBy(key, field + META_FIELD_COUNT_POSTFIX, 1); + } catch (JedisConnectionException e) { + reset(); + throw e; + } + setExpire(date, key); + return new Metric(c, r); + } + + @VisibleForTesting + public Metric get(Date date, String endpoint, String field) { + String key = redisKey(date, endpoint); + try { + double r = Double.parseDouble(redis().hget(key, field)); + long c = Long.parseLong(redis().hget(key, field + META_FIELD_COUNT_POSTFIX)); + return new Metric(c, r); + } catch (NullPointerException e) { + return null; + } catch (JedisConnectionException e) { + reset(); + throw e; + } + } + + @VisibleForTesting + public Map get(Date date, String endpoint) { + String key = redisKey(date, endpoint); + try { + return getMetricMapFromKey(key); + } catch (JedisConnectionException e) { + reset(); + throw e; + } + } + + @VisibleForTesting + public Map get(Date date, String noteId, String revId, String endpoint) { + String key = redisKey(date, noteId, revId, endpoint); + try { + return getMetricMapFromKey(key); + } catch (JedisConnectionException e) { + reset(); + throw e; + } + } + + private Map getMetricMapFromKey(String key) { + Map metricMap = new HashMap<>(); + Map map = redis().hgetAll(key); + for (String k : map.keySet()) { + if (k.endsWith(META_FIELD_COUNT_POSTFIX)) { + continue; + } + + if (map.containsKey(k + META_FIELD_COUNT_POSTFIX)) { + try { + double r = Double.parseDouble(map.get(k)); + long c = Long.parseLong(map.get(k + META_FIELD_COUNT_POSTFIX)); + metricMap.put(k, new Metric(c, r)); + } catch (NullPointerException e) { + metricMap.put(k, null); + } + } + } + return metricMap; + } + + @Override + public synchronized List> get(Date from, Date to, String noteId, String revId, String endpoint) { + List> series = new LinkedList<>(); + + Date p = from; + while (p.before(to)) { + series.add(ImmutableMap.of( + "timestamp", p.getTime()/1000, + "metric", get(p, noteId, revId, endpoint) + )); + p = new Date(p.getTime() + (1000 * 60)); + } + + return series; + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/serving/RestApiHandler.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/serving/RestApiHandler.java new file mode 100644 index 00000000000..ca8efea27ff --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/serving/RestApiHandler.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.serving; + +import java.io.IOException; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * + */ +public abstract class RestApiHandler { + protected abstract void handle(HttpServletRequest request, HttpServletResponse response) throws IOException; +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/serving/RestApiServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/serving/RestApiServer.java new file mode 100644 index 00000000000..4b1f64e8088 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/serving/RestApiServer.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.serving; + +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServlet; + +/** + * Provides an rest api endpoint in the interpreter process. + * z.addRestApi() will add endpoint here. + */ +public class RestApiServer extends HttpServlet { + private static final Logger LOGGER = LoggerFactory.getLogger(RestApiServer.class); + public static int PORT; + + private Server server; + + private static RestApiServer singletonInstance; + + private final ConcurrentHashMap endpoints = new ConcurrentHashMap(); + private final ConcurrentLinkedQueue metricStorages = new ConcurrentLinkedQueue<>(); + + public RestApiServer() { + } + + public static void setPort(int port) { + PORT = port; + } + + public static int getPort() { + return PORT; + } + + public static RestApiServer singleton() { + if (singletonInstance == null) { + singletonInstance = new RestApiServer(); + } + + return singletonInstance; + } + + private synchronized void start() { + if (server != null) { + return; + } + + ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); + context.setContextPath("/"); + + server = new Server(PORT); + server.setHandler(context); + + context.addServlet(RestApiServlet.class, "/*"); + + try { + server.start(); + } catch (Exception e) { + LOGGER.error("Can't start rest api server", e); + } + } + + public void shutdown() throws Exception { + if (server != null) { + server.stop(); + } + } + + public void addEndpoint(String endpoint, RestApiHandler handler) { + // start server if not started + start(); + + endpoints.put(endpoint, handler); + } + + public RestApiHandler getEndpoint(String endpoint) { + return endpoints.get(endpoint); + } + + public ConcurrentLinkedQueue getMetricStorages() { + return metricStorages; + } + + public void addMetricStorage(MetricStorage storage) { + metricStorages.add(storage); + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/serving/RestApiServlet.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/serving/RestApiServlet.java new file mode 100644 index 00000000000..5aeb073a688 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/serving/RestApiServlet.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.serving; + +import java.io.IOException; +import java.util.Collection; +import java.util.Date; +import java.util.concurrent.ConcurrentLinkedQueue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Servlet that handles request to rest api endpoint added by z.addRestApi() + */ +public class RestApiServlet extends HttpServlet { + private static final Logger LOGGER = LoggerFactory.getLogger(RestApiServlet.class); + + @Override + protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + String path = request.getRequestURI().substring(request.getContextPath().length()); + + if (path.length() == 0) { + LOGGER.warn("Empty request path"); + response.setStatus(HttpServletResponse.SC_BAD_REQUEST); + return; + } + + RestApiServer server = RestApiServer.singleton(); + String endpoint = getEndpointNameFromRequestPath(path); + RestApiHandler handler = server.getEndpoint(endpoint); + if (handler == null) { + LOGGER.warn("Endpoint {} does not exists", endpoint); + response.setStatus(HttpServletResponse.SC_NOT_FOUND); + return; + } + + long start = System.currentTimeMillis(); + handler.handle(request, response); + long end = System.currentTimeMillis(); + + try { + writeMetrics(server, endpoint, request, response, end - start); + } catch(Throwable e) { + LOGGER.error("Failed to write metric", e); + } + } + + void writeMetrics(RestApiServer server, + String endpoint, + HttpServletRequest request, + HttpServletResponse response, + long elapsed) { + // metrics + String keyCountPerStatus = String.format("elapsed.%d", response.getStatus()); + ConcurrentLinkedQueue metricStorages = server.getMetricStorages(); + Date now = new Date(); + + String metricHeaderPrefix = "z-metric-"; + + metricStorages.forEach(m -> { + m.add(now, endpoint, keyCountPerStatus, elapsed); + + Collection headers = response.getHeaderNames(); + for (String header : headers) { + if (header.toLowerCase().startsWith(metricHeaderPrefix)) { + String userMetricName = header.substring(metricHeaderPrefix.length()); + try { + double userMetricValue = Double.parseDouble(response.getHeader(header)); + m.add(now, endpoint, userMetricName, userMetricValue); + } catch (Exception e) { + LOGGER.error("Invalid custom metric value {}: {}", header, response.getHeader(header)); + } + } + } + }); + } + + String getEndpointNameFromRequestPath(String path) { + int start = 0; + if (path.charAt(0) == '/') { + start = 1; + } + + int end = path.indexOf('/', start); + if (end < 0) { + end = path.length(); + } + + return path.substring(start, end); + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/Util.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/Util.java index e36afca71a2..4a8a5c791d7 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/Util.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/Util.java @@ -17,6 +17,7 @@ package org.apache.zeppelin.util; +import java.util.Random; import org.apache.commons.lang.StringUtils; import java.io.IOException; @@ -73,4 +74,18 @@ public static String getGitTimestamp() { return StringUtils.defaultIfEmpty(gitProperties.getProperty(GIT_PROPERTIES_COMMIT_TS_KEY), StringUtils.EMPTY); } + + public static String getRandomString(int length) { + char[] chars = "abcdefghijklmnopqrstuvwxyz".toCharArray(); + + StringBuilder sb = new StringBuilder(); + Random random = new Random(); + for (int i = 0; i < length; i++) { + char c = chars[random.nextInt(chars.length)]; + sb.append(c); + } + String randomStr = sb.toString(); + + return randomStr; + } } diff --git a/zeppelin-interpreter/src/main/thrift/RemoteInterpreterEventService.thrift b/zeppelin-interpreter/src/main/thrift/RemoteInterpreterEventService.thrift index 6470a67d09d..125375a1f44 100644 --- a/zeppelin-interpreter/src/main/thrift/RemoteInterpreterEventService.thrift +++ b/zeppelin-interpreter/src/main/thrift/RemoteInterpreterEventService.thrift @@ -93,6 +93,14 @@ struct ParagraphInfo { 4: string paragraphText } +struct RestApiInfo { + 1: string intpGroupId, + 2: string noteId, + 3: string endpointName, + 4: string hostname, + 5: i32 serverPort +} + exception ServiceException{ 1: required string message; } @@ -120,4 +128,6 @@ service RemoteInterpreterEventService { binary invokeMethod(1: string intpGroupId, 2: string invokeMethodJson); list getParagraphList(1: string user, 2: string noteId) throws (1: ServiceException e); + + void addRestApi(1: RestApiInfo restApiInfo); } \ No newline at end of file diff --git a/zeppelin-interpreter/src/main/thrift/thrift.rb b/zeppelin-interpreter/src/main/thrift/thrift.rb new file mode 100644 index 00000000000..4104865cefc --- /dev/null +++ b/zeppelin-interpreter/src/main/thrift/thrift.rb @@ -0,0 +1,81 @@ +class Thrift < Formula + desc "Framework for scalable cross-language services development" + homepage "https://thrift.apache.org/" + + stable do + url "https://www.apache.org/dyn/closer.cgi?path=/thrift/0.9.3/thrift-0.9.3.tar.gz" + sha256 "b0740a070ac09adde04d43e852ce4c320564a292f26521c46b78e0641564969e" + + # Apply any necessary patches (none currently required) + [ + # Example patch: + # + # Apply THRIFT-2201 fix from master to 0.9.1 branch (required for clang to compile with C++11 support) + # %w{836d95f9f00be73c6936d407977796181d1a506c f8e14cbae1810ade4eab49d91e351459b055c81dba144c1ca3c5d6f4fe440925}, + ].each do |name, sha| + patch do + url "https://git-wip-us.apache.org/repos/asf?p=thrift.git;a=commitdiff_plain;h=#{name}" + sha256 sha + end + end + end + + bottle do + cellar :any + rebuild 1 + sha256 "f3bd35df2ba94af77f15a836668db5eb7dfc0d37c77a2f77bc6cc980e1524f27" => :sierra + sha256 "528061b3a3689341d76d76a7faaa6345100bbbadeb4055a26f1acb6377aad3ba" => :el_capitan + sha256 "7b1c9edc94356d9cb426237fab09143c64d2bb2a85d86f7d8236702fa110f90c" => :yosemite + end + + head do + url "https://git-wip-us.apache.org/repos/asf/thrift.git" + + depends_on "autoconf" => :build + depends_on "automake" => :build + depends_on "libtool" => :build + depends_on "pkg-config" => :build + end + + option "with-haskell", "Install Haskell binding" + option "with-erlang", "Install Erlang binding" + option "with-java", "Install Java binding" + option "with-perl", "Install Perl binding" + option "with-php", "Install PHP binding" + option "with-libevent", "Install nonblocking server libraries" + + depends_on "bison" => :build + depends_on "boost" + depends_on "openssl" + depends_on "libevent" => :optional + # depends_on :python => :optional + + def install + system "./bootstrap.sh" unless build.stable? + + exclusions = ["--without-ruby", "--disable-tests", "--without-php_extension"] + + exclusions << "--without-python" if build.without? "python" + exclusions << "--without-haskell" if build.without? "haskell" + exclusions << "--without-java" if build.without? "java" + exclusions << "--without-perl" if build.without? "perl" + exclusions << "--without-php" if build.without? "php" + exclusions << "--without-erlang" if build.without? "erlang" + + ENV.cxx11 if MacOS.version >= :mavericks && ENV.compiler == :clang + + # Don't install extensions to /usr: + ENV["PY_PREFIX"] = prefix + ENV["PHP_PREFIX"] = prefix + + system "./configure", "--disable-debug", + "--prefix=#{prefix}", + "--libdir=#{lib}", + "--with-openssl=#{Formula["openssl"].opt_prefix}", + *exclusions + ENV.deparallelize + system "make" + system "make", "install" + end + +end diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServerTest.java index 7beeee84a72..7967930b510 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServerTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServerTest.java @@ -18,6 +18,7 @@ package org.apache.zeppelin.interpreter.remote; import org.apache.thrift.TException; +import org.apache.zeppelin.interpreter.Constants; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; @@ -43,7 +44,7 @@ public class RemoteInterpreterServerTest { @Test public void testStartStop() throws InterruptedException, IOException, TException { RemoteInterpreterServer server = new RemoteInterpreterServer("localhost", - RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(), ":", "groupId", true); + RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(), ":", "groupId", 0, true); startRemoteInterpreterServer(server, 10 * 1000); stopRemoteInterpreterServer(server, 10 * 10000); @@ -52,7 +53,7 @@ public void testStartStop() throws InterruptedException, IOException, TException @Test public void testStartStopWithQueuedEvents() throws InterruptedException, IOException, TException { RemoteInterpreterServer server = new RemoteInterpreterServer("localhost", - RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(), ":", "groupId", true); + RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(), ":", "groupId", 0, true); server.intpEventClient = mock(RemoteInterpreterEventClient.class); startRemoteInterpreterServer(server, 10 * 1000); @@ -60,7 +61,7 @@ public void testStartStopWithQueuedEvents() throws InterruptedException, IOExcep stopRemoteInterpreterServer(server, 10 * 10000); } - private void startRemoteInterpreterServer(RemoteInterpreterServer server, int timeout) + public static void startRemoteInterpreterServer(RemoteInterpreterServer server, int timeout) throws InterruptedException { assertEquals(false, server.isRunning()); server.start(); @@ -76,7 +77,7 @@ private void startRemoteInterpreterServer(RemoteInterpreterServer server, int ti server.getPort())); } - private void stopRemoteInterpreterServer(RemoteInterpreterServer server, int timeout) + public static void stopRemoteInterpreterServer(RemoteInterpreterServer server, int timeout) throws TException, InterruptedException { assertEquals(true, server.isRunning()); server.shutdown(); @@ -95,7 +96,7 @@ private void stopRemoteInterpreterServer(RemoteInterpreterServer server, int tim @Test public void testInterpreter() throws IOException, TException, InterruptedException { final RemoteInterpreterServer server = new RemoteInterpreterServer("localhost", - RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(), ":", "groupId", true); + RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(), ":", "groupId", 0, true); server.intpEventClient = mock(RemoteInterpreterEventClient.class); Map intpProperties = new HashMap<>(); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/serving/RedisMetricStorageTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/serving/RedisMetricStorageTest.java new file mode 100644 index 00000000000..719f00bf82a --- /dev/null +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/serving/RedisMetricStorageTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.serving; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Date; +import java.util.Map; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import redis.clients.jedis.exceptions.JedisConnectionException; +import redis.embedded.RedisServer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +public class RedisMetricStorageTest { + static int redisPort = 16380; + private RedisServer redisServer; + + @Before + public void setUp() throws IOException, URISyntaxException { + redisServer = new RedisServer(redisPort); + redisServer.start(); + } + + @After + public void tearDown() throws InterruptedException { + redisServer.stop(); + } + + @Test + public void testAdd() throws InterruptedException { + RedisMetricStorage m = new RedisMetricStorage("localhost:" + redisPort, "note1", "rev1", 1); + Date now = new Date(); + + Metric m1 = m.add(now, "ep1", "count", 1); + assertEquals(1, m1.getCount(), 0); + assertEquals(1.0, m1.getSum(), 0); + + Metric m2 = m.add(now, "ep1", "count", 2); + assertEquals(2, m2.getCount(), 0); + assertEquals(3.0, m2.getSum(), 0); + } + + @Test + public void testExpire() throws InterruptedException { + // given + RedisMetricStorage m = new RedisMetricStorage("localhost:" + redisPort, "note1", "rev1", 1); + Date now = new Date(); + + // when + assertEquals(null, m.get(now, "ep2", "count")); + m.add(now, "ep2", "count", 1); + assertEquals(1, m.get(now, "ep2", "count").getCount()); + Thread.sleep(1000); + + // then + assertEquals(null, m.get(now, "ep2", "count")); + } + + @Test + public void testGetAll() { + // given + RedisMetricStorage m = new RedisMetricStorage("localhost:" + redisPort, "note1", "rev1", 1); + Date now = new Date(); + + // when + m.add(now, "ep3", "count", 1); + + // then + Map map = m.get(now, "ep3"); + assertEquals(1, map.get("count").getCount()); + assertEquals(1.0, map.get("count").getSum(), 1.0); + } + + @Test + public void testReconnect() throws InterruptedException, IOException { + // given + RedisMetricStorage m = new RedisMetricStorage("localhost:" + redisPort, "note1", "rev1", 1); + + Date now = new Date(); + + // when + m.add(now, "ep4", "count", 1); + redisServer.stop(); + redisServer.start(); + + // then + try { + m.add(now, "ep4", "count", 1); + assertFalse(true); + } catch (JedisConnectionException e) { + // exception expected + } + + // then + Metric metric = m.add(now, "ep4", "count", 1); + assertEquals(1, m.get(now, "ep4", "count").getCount()); + } +} diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/serving/RestApiServerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/serving/RestApiServerTest.java new file mode 100644 index 00000000000..418cb88de10 --- /dev/null +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/serving/RestApiServerTest.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.serving; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; +import org.apache.commons.httpclient.methods.GetMethod; +import org.apache.thrift.TException; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.LazyOpenInterpreter; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventClient; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; +import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +/** + * RestApiServerTest. + */ +public class RestApiServerTest { + + private RemoteInterpreterServer server; + private HttpClient client; + + @Before + public void setUp() throws TException, IOException { + server = new RemoteInterpreterServer("localhost", + RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(), ":", "groupId", 0,true); + server.setIntpEventClient(mock(RemoteInterpreterEventClient.class)); + + Map intpProperties = new HashMap<>(); + intpProperties.put("zeppelin.interpreter.localRepo", "/tmp"); + + server.createInterpreter("group_1", "session_1", RestApiTestInterpreter.class.getName(), + intpProperties, "user_1"); + RestApiTestInterpreter interpreter1 = (RestApiTestInterpreter) + ((LazyOpenInterpreter) server.getInterpreterGroup().get("session_1").get(0)) + .getInnerInterpreter(); + + final RemoteInterpreterContext intpContext = new RemoteInterpreterContext(); + intpContext.setNoteId("note_1"); + intpContext.setParagraphId("paragraph_1"); + intpContext.setLocalProperties(new HashMap<>()); + client = new HttpClient(new MultiThreadedHttpConnectionManager()); + } + + @After + public void tearDown() throws TException { + server.shutdown(); + } + + @Test + public void testAddRestApiAccessEndpoint() throws IOException, TException, InterruptedException { + // given + final RemoteInterpreterContext intpContext = new RemoteInterpreterContext(); + intpContext.setNoteId("note_1"); + intpContext.setParagraphId("paragraph_1"); + intpContext.setGui("{}"); + intpContext.setNoteGui("{}"); + intpContext.setLocalProperties(new HashMap<>()); + + // when accessing correct endpoint + server.interpret("session_1", RestApiTestInterpreter.class.getName(), "ep1", intpContext); + waitForRestApiEndpointAvailable(); + + // then + GetMethod get = new GetMethod(String.format("http://localhost:%d/%s", RestApiServer.PORT, "ep1")); + int code = client.executeMethod(get); + assertEquals(200, code); + get.releaseConnection(); + + // when accessing invalid endpoint + get = new GetMethod(String.format("http://localhost:%d/%s", RestApiServer.PORT, "ep2")); + + code = client.executeMethod(get); + assertEquals(404, code); + } + + public static class RestApiTestInterpreter extends Interpreter { + + public RestApiTestInterpreter(Properties properties) { + super(properties); + } + + @Override + public void open() { + } + + @Override + public InterpreterResult interpret(String st, InterpreterContext context) { + String endpoint = st; + context.addRestApi(endpoint, new RestApiHandler() { + + @Override + protected void handle(HttpServletRequest request, HttpServletResponse response) { + response.setStatus(HttpServletResponse.SC_OK); + } + }); + return new InterpreterResult(InterpreterResult.Code.SUCCESS); + } + + @Override + public void cancel(InterpreterContext context) throws InterpreterException { + } + + @Override + public FormType getFormType() throws InterpreterException { + return FormType.NATIVE; + } + + @Override + public int getProgress(InterpreterContext context) throws InterpreterException { + return 0; + } + + @Override + public void close() { + } + + } + + private void waitForRestApiEndpointAvailable() throws IOException { + long start = System.currentTimeMillis(); + + while (System.currentTimeMillis() - start < 10 * 1000) { + if (RemoteInterpreterUtils.checkIfRemoteEndpointAccessible("localhost", RestApiServer.PORT)) { + return; + } else { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + } + } + } + + throw new IOException("Can't access rest api endpoint"); + } +} diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/serving/RestApiServletTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/serving/RestApiServletTest.java new file mode 100644 index 00000000000..43a1200547e --- /dev/null +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/serving/RestApiServletTest.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.serving; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * RestApiServletTest. + */ +public class RestApiServletTest { + @Test + public void testGetEndpointNameFromRequestedPath() { + RestApiServlet server = new RestApiServlet(); + assertEquals("ep1", server.getEndpointNameFromRequestPath("ep1")); + assertEquals("ep1", server.getEndpointNameFromRequestPath("/ep1")); + assertEquals("ep1", server.getEndpointNameFromRequestPath("/ep1/")); + assertEquals("ep1", server.getEndpointNameFromRequestPath("/ep1/other")); + } +} diff --git a/zeppelin-plugins/launcher/k8s-standard/pom.xml b/zeppelin-plugins/launcher/k8s-standard/pom.xml index 56696b0ce7a..c85268626e0 100644 --- a/zeppelin-plugins/launcher/k8s-standard/pom.xml +++ b/zeppelin-plugins/launcher/k8s-standard/pom.xml @@ -46,6 +46,12 @@ jinjava 2.4.12 + + + io.fabric8 + kubernetes-client + 4.2.2 + diff --git a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/background/K8sNoteBackgroundTask.java b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/background/K8sNoteBackgroundTask.java new file mode 100644 index 00000000000..6f482fffbf9 --- /dev/null +++ b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/background/K8sNoteBackgroundTask.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.background; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.io.File; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import org.apache.commons.lang3.StringUtils; +import org.apache.zeppelin.interpreter.launcher.Kubectl; + +/** + * K8s note test task. + */ +public abstract class K8sNoteBackgroundTask extends NoteBackgroundTask { + private final Kubectl kubectl; + private final File k8sTemplateDir; + private final Gson gson = new Gson(); + + public K8sNoteBackgroundTask(Kubectl kubectl, TaskContext taskContext, File k8sTemplateDir) { + super(taskContext); + this.kubectl = kubectl; + this.k8sTemplateDir = k8sTemplateDir; + } + + @Override + public void start() throws IOException { + kubectl.apply(k8sTemplateDir, getTemplateBindings(), false); + } + + protected Properties getTemplateBindings() throws IOException { + TaskContext taskContext = getTaskContext(); + Properties k8sProperties = new Properties(); + String taskId = taskContext.getId(); + String servingName = getResourceName(); + + // k8s template properties + k8sProperties.put("zeppelin.k8s.background.resource.type", getResourceType()); + k8sProperties.put("zeppelin.k8s.background.resource.apiversion", getResourceApiVersion()); + + k8sProperties.put("zeppelin.k8s.background.taskId", taskId); + k8sProperties.put("zeppelin.k8s.background.namespace", kubectl.getNamespace()); + k8sProperties.put("zeppelin.k8s.background.name", servingName); + k8sProperties.put("zeppelin.k8s.background.noteId", taskContext.getNote().getId()); + k8sProperties.put("zeppelin.k8s.background.revId", taskContext.getRevId()); + k8sProperties.put("zeppelin.k8s.background.serviceContext", ""); + k8sProperties.put("zeppelin.k8s.background.autoshutdown", "true"); + return k8sProperties; + } + + protected abstract String getResourceName(); + protected abstract String getResourceType(); + protected abstract String getResourceApiVersion(); + + @Override + public void stop() throws IOException { + kubectl.apply(k8sTemplateDir, getTemplateBindings(), true); + } + + @Override + public Map getInfo() throws IOException { + String resourceJsonString; + try { + resourceJsonString = kubectl.get(getResourceType(), getResourceName()); + } catch (IOException e) { + // not exists; + return null; + } + if (StringUtils.isEmpty(resourceJsonString)) { + return null; + } + + Map resource = gson.fromJson( + resourceJsonString, new TypeToken>() {}.getType()); + return resource; + } + + protected Kubectl getKubectl() { + return kubectl; + } +} diff --git a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/background/K8sNoteBackgroundTaskManager.java b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/background/K8sNoteBackgroundTaskManager.java new file mode 100644 index 00000000000..d1e296786a4 --- /dev/null +++ b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/background/K8sNoteBackgroundTaskManager.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.background; + +import java.io.IOException; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.interpreter.launcher.Kubectl; + +/** + * Manage note test task. + */ +public abstract class K8sNoteBackgroundTaskManager extends NoteBackgroundTaskManager { + + private TaskContextStorage taskContextStorage = null; + private final Kubectl kubectl; + + public K8sNoteBackgroundTaskManager(ZeppelinConfiguration zConf) throws IOException { + super(zConf); + kubectl = new Kubectl(zConf.getK8sKubectlCmd()); + kubectl.setNamespace(Kubectl.getNamespaceFromContainer()); + } + + protected abstract TaskContextStorage createTaskContextStorage(); + + @Override + protected synchronized TaskContextStorage getTaskContextStorage() { + if (taskContextStorage == null) { + taskContextStorage = createTaskContextStorage(); + } + return taskContextStorage; + } + + protected Kubectl getKubectl() { + return kubectl; + } +} diff --git a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/BackgroundTaskLifecycleWatcherImpl.java b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/BackgroundTaskLifecycleWatcherImpl.java new file mode 100644 index 00000000000..75834583acc --- /dev/null +++ b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/BackgroundTaskLifecycleWatcherImpl.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.interpreter.launcher; + +import io.fabric8.kubernetes.client.KubernetesClientException; +import io.fabric8.kubernetes.client.Watcher; +import org.apache.zeppelin.background.BackgroundTaskLifecycleListener; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class BackgroundTaskLifecycleWatcherImpl implements Watcher { + Logger LOGGER = LoggerFactory.getLogger(BackgroundTaskLifecycleWatcherImpl.class); + + private BackgroundTaskLifecycleListener listener; + + public BackgroundTaskLifecycleWatcherImpl(BackgroundTaskLifecycleListener listener) { + this.listener = listener; + } + + @Override + public void eventReceived(Action action, T t) { + switch (action) { + case ADDED: + if (listener != null) { + listener.onBackgroundTaskStart(getTaskId(t)); + } + break; + case DELETED: + if (listener != null) { + listener.onBackgroundTaskStop(getTaskId(t)); + } + break; + case MODIFIED: + if (listener != null) { + listener.onBackgroundTaskUpdate(getTaskId(t)); + } + break; + case ERROR: + if (listener != null) { + listener.onBackgroundTaskError(getTaskId(t)); + } + break; + } + } + + @Override + public void onClose(KubernetesClientException e) { + LOGGER.info("Kubernetes watcher closed"); + } + + protected abstract String getTaskId(T t); + + public BackgroundTaskLifecycleListener getListener() { + return listener; + } + + public void setListener(BackgroundTaskLifecycleListener listener) { + this.listener = listener; + } +} diff --git a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sRemoteInterpreterProcess.java b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sRemoteInterpreterProcess.java index afa8541f445..780a7891232 100644 --- a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sRemoteInterpreterProcess.java +++ b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sRemoteInterpreterProcess.java @@ -10,15 +10,17 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.exec.ExecuteWatchdog; -import org.apache.commons.lang3.ArrayUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; +import org.apache.zeppelin.util.Util; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class K8sRemoteInterpreterProcess extends RemoteInterpreterProcess { private static final Logger LOGGER = LoggerFactory.getLogger(K8sStandardInterpreterLauncher.class); private static final int K8S_INTERPRETER_SERVICE_PORT = 12321; + private static final int K8S_INTERPRETER_RESTAPISERVER_PORT = 8090; private final Kubectl kubectl; private final String interpreterGroupId; private final String interpreterGroupName; @@ -67,7 +69,7 @@ public K8sRemoteInterpreterProcess( this.zeppelinServiceRpcPort = zeppelinServiceRpcPort; this.portForward = portForward; this.sparkImage = sparkImage; - this.podName = interpreterGroupName.toLowerCase() + "-" + getRandomString(6); + this.podName = interpreterGroupName.toLowerCase() + "-" + Util.getRandomString(6); } @@ -88,7 +90,7 @@ public String getInterpreterSettingName() { @Override public void start(String userName) throws IOException { // create new pod - apply(specTempaltes, false); + kubectl.apply(specTempaltes, getTemplateBindings(), false); kubectl.wait(String.format("pod/%s", getPodName()), "condition=Ready", getConnectTimeout()/1000); if (portForward) { @@ -136,7 +138,7 @@ public void start(String userName) throws IOException { public void stop() { // delete pod try { - apply(specTempaltes, true); + kubectl.apply(specTempaltes, getTemplateBindings(), true); } catch (IOException e) { LOGGER.info("Error on removing interpreter pod", e); } @@ -202,41 +204,6 @@ public boolean isRunning() { } } - /** - * Apply spec file(s) in the path. - * @param path - */ - void apply(File path, boolean delete) throws IOException { - if (path.getName().startsWith(".") || path.isHidden() || path.getName().endsWith("~")) { - LOGGER.info("Skip " + path.getAbsolutePath()); - } - - if (path.isDirectory()) { - File[] files = path.listFiles(); - Arrays.sort(files); - if (delete) { - ArrayUtils.reverse(files); - } - - for (File f : files) { - apply(f, delete); - } - } else if (path.isFile()) { - LOGGER.info("Apply " + path.getAbsolutePath()); - K8sSpecTemplate specTemplate = new K8sSpecTemplate(); - specTemplate.loadProperties(getTemplateBindings()); - - String spec = specTemplate.render(path); - if (delete) { - kubectl.delete(spec); - } else { - kubectl.apply(spec); - } - } else { - LOGGER.error("Can't apply " + path.getAbsolutePath()); - } - } - @VisibleForTesting Properties getTemplateBindings() throws IOException { Properties k8sProperties = new Properties(); @@ -260,7 +227,18 @@ Properties getTemplateBindings() throws IOException { // environment variables envs.put("SERVICE_DOMAIN", envs.getOrDefault("SERVICE_DOMAIN", System.getenv("SERVICE_DOMAIN"))); - envs.put("ZEPPELIN_HOME", envs.getOrDefault("ZEPPELIN_HOME", "/zeppelin")); + envs.put("SERVICE_NAME", envs.getOrDefault("SERVICE_NAME", System.getenv("SERVICE_NAME"))); + envs.put(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.name(), + envs.getOrDefault("ZEPPELIN_HOME", "/zeppelin")); + envs.put(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_RESTAPI_PORT.name(), + String.valueOf(K8S_INTERPRETER_RESTAPISERVER_PORT)); + + // pass all env variables starts with ZEPPELIN_INTERPRETER + for (String envName : System.getenv().keySet()) { + if (envName.startsWith("ZEPPELIN_INTERPRETER")) { + envs.put(envName, System.getenv(envName)); + } + } if (isSpark()) { int webUiPort = 4040; @@ -357,20 +335,6 @@ private String ownerName() { return System.getenv("POD_NAME"); } - private String getRandomString(int length) { - char[] chars = "abcdefghijklmnopqrstuvwxyz".toCharArray(); - - StringBuilder sb = new StringBuilder(); - Random random = new Random(); - for (int i = 0; i < length; i++) { - char c = chars[random.nextInt(chars.length)]; - sb.append(c); - } - String randomStr = sb.toString(); - - return randomStr; - } - @Override public void processStarted(int port, String host) { LOGGER.info("Interpreter pod created {}:{}", host, port); diff --git a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sStandardInterpreterLauncher.java b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sStandardInterpreterLauncher.java index 4f2ed917d8d..d298576b0d3 100644 --- a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sStandardInterpreterLauncher.java +++ b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sStandardInterpreterLauncher.java @@ -25,6 +25,7 @@ import java.nio.file.Paths; import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang.StringUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.recovery.RecoveryStorage; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; @@ -48,7 +49,7 @@ public class K8sStandardInterpreterLauncher extends InterpreterLauncher { public K8sStandardInterpreterLauncher(ZeppelinConfiguration zConf, RecoveryStorage recoveryStorage) throws IOException { super(zConf, recoveryStorage); kubectl = new Kubectl(zConf.getK8sKubectlCmd()); - kubectl.setNamespace(getNamespace()); + kubectl.setNamespace(Kubectl.getNamespaceFromContainer()); } @VisibleForTesting @@ -77,30 +78,6 @@ boolean isRunningOnKubernetes() { } } - /** - * Get current namespace - * @throws IOException - */ - String getNamespace() throws IOException { - if (isRunningOnKubernetes()) { - return readFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace", Charset.defaultCharset()).trim(); - } else { - return "default"; - } - } - - /** - * Get hostname. It should be the same to Service name (and Pod name) of the Kubernetes - * @return - */ - String getHostname() { - try { - return InetAddress.getLocalHost().getHostName(); - } catch (UnknownHostException e) { - return "localhost"; - } - } - /** * get Zeppelin server host dns. * return ..svc.cluster.local @@ -108,9 +85,15 @@ String getHostname() { */ private String getZeppelinServiceHost() throws IOException { if (isRunningOnKubernetes()) { + String serviceName = System.getenv("SERVICE_NAME"); + if (StringUtils.isEmpty(serviceName)) { + // if SERVICE_NAME env is not defined, try pod host name as a service name. + serviceName = Kubectl.getHostname(); + } + return String.format("%s.%s.svc.cluster.local", - getHostname(), // service name and pod name should be the same - getNamespace()); + serviceName, + Kubectl.getNamespaceFromContainer()); } else { return context.getZeppelinServerHost(); } @@ -122,7 +105,7 @@ private String getZeppelinServiceHost() throws IOException { */ private String getZeppelinServiceRpcPort() { String envServicePort = System.getenv( - String.format("%s_SERVICE_PORT_RPC", getHostname().replaceAll("[-.]", "_").toUpperCase())); + String.format("%s_SERVICE_PORT_RPC", Kubectl.getHostname().replaceAll("[-.]", "_").toUpperCase())); if (envServicePort != null) { return envServicePort; } else { @@ -169,9 +152,4 @@ protected Map buildEnvFromProperties(InterpreterLaunchContext co env.put("INTERPRETER_GROUP_ID", context.getInterpreterGroupId()); return env; } - - String readFile(String path, Charset encoding) throws IOException { - byte[] encoded = Files.readAllBytes(Paths.get(path)); - return new String(encoded, encoding); - } } diff --git a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/Kubectl.java b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/Kubectl.java index 2079d16db42..b744b25ac1a 100644 --- a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/Kubectl.java +++ b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/Kubectl.java @@ -19,14 +19,28 @@ import com.google.common.annotations.VisibleForTesting; import com.google.gson.Gson; +import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.fabric8.kubernetes.api.model.batch.Job; +import io.fabric8.kubernetes.client.Config; +import io.fabric8.kubernetes.client.ConfigBuilder; +import io.fabric8.kubernetes.client.DefaultKubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.Watcher; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.Properties; import org.apache.commons.exec.*; import org.apache.commons.io.IOUtils; import java.io.*; +import org.apache.commons.lang3.ArrayUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,10 +48,14 @@ public class Kubectl { private final Logger LOGGER = LoggerFactory.getLogger(Kubectl.class); private final String kubectlCmd; private final Gson gson = new Gson(); + private final KubernetesClient client; private String namespace; public Kubectl(String kubectlCmd) { this.kubectlCmd = kubectlCmd; + + Config config = new ConfigBuilder().build(); + client = new DefaultKubernetesClient(config); } /** @@ -60,6 +78,19 @@ public String delete(String spec) throws IOException { return execAndGet(new String[]{"delete", "-f", "-"}, spec); } + public String get(String resourceType, String resourceName) throws IOException { + return execAndGet(new String[]{"get", resourceType, resourceName, "-o", "json"}); + } + + public String getByLabel(String resourceType, String labelExpr) throws IOException { + return execAndGet(new String[]{"get", resourceType, "-l", labelExpr, "-o", "json"}); + } + + public void label(String resourceType, String resourceName, String key, String value) throws IOException { + execAndGet(new String[]{"label", "--overwrite", resourceType, resourceName, + String.format("%s=%s", key, value)}, ""); + } + public String wait(String resource, String waitFor, int timeoutSec) throws IOException { try { return execAndGet(new String[]{ @@ -154,4 +185,101 @@ public int execute(String [] args, InputStream stdin, OutputStream stdout, Outpu executor.setStreamHandler(streamHandler); return executor.execute(cmd); } + + + /** + * Check if i'm running inside of kubernetes or not. + * It should return truth regardless of ZeppelinConfiguration.getRunMode(). + * + * Normally, unless Zeppelin is running on Kubernetes, K8sStandardInterpreterLauncher shouldn't even have initialized. + * However, when ZeppelinConfiguration.getRunMode() is force 'k8s', InterpreterSetting.getLauncherPlugin() will try + * to use K8sStandardInterpreterLauncher. This is useful for development. It allows Zeppelin server running on your + * IDE and creates your interpreters in Kubernetes. So any code changes on Zeppelin server or kubernetes yaml spec + * can be applied without re-building docker image. + * @return + */ + public static boolean isRunningOnKubernetes() { + if (new File("/var/run/secrets/kubernetes.io").exists()) { + return true; + } else { + return false; + } + } + + /** + * Get current namespace + * @throws IOException + */ + public static String getNamespaceFromContainer() throws IOException { + if (isRunningOnKubernetes()) { + return readFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace", Charset.defaultCharset()).trim(); + } else { + return "default"; + } + } + + /** + * Get hostname. It should be the same to Service name (and Pod name) of the Kubernetes + * @return + */ + public static String getHostname() { + try { + return InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + return "localhost"; + } + } + + + static String readFile(String path, Charset encoding) throws IOException { + byte[] encoded = Files.readAllBytes(Paths.get(path)); + return new String(encoded, encoding); + } + + /** + * Apply spec file(s) in the path. + * @param path + */ + public void apply(File path, Properties binding, boolean delete) throws IOException { + if (path.getName().startsWith(".") || path.isHidden() || path.getName().endsWith("~")) { + LOGGER.info("Skip " + path.getAbsolutePath()); + } + + if (path.isDirectory()) { + File[] files = path.listFiles(); + Arrays.sort(files); + if (delete) { + ArrayUtils.reverse(files); + } + + for (File f : files) { + apply(f, binding, delete); + } + } else if (path.isFile()) { + LOGGER.info("Apply " + path.getAbsolutePath()); + K8sSpecTemplate specTemplate = new K8sSpecTemplate(); + specTemplate.loadProperties(binding); + + String spec = specTemplate.render(path); + if (delete) { + delete(spec); + } else { + apply(spec); + } + } else { + LOGGER.error("Can't apply " + path.getAbsolutePath()); + } + } + + public void watchDeployments(Watcher watcher, String labelKey, String labelValue) { + client.apps().deployments().inNamespace(getNamespace()) + .withLabel(labelKey, labelValue) + .watch(watcher); + } + + public void watchJobs(Watcher watcher, String labelKey, String labelValue) { + client.batch().jobs().inNamespace(getNamespace()) + .withLabel(labelKey, labelValue) + .watch(watcher); + } } diff --git a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/serving/K8sNoteServingTask.java b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/serving/K8sNoteServingTask.java new file mode 100644 index 00000000000..e616d281b7a --- /dev/null +++ b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/serving/K8sNoteServingTask.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.serving; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import org.apache.commons.lang.StringUtils; +import org.apache.zeppelin.background.K8sNoteBackgroundTask; +import org.apache.zeppelin.background.NoteBackgroundTask; +import org.apache.zeppelin.background.TaskContext; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.interpreter.launcher.Kubectl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Start / Stop / Monitor serving task. + */ +public class K8sNoteServingTask extends K8sNoteBackgroundTask { + private static final Logger LOGGER = LoggerFactory.getLogger(K8sNoteServingTask.class); + private final String notebookDir; + private final Gson gson = new Gson(); + + public K8sNoteServingTask(Kubectl kubectl, + TaskContext taskContext, + String notebookDir, + File k8sTemplateDir) { + super(kubectl, taskContext, k8sTemplateDir); + this.notebookDir = notebookDir; + } + + @Override + protected Properties getTemplateBindings() throws IOException { + Properties properties = super.getTemplateBindings(); + properties.put("zeppelin.k8s.background.notebook.dir", notebookDir); + properties.put("zeppelin.k8s.background.autoshutdown", "false"); + properties.put("zeppelin.k8s.background.type", "serving"); + properties.put("zeppelin.k8s.serving.metric.redis.addr", + System.getenv(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_METRIC_REDIS_ADDR.name())); + return properties; + } + + @Override + protected String getResourceName() { + return String.format("serving-%s", getTaskContext().getId()); + } + + @Override + protected String getResourceApiVersion() { + return "apps/v1"; + } + + @Override + protected String getResourceType() { + return "Deployment"; + } + + public Map getInfo() throws IOException { + HashMap combinedInfo = new HashMap(); + Map info = super.getInfo(); + + if (info != null) { + combinedInfo.putAll(info); + + Kubectl kubectl = getKubectl(); + TaskContext context = getTaskContext(); + String labelFilter = String.format("serving=true,noteId=%s,revId=%s", + context.getNote().getId(), + context.getRevId()); + String resourceJsonString = kubectl.getByLabel("service", labelFilter); + + HashMap endPointServiceNameMap = new HashMap(); + + if (!StringUtils.isEmpty(resourceJsonString)) { + Map services = gson.fromJson(resourceJsonString, + new TypeToken>() { + }.getType()); + + if (services.containsKey("items")) { + List> items = (List>) services.get("items"); + for (Map item : items) { + Map metadata = (Map) item.get("metadata"); + Map labels = (Map) metadata.get("labels"); + for (String labelkey : labels.keySet()) { + if (labelkey.startsWith("endpoint-")) { + endPointServiceNameMap.put((String) labels.get(labelkey), (String) metadata.get("name")); + } + } + } + } + } + + combinedInfo.put("endpoints", endPointServiceNameMap); + } + return combinedInfo; + } + + @Override + public boolean isRunning() { + Map resource = null; + try { + resource = getInfo(); + } catch (IOException e) { + // does not exists + return false; + } + if (resource == null) { + return false; + } + + if (resource.size() == 0) { + return false; + } + + Map status = (Map) resource.get("status"); + if (status == null) { + return false; + } + + if (!status.containsKey("availableReplicas") || status.get("availableReplicas") == null) { + return false; + } + + return (int) status.get("availableReplicas") > 0; + } +} diff --git a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/serving/K8sNoteServingTaskManager.java b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/serving/K8sNoteServingTaskManager.java new file mode 100644 index 00000000000..230a3821a64 --- /dev/null +++ b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/serving/K8sNoteServingTaskManager.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.serving; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import io.fabric8.kubernetes.api.model.apps.Deployment; +import java.io.File; +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.zeppelin.background.BackgroundTaskLifecycleListener; +import org.apache.zeppelin.background.FileSystemTaskContextStorage; +import org.apache.zeppelin.background.K8sNoteBackgroundTaskManager; +import org.apache.zeppelin.background.NoteBackgroundTask; +import org.apache.zeppelin.background.TaskContext; +import org.apache.zeppelin.background.TaskContextStorage; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.interpreter.launcher.BackgroundTaskLifecycleWatcherImpl; +import org.apache.zeppelin.interpreter.launcher.Kubectl; +import org.apache.zeppelin.notebook.Note; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Provide TaskContextStorage and creates NoteServingTask. + */ +public class K8sNoteServingTaskManager extends K8sNoteBackgroundTaskManager { + private static final Logger LOGGER = LoggerFactory.getLogger(K8sNoteServingTaskManager.class); + private final BackgroundTaskLifecycleWatcherImpl watcher; + Gson gson = new Gson(); + + public K8sNoteServingTaskManager(ZeppelinConfiguration zConf) throws IOException { + super(zConf); + + watcher = new BackgroundTaskLifecycleWatcherImpl(getListener()) { + @Override + protected String getTaskId(Deployment deployment) { + return deployment.getMetadata().getName().replaceFirst("serving-", ""); + } + }; + + getKubectl().watchDeployments(watcher, "taskType", "serving"); + } + + @Override + protected TaskContextStorage createTaskContextStorage() { + return new FileSystemTaskContextStorage(getConf().getK8sServingContextDir()); + } + + @Override + protected NoteBackgroundTask createOrGetBackgroundTask(TaskContext taskContext) { + File servingTemplateDir = new File(getConf().getK8sTemplatesDir(), "background"); + K8sNoteServingTask servingTask = new K8sNoteServingTask( + getKubectl(), + taskContext, + String.format("%s/%s/notebook", + new File(getConf().getK8sServingContextDir()).getAbsolutePath(), + taskContext.getId()), + servingTemplateDir); + return servingTask; + } + + @Override + public List list() { + Kubectl kubectl = getKubectl(); + Map deployments = null; + List tasks = new LinkedList<>(); + + try { + String deploymentJsonString = kubectl.getByLabel("deployment", "taskType=serving"); + deployments = gson.fromJson(deploymentJsonString, new TypeToken() {}.getType()); + } catch (IOException e) { + LOGGER.error("Error", e); + return tasks; + } + + List items = (List) deployments.get("items"); + if (items == null) { + return tasks; + } + + for (Map item : items) { + Map metadata = (Map) item.get("metadata"); + + // name format is "serving--" + String name = (String) metadata.get("name"); + String[] tokens = name.split("-"); + if (tokens == null || tokens.length != 3) { + LOGGER.warn("Unrecognized serving name {}", name); + continue; + } + + String noteId = tokens[1].toUpperCase(); + String revId = tokens[2]; + + TaskContext context = null; + try { + context = getTaskContextStorage().load(TaskContext.getTaskId(noteId, revId)); + } catch (IOException e) { + LOGGER.warn("Serving {} does not exists in task storage", name); + context = null; + } + + if (context == null) { + // serving exists, but no info in task context. + // create empty note. so at least we can stop it. + Note note = new Note(); + note.setName("empty"); + note.setId(noteId); + context = new TaskContext(note, revId); + } + + tasks.add(createOrGetBackgroundTask(context)); + } + + return tasks; + } + + public void setListener(BackgroundTaskLifecycleListener listener) { + super.setListener(listener); + watcher.setListener(listener); + } +} diff --git a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/serving/K8sRestApiRouter.java b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/serving/K8sRestApiRouter.java new file mode 100644 index 00000000000..85e461bce69 --- /dev/null +++ b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/serving/K8sRestApiRouter.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.serving; + +import java.io.IOException; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.interpreter.launcher.Kubectl; +import org.apache.zeppelin.util.Util; + +/** + * Configure note serving RestAPI router in k8s environment. + * RestAPI router will construct routing table based on informations in labels of Service of Interpreter Pod, + * after this class's addRoute() update labels of Service. + */ +public class K8sRestApiRouter implements RestApiRouter { + private final Kubectl kubectl; + + public K8sRestApiRouter(ZeppelinConfiguration zConf) { + kubectl = new Kubectl(zConf.getK8sKubectlCmd()); + } + + /** + * Add labels to kubernetes Service of Interpreter Pod. + * So API router can scan lables of all the Services and construct routing table. + */ + @Override + public void addRoute(String noteId, String revId, String dnsName, String hostname, int port, String endpoint) throws IOException { + String randomEndpointId = Util.getRandomString(5); + + String serviceName = hostname; + + kubectl.label("service", serviceName, "serving", "true"); + kubectl.label("service", serviceName, "noteId", noteId); + kubectl.label("service", serviceName, "revId", revId); + kubectl.label("service", serviceName, String.format("endpoint-%s", randomEndpointId), endpoint); + } + + @Override + public void removeRoute(String noteId, String revId) throws IOException { + } +} diff --git a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/test/K8sNoteTestTask.java b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/test/K8sNoteTestTask.java new file mode 100644 index 00000000000..d1cda0333e8 --- /dev/null +++ b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/test/K8sNoteTestTask.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.test; + +import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import org.apache.zeppelin.background.K8sNoteBackgroundTask; +import org.apache.zeppelin.background.TaskContext; +import org.apache.zeppelin.background.TaskContextStorage; +import org.apache.zeppelin.interpreter.launcher.Kubectl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Note test task. + */ +public class K8sNoteTestTask extends K8sNoteBackgroundTask { + private static final Logger LOGGER = LoggerFactory.getLogger(K8sNoteTestTask.class); + private final String notebookDir; + + public K8sNoteTestTask(Kubectl kubectl, + TaskContext taskContext, + String notebookDir, + File k8sTemplateDir) { + super(kubectl, taskContext, k8sTemplateDir); + this.notebookDir = notebookDir; + } + + @Override + protected Properties getTemplateBindings() throws IOException { + Properties properties = super.getTemplateBindings(); + properties.put("zeppelin.k8s.background.notebook.dir", notebookDir); + properties.put("zeppelin.k8s.background.autoshutdown", "true"); + properties.put("zeppelin.k8s.background.type", "test"); + return properties; + } + + @Override + protected String getResourceName() { + return String.format("test-%s", getTaskContext().getId()); + } + + @Override + protected String getResourceApiVersion() { + return "batch/v1"; + } + + @Override + protected String getResourceType() { + return "Job"; + } + + @Override + public Map getInfo() throws IOException { + Map info = super.getInfo(); + if (info != null) { + Map status = (Map) info.get("status"); + if (status != null) { + if (status.containsKey("completionTime")) { + // job completed. remove job from kubernetes. + stop(); + } + } + } + return info; + } + + /** + * Return false when job does not exists or job is completed. + * Otherwise return true (on job exists or running) + * @return + */ + @Override + public boolean isRunning() { + Map resource = null; + try { + resource = getInfo(); + } catch (IOException e) { + LOGGER.error("Can't get task info", e); + return false; + } + if (resource == null) { + return false; + } + + if (resource.size() == 0) { + return true; + } + + Map status = (Map) resource.get("status"); + if (status == null) { + return true; + } + + if (status.containsKey("completionTime")) { + return false; + } + + return true; + } +} diff --git a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/test/K8sNoteTestTaskManager.java b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/test/K8sNoteTestTaskManager.java new file mode 100644 index 00000000000..6af51aa0865 --- /dev/null +++ b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/test/K8sNoteTestTaskManager.java @@ -0,0 +1,54 @@ +package org.apache.zeppelin.test; + +import io.fabric8.kubernetes.api.model.batch.Job; +import java.io.File; +import java.io.IOException; +import org.apache.zeppelin.background.BackgroundTaskLifecycleListener; +import org.apache.zeppelin.background.FileSystemTaskContextStorage; +import org.apache.zeppelin.background.K8sNoteBackgroundTaskManager; +import org.apache.zeppelin.background.NoteBackgroundTask; +import org.apache.zeppelin.background.TaskContext; +import org.apache.zeppelin.background.TaskContextStorage; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.interpreter.launcher.BackgroundTaskLifecycleWatcherImpl; + +public class K8sNoteTestTaskManager extends K8sNoteBackgroundTaskManager { + private final BackgroundTaskLifecycleWatcherImpl watcher; + + public K8sNoteTestTaskManager(ZeppelinConfiguration zConf) throws IOException { + super(zConf); + + watcher = new BackgroundTaskLifecycleWatcherImpl(getListener()) { + @Override + protected String getTaskId(Job job) { + return job.getMetadata().getName().replaceFirst("test-", ""); + } + }; + + getKubectl().watchJobs(watcher, "taskType", "test"); + } + + @Override + protected TaskContextStorage createTaskContextStorage() { + return new FileSystemTaskContextStorage(getConf().getK8sTestContextDir()); + } + + @Override + protected NoteBackgroundTask createOrGetBackgroundTask(TaskContext taskContext) { + File servingTemplateDir = new File(getConf().getK8sTemplatesDir(), "background"); + K8sNoteTestTask testTask = new K8sNoteTestTask( + getKubectl(), + taskContext, + String.format("%s/%s/notebook", + new File(getConf().getK8sTestContextDir()).getAbsolutePath(), + taskContext.getId()), + servingTemplateDir); + return testTask; + } + + + public void setListener(BackgroundTaskLifecycleListener listener) { + super.setListener(listener); + watcher.setListener(listener); + } +} diff --git a/zeppelin-plugins/launcher/k8s-standard/src/test/java/org/apache/zeppelin/serving/K8sNoteServingTaskTest.java b/zeppelin-plugins/launcher/k8s-standard/src/test/java/org/apache/zeppelin/serving/K8sNoteServingTaskTest.java new file mode 100644 index 00000000000..c3f1c2042d7 --- /dev/null +++ b/zeppelin-plugins/launcher/k8s-standard/src/test/java/org/apache/zeppelin/serving/K8sNoteServingTaskTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.serving; + +import com.google.common.io.Files; +import java.io.File; +import java.io.IOException; +import org.apache.commons.io.FileUtils; +import org.apache.zeppelin.background.FileSystemTaskContextStorage; +import org.apache.zeppelin.background.TaskContext; +import org.apache.zeppelin.interpreter.launcher.Kubectl; +import org.apache.zeppelin.notebook.Note; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.mockito.Mockito.mock; + +/** + * K8sNoteServingTaskTest + */ +public class K8sNoteServingTaskTest { + private File contextRoot; + private FileSystemTaskContextStorage storage; + + @Before + public void setUp() { + contextRoot = Files.createTempDir(); + } + + @After + public void tearDown() throws IOException { + FileUtils.deleteDirectory(contextRoot); + } + + @Test + public void testStart() throws IOException { + Note note = new Note(); + note.setId("2E63B9RE6"); + TaskContext task = new TaskContext(note, "rev1"); + + storage = new FileSystemTaskContextStorage(contextRoot.getAbsolutePath()); + storage.save(task); + + Kubectl kubectl = mock(Kubectl.class); + kubectl.setNamespace("default"); + File servingTemplate = new File("../../../k8s/serving"); + System.out.println(servingTemplate.getAbsolutePath()); + K8sNoteServingTask noteServingTask = new K8sNoteServingTask( + kubectl, + task, + String.format("%s/%s/notebook", contextRoot.getAbsolutePath(), task.getId()), + servingTemplate); + noteServingTask.start(); + } + + @Test + public void testStop() throws IOException { + Note note = new Note(); + note.setId("2E63B9RE6"); + TaskContext task = new TaskContext(note, "rev1"); + + storage = new FileSystemTaskContextStorage(contextRoot.getAbsolutePath()); + + Kubectl kubectl = new Kubectl("kubectl"); + kubectl.setNamespace("default"); + File servingTemplate = new File("../../../k8s/serving"); + K8sNoteServingTask noteServingTask = new K8sNoteServingTask( + kubectl, + task, + String.format("%s/%s/notebook", contextRoot.getAbsolutePath(), task.getId()), + servingTemplate); + noteServingTask.stop(); + } + + @Test + public void testIsRunning() throws IOException { + Note note = new Note(); + note.setId("2E63B9RE6"); + TaskContext task = new TaskContext(note, "rev1"); + + storage = new FileSystemTaskContextStorage(contextRoot.getAbsolutePath()); + + Kubectl kubectl = new Kubectl("kubectl"); + kubectl.setNamespace("default"); + File servingTemplate = new File("../../../k8s/serving"); + K8sNoteServingTask noteServingTask = new K8sNoteServingTask( + kubectl, + task, + String.format("%s/%s/notebook", contextRoot.getAbsolutePath(), task.getId()), + servingTemplate); + + boolean isRunning = noteServingTask.isRunning(); + } +} diff --git a/zeppelin-plugins/launcher/standard/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java b/zeppelin-plugins/launcher/standard/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java index 47756c9901f..3a02f2a3dc7 100644 --- a/zeppelin-plugins/launcher/standard/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java +++ b/zeppelin-plugins/launcher/standard/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java @@ -82,6 +82,7 @@ public InterpreterClient launch(InterpreterLaunchContext context) throws IOExcep return new RemoteInterpreterManagedProcess( runner != null ? runner.getPath() : zConf.getInterpreterRemoteRunnerPath(), context.getZeppelinServerRPCPort(), context.getZeppelinServerHost(), zConf.getInterpreterPortRange(), + zConf.getInterpreterRestApiServerPort(), zConf.getInterpreterDir() + "/" + groupName, localRepoPath, buildEnvFromProperties(context), connectTimeout, name, context.getInterpreterGroupId(), option.isUserImpersonate()); diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index 51636806e92..ce087473456 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -234,6 +234,18 @@ ${jetty.version} + + org.eclipse.jetty.websocket + javax-websocket-server-impl + ${jetty.version} + + + org.ow2.asm + asm + + + + org.eclipse.jetty.websocket websocket-server @@ -275,6 +287,53 @@ ${scala.version} + + + com.jayway.jsonpath + json-path + 2.3.0 + + + + org.redisson + redisson + 3.7.0 + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-core + + + io.netty + netty-common + + + io.netty + netty-codec + + + io.netty + netty-transport + + + io.netty + netty-buffer + + + io.netty + netty-handler + + + io.netty + netty-resolver + + + + junit @@ -288,6 +347,13 @@ test + + com.orange.redis-embedded + embedded-redis + 0.6 + test + + org.scalatest scalatest_${scala.binary.version} diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index 5aa776aa77f..e829b9646ed 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -17,10 +17,13 @@ package org.apache.zeppelin.rest; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import com.google.common.reflect.TypeToken; import com.google.gson.Gson; import java.io.IOException; +import java.io.Serializable; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -39,8 +42,11 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; + +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.zeppelin.annotation.ZeppelinApi; +import org.apache.zeppelin.background.NoteBackgroundTask; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.notebook.Note; @@ -59,12 +65,16 @@ import org.apache.zeppelin.rest.message.RenameNoteRequest; import org.apache.zeppelin.rest.message.RunParagraphWithParametersRequest; import org.apache.zeppelin.rest.message.UpdateParagraphRequest; +import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.search.SearchService; import org.apache.zeppelin.server.JsonResponse; import org.apache.zeppelin.service.AuthenticationService; import org.apache.zeppelin.service.JobManagerService; +import org.apache.zeppelin.service.NoteServingTaskManagerService; +import org.apache.zeppelin.service.NoteTestTaskManagerService; import org.apache.zeppelin.service.NotebookService; import org.apache.zeppelin.service.ServiceContext; +import org.apache.zeppelin.serving.MetricStorage; import org.apache.zeppelin.socket.NotebookServer; import org.apache.zeppelin.user.AuthenticationInfo; import org.quartz.CronExpression; @@ -90,18 +100,22 @@ public class NotebookRestApi extends AbstractRestApi { private JobManagerService jobManagerService; private AuthenticationService authenticationService; private SchedulerService schedulerService; + private NoteServingTaskManagerService noteServingTaskManagerService; + private NoteTestTaskManagerService noteTestTaskManagerService; @Inject public NotebookRestApi( - Notebook notebook, - NotebookServer notebookServer, - NotebookService notebookService, - SearchService search, - AuthorizationService authorizationService, - ZeppelinConfiguration zConf, - AuthenticationService authenticationService, - JobManagerService jobManagerService, - SchedulerService schedulerService) { + Notebook notebook, + NotebookServer notebookServer, + NotebookService notebookService, + SearchService search, + AuthorizationService authorizationService, + ZeppelinConfiguration zConf, + AuthenticationService authenticationService, + JobManagerService jobManagerService, + SchedulerService schedulerService, + NoteServingTaskManagerService noteServingTaskManagerService, + NoteTestTaskManagerService noteTestTaskManagerService) { super(authenticationService); this.notebook = notebook; this.notebookServer = notebookServer; @@ -112,6 +126,8 @@ public NotebookRestApi( this.zConf = zConf; this.authenticationService = authenticationService; this.schedulerService = schedulerService; + this.noteServingTaskManagerService = noteServingTaskManagerService; + this.noteTestTaskManagerService = noteTestTaskManagerService; } /** @@ -1018,6 +1034,178 @@ public Response search(@QueryParam("q") String queryTerm) { return new JsonResponse<>(Status.OK, notesFound).build(); } + @POST + @Path("serving/{noteId}/{revId}") + @ZeppelinApi + public Response servingStart(@PathParam("noteId") String noteId, @PathParam("revId") String revId) throws Exception { + NoteBackgroundTask task = noteServingTaskManagerService.startServing(noteId, revId, getServiceContext()); + + return new JsonResponse<>(Status.OK, ImmutableMap.of( + "taskId", task.getTaskContext().getId() + )).build(); + } + + @DELETE + @Path("serving/{noteId}/{revId}") + @ZeppelinApi + public Response servingStop(@PathParam("noteId") String noteId, @PathParam("revId") String revId) throws Exception { + NoteBackgroundTask task = noteServingTaskManagerService.stopServing(noteId, revId, getServiceContext()); + return new JsonResponse<>(Status.OK).build(); + } + + @GET + @Path("serving/{noteId}/{revId}") + @ZeppelinApi + public Response servingInfo(@PathParam("noteId") String noteId, @PathParam("revId") String revId) throws IOException { + NoteBackgroundTask task = noteServingTaskManagerService.getServing(noteId, revId, getServiceContext()); + if (task != null) { + Map taskInfo = task.getInfo(); + if (taskInfo == null) { + taskInfo = ImmutableMap.of(); + } + return new JsonResponse<>(Status.OK, ImmutableMap.of( + "taskId", task.getTaskContext().getId(), + "task", taskInfo + )).build(); + } else { + return new JsonResponse<>(Status.NOT_FOUND).build(); + } + } + + @GET + @Path("serving/{noteId}/{revId}/{endpoint}") + @ZeppelinApi + public Response servingMetrics(@PathParam("noteId") String noteId, + @PathParam("revId") String revId, + @PathParam("endpoint") String endpoint, + @QueryParam("from") int fromTimestamp, + @QueryParam("to") int toTimestamp) + throws IOException { + NoteBackgroundTask task = noteServingTaskManagerService.getServing(noteId, revId, getServiceContext()); + MetricStorage metricStorage = noteServingTaskManagerService.getMetricStorage(); + + Date toDate = (toTimestamp > 0) ? new Date(toTimestamp * 1000) : new Date(System.currentTimeMillis() + 1000 * 60); + Date fromDate = (fromTimestamp > 0) ? new Date(fromTimestamp * 1000) : new Date(toDate.getTime() - 1000 * 60 * 30); + List> series = metricStorage.get(fromDate, toDate, noteId, revId, endpoint); + + return new JsonResponse<>(Status.OK, series).build(); + } + + @GET + @Path("serving") + @ZeppelinApi + public Response servingList() throws IOException { + List tasks = noteServingTaskManagerService.getAllServing(); + List> taskInfoList = tasks.stream().map(task -> { + try { + Note note = task.getTaskContext().getNote(); + + return ImmutableMap.of( + "taskId", task.getTaskContext().getId(), + "info", task.getInfo(), + "note", ImmutableMap.of( + "name", note.getName(), + "id", note.getId(), + "revision", task.getTaskContext().getRevId() + ) + ); + } catch (IOException e) { + LOG.error("Not able to get task info", e); + } + return ImmutableMap.of( + "taskId", task.getTaskContext().getId() + ); + }).collect(Collectors.toList()); + return new JsonResponse<>(Status.OK, taskInfoList).build(); + } + + + @POST + @Path("test/{noteId}/{revId}") + @ZeppelinApi + public Response testStart(@PathParam("noteId") String noteId, @PathParam("revId") String revId) throws Exception { + NoteBackgroundTask task = noteTestTaskManagerService.startTest(noteId, revId, getServiceContext()); + + return new JsonResponse<>(Status.OK, ImmutableMap.of( + "taskId", task.getTaskContext().getId() + )).build(); + } + + @DELETE + @Path("test/{noteId}/{revId}") + @ZeppelinApi + public Response testStop(@PathParam("noteId") String noteId, @PathParam("revId") String revId) throws Exception { + NoteBackgroundTask task = noteTestTaskManagerService.stopTest(noteId, revId, getServiceContext()); + return new JsonResponse<>(Status.OK).build(); + } + + @GET + @Path("test/{noteId}/{revId}") + @ZeppelinApi + public Response testInfo(@PathParam("noteId") String noteId, @PathParam("revId") String revId) throws IOException { + NoteBackgroundTask task = noteTestTaskManagerService.getTest(noteId, revId, getServiceContext()); + + if (task != null) { + Map taskInfo = null; + taskInfo = task.getInfo(); + boolean taskRunning = taskInfo != null; + + if (taskInfo == null) { + taskInfo = ImmutableMap.of(); + } + + Map result = ImmutableMap.of(); + if (!taskRunning) { + Note note = task.getTaskContext().getNote(); + + int pCount = 0; + int pSuccess = 0; + int pFail = 0; + + for (Paragraph p : note.getParagraphs()) { + pCount++; + if (p.getStatus() == Paragraph.Status.FINISHED || + p.getStatus() == Paragraph.Status.READY) { + pSuccess++; + } else { + pFail++; + } + } + + result = ImmutableMap.of( + "count", pCount, + "success", pSuccess, + "fail", pFail + ); + } + + + Response response = new JsonResponse<>(Status.OK, ImmutableMap.of( + "taskId", task.getTaskContext().getId(), + "task", taskInfo, + "running", taskRunning, + "result", result + )).build(); + return response; + } else { + return new JsonResponse<>(Status.NOT_FOUND).build(); + } + } + + @GET + @Path("test/{noteId}/{revId}/note") + @ZeppelinApi + public Response getTestNote(@PathParam("noteId") String noteId, @PathParam("revId") String revId) throws IOException { + NoteBackgroundTask task = noteTestTaskManagerService.getTest(noteId, revId, getServiceContext()); + if (task != null) { + Response response = new JsonResponse<>(Status.OK, ImmutableMap.of( + "note", task.getTaskContext().getNote() + )).build(); + return response; + } else { + return new JsonResponse<>(Status.NOT_FOUND).build(); + } + } private void handleParagraphParams(String message, Note note, Paragraph paragraph) throws IOException { diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index fe85f75d214..908af6de400 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -16,11 +16,15 @@ */ package org.apache.zeppelin.server; +import com.google.gson.Gson; import java.io.File; import java.io.IOException; import java.lang.management.ManagementFactory; +import java.util.Base64; import java.util.EnumSet; +import java.util.HashSet; import java.util.Objects; +import java.util.Set; import java.util.stream.Stream; import javax.inject.Inject; import javax.inject.Singleton; @@ -29,6 +33,7 @@ import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; import org.apache.commons.lang.StringUtils; +import org.apache.directory.api.util.Strings; import org.apache.shiro.web.env.EnvironmentLoaderListener; import org.apache.shiro.web.servlet.ShiroFilter; import org.apache.zeppelin.conf.ZeppelinConfiguration; @@ -42,9 +47,11 @@ import org.apache.zeppelin.interpreter.InterpreterOutput; import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; +import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.NoteEventListener; import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.notebook.AuthorizationService; +import org.apache.zeppelin.notebook.Paragraph; import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.apache.zeppelin.notebook.repo.NotebookRepoSync; import org.apache.zeppelin.notebook.scheduler.NoSchedulerService; @@ -56,6 +63,7 @@ import org.apache.zeppelin.service.*; import org.apache.zeppelin.service.AuthenticationService; import org.apache.zeppelin.socket.NotebookServer; +import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.user.Credentials; import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.jmx.ConnectorServer; @@ -102,7 +110,7 @@ public ZeppelinServer() { packages("org.apache.zeppelin.rest"); } - public static void main(String[] args) throws InterruptedException { + public static void main(String[] args) throws InterruptedException, IOException { final ZeppelinConfiguration conf = ZeppelinConfiguration.create(); conf.setProperty("args", args); @@ -152,6 +160,8 @@ protected void configure() { bindAsContract(NotebookService.class).in(Singleton.class); bindAsContract(JobManagerService.class).in(Singleton.class); bindAsContract(Notebook.class).in(Singleton.class); + bindAsContract(NoteServingTaskManagerService.class).in(Singleton.class); + bindAsContract(NoteTestTaskManagerService.class).in(Singleton.class); bindAsContract(NotebookServer.class) .to(AngularObjectRegistryListener.class) .to(RemoteInterpreterProcessListener.class) @@ -243,23 +253,9 @@ public void contextDestroyed(ServletContextEvent servletContextEvent) {} } LOG.info("Done, zeppelin server started"); - Runtime.getRuntime() - .addShutdownHook( - new Thread( - () -> { - LOG.info("Shutting down Zeppelin Server ... "); - try { - jettyWebServer.stop(); - if (!conf.isRecoveryEnabled()) { - sharedServiceLocator.getService(InterpreterSettingManager.class).close(); - } - sharedServiceLocator.getService(Notebook.class).close(); - Thread.sleep(3000); - } catch (Exception e) { - LOG.error("Error while stopping servlet container", e); - } - LOG.info("Bye"); - })); + runNoteOnStart(conf); + + Runtime.getRuntime().addShutdownHook(shutdown(conf)); // when zeppelin is started inside of ide (especially for eclipse) // for graceful shutdown, input any key in console window @@ -278,6 +274,24 @@ public void contextDestroyed(ServletContextEvent servletContextEvent) {} } } + private static Thread shutdown(ZeppelinConfiguration conf) { + return new Thread( + () -> { + LOG.info("Shutting down Zeppelin Server ... "); + try { + jettyWebServer.stop(); + if (!conf.isRecoveryEnabled()) { + sharedServiceLocator.getService(InterpreterSettingManager.class).close(); + } + sharedServiceLocator.getService(Notebook.class).close(); + Thread.sleep(3000); + } catch (Exception e) { + LOG.error("Error while stopping servlet container", e); + } + LOG.info("Bye"); + }); + } + private static Server setupJettyServer(ZeppelinConfiguration conf) { ThreadPool threadPool = new QueuedThreadPool(conf.getInt(ConfVars.ZEPPELIN_SERVER_JETTY_THREAD_POOL_MAX), @@ -325,6 +339,46 @@ private static Server setupJettyServer(ZeppelinConfiguration conf) { return server; } + private static void runNoteOnStart(ZeppelinConfiguration conf) throws IOException, InterruptedException { + String noteIdToRun = conf.getNotebookRunId(); + if (!Strings.isEmpty(noteIdToRun)) { + NotebookService notebookService = (NotebookService) ServiceLocatorUtilities.getService( + sharedServiceLocator, NotebookService.class.getName()); + + ServiceContext serviceContext; + String base64EncodedJsonSerializedServiceContext = conf.getNotebookRunServiceContext(); + if (Strings.isEmpty(base64EncodedJsonSerializedServiceContext)) { + LOG.info("No service context provided. use ANONYMOUS"); + serviceContext = new ServiceContext(AuthenticationInfo.ANONYMOUS, new HashSet() {}); + } else { + serviceContext = new Gson().fromJson( + new String(Base64.getDecoder().decode(base64EncodedJsonSerializedServiceContext)), + ServiceContext.class); + } + + notebookService.runNote(noteIdToRun, serviceContext, new ServiceCallback() { + @Override + public void onStart(String message, ServiceContext context) throws IOException { + } + + @Override + public void onSuccess(Paragraph result, ServiceContext context) throws IOException { + } + + @Override + public void onFailure(Exception ex, ServiceContext context) throws IOException { + } + }); + + if (conf.getNotebookRunAutoShutdown()) { + Thread t = shutdown(conf); + t.start(); + t.join(); + System.exit(0); + } + } + } + private static void configureRequestHeaderSize( ZeppelinConfiguration conf, ServerConnector connector) { HttpConnectionFactory cf = diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/service/NoteServingTaskManagerService.java b/zeppelin-server/src/main/java/org/apache/zeppelin/service/NoteServingTaskManagerService.java new file mode 100644 index 00000000000..0af16abd559 --- /dev/null +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/service/NoteServingTaskManagerService.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.service; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.zeppelin.background.NoteBackgroundTask; +import org.apache.zeppelin.background.NoteBackgroundTaskManager; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.plugin.PluginManager; +import org.apache.zeppelin.serving.MetricStorage; + +import javax.inject.Inject; + +public class NoteServingTaskManagerService { + + private final ZeppelinConfiguration zConf; + private final NotebookService notebookService; + private final NoteBackgroundTaskManager servingTaskManager; + private final MetricStorage metricStorage; + + @Inject + public NoteServingTaskManagerService(ZeppelinConfiguration zConf, + NotebookService notebookService) throws IOException { + this.zConf = zConf; + this.notebookService = notebookService; + + PluginManager pluginManager = PluginManager.get(); + servingTaskManager = pluginManager.loadNoteBackgroundTaskManager(); + metricStorage = pluginManager.loadNoteServingMetricStorage(); + } + + public NoteBackgroundTask startServing(String noteId, String revId, ServiceContext serviceContext) throws Exception { + final AtomicReference noteRef = new AtomicReference<>(); + final AtomicReference exRef = new AtomicReference<>(); + + notebookService.getNotebyRevision(noteId, revId, serviceContext, new ServiceCallback() { + @Override + public void onStart(String message, ServiceContext context) throws IOException { + + } + + @Override + public void onSuccess(Note result, ServiceContext context) throws IOException { + noteRef.set(result); + synchronized (noteRef) { + noteRef.notify(); + } + } + + @Override + public void onFailure(Exception ex, ServiceContext context) throws IOException { + exRef.set(ex); + synchronized (noteRef) { + noteRef.notify(); + } + } + }); + + synchronized (noteRef) { + while (noteRef.get() == null && exRef.get() == null) { + noteRef.wait(100); + } + + if (exRef.get() != null) { + throw exRef.get(); + } + + Note note = noteRef.get(); + NoteBackgroundTask servingTask = servingTaskManager.start(note, revId); + return servingTask; + } + } + + public NoteBackgroundTask stopServing(String noteId, String revId, ServiceContext serviceContext) throws Exception { + // TODO check permission + return servingTaskManager.stop(noteId, revId); + } + + public NoteBackgroundTask getServing(String noteId, String revId, ServiceContext serviceContext) throws IOException { + return servingTaskManager.get(noteId, revId); + } + + public List getAllServing() { + return servingTaskManager.list(); + } + + public MetricStorage getMetricStorage() { + return metricStorage; + } +} diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/service/NoteTestTaskManagerService.java b/zeppelin-server/src/main/java/org/apache/zeppelin/service/NoteTestTaskManagerService.java new file mode 100644 index 00000000000..8937a012264 --- /dev/null +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/service/NoteTestTaskManagerService.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.service; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.zeppelin.background.NoteBackgroundTask; +import org.apache.zeppelin.background.NoteBackgroundTaskManager; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.plugin.PluginManager; + +import javax.inject.Inject; + +public class NoteTestTaskManagerService { + private final ZeppelinConfiguration zConf; + private final NotebookService notebookService; + private final NoteBackgroundTaskManager testTaskManager; + + @Inject + public NoteTestTaskManagerService(ZeppelinConfiguration zConf, + NotebookService notebookService) throws IOException { + this.zConf = zConf; + this.notebookService = notebookService; + + PluginManager pluginManager = PluginManager.get(); + testTaskManager = pluginManager.loadNoteTestTaskManager(); + } + + public NoteBackgroundTask startTest(String noteId, String revId, ServiceContext serviceContext) throws Exception { + final AtomicReference noteRef = new AtomicReference<>(); + final AtomicReference exRef = new AtomicReference<>(); + + notebookService.getNotebyRevision(noteId, revId, serviceContext, new ServiceCallback() { + @Override + public void onStart(String message, ServiceContext context) throws IOException { + + } + + @Override + public void onSuccess(Note result, ServiceContext context) throws IOException { + noteRef.set(result); + synchronized (noteRef) { + noteRef.notify(); + } + } + + @Override + public void onFailure(Exception ex, ServiceContext context) throws IOException { + exRef.set(ex); + synchronized (noteRef) { + noteRef.notify(); + } + } + }); + + synchronized (noteRef) { + while (noteRef.get() == null && exRef.get() == null) { + noteRef.wait(100); + } + + if (exRef.get() != null) { + throw exRef.get(); + } + + Note note = noteRef.get(); + NoteBackgroundTask testTask = testTaskManager.start(note, revId); + return testTask; + } + } + + public NoteBackgroundTask stopTest(String noteId, String revId, ServiceContext serviceContext) throws Exception { + // TODO check permission + return testTaskManager.stop(noteId, revId); + } + + public NoteBackgroundTask getTest(String noteId, String revId, ServiceContext serviceContext) throws IOException { + return testTaskManager.get(noteId, revId); + } +} diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java b/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java index 2b829fda777..8a1ab584ba7 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java @@ -328,6 +328,40 @@ public boolean runParagraph(String noteId, } } + public void runNote(String noteId, + ServiceContext context, + ServiceCallback callback) throws IOException { + if (!checkPermission(noteId, Permission.RUNNER, Message.OP.RUN_ALL_PARAGRAPHS, context, + callback)) { + return; + } + + Note note = notebook.getNote(noteId); + if (note == null) { + callback.onFailure(new NoteNotFoundException(noteId), context); + return; + } + + note.setRunning(true); + try { + for (Paragraph p : note.getParagraphs()) { + String paragraphId = p.getId(); + String text = p.getText(); + String title = p.getTitle(); + Map params = p.settings.getParams(); + Map config = p.getConfig(); + + if (!runParagraph(noteId, paragraphId, title, text, params, config, false, true, + context, callback)) { + // stop execution when one paragraph fails. + break; + } + } + } finally { + note.setRunning(false); + } + } + public void runAllParagraphs(String noteId, List> paragraphs, ServiceContext context, diff --git a/zeppelin-web/package.json b/zeppelin-web/package.json index de83a629d0b..fab8fe9984d 100644 --- a/zeppelin-web/package.json +++ b/zeppelin-web/package.json @@ -14,7 +14,7 @@ "lint:watch": "esw --watch src", "lint:once": "eslint src", "predev": "grunt pre-webpack-dev", - "dev:server": "webpack-dev-server --hot", + "dev:server": "webpack-dev-server --hot --mode development", "dev:helium": "HELIUM_BUNDLE_DEV=true webpack-dev-server --hot", "dev:watch": "grunt watch-webpack-dev", "dev": "npm-run-all --parallel dev:server lint:watch dev:watch", diff --git a/zeppelin-web/src/app/app.js b/zeppelin-web/src/app/app.js index 826829a5aed..2d56c4ee579 100644 --- a/zeppelin-web/src/app/app.js +++ b/zeppelin-web/src/app/app.js @@ -107,10 +107,19 @@ let zeppelinWebApp = angular.module('zeppelinWebApp', requiredModules) controller: 'NotebookCtrl', resolve: visBundleLoad, }) + .when('/notebook/:noteId/task/:revisionId', { + templateUrl: 'app/notebook/notebook.html', + controller: 'NotebookCtrl', + resolve: visBundleLoad, + }) .when('/jobmanager', { templateUrl: 'app/jobmanager/jobmanager.html', controller: 'JobManagerCtrl', }) + .when('/serving', { + templateUrl: 'app/serving/serving.html', + controller: 'ServingCtrl', + }) .when('/interpreter', { templateUrl: 'app/interpreter/interpreter.html', controller: 'InterpreterCtrl', diff --git a/zeppelin-web/src/app/notebook/note-serving-status/note-serving-status.component.js b/zeppelin-web/src/app/notebook/note-serving-status/note-serving-status.component.js new file mode 100644 index 00000000000..c8d2d3770c1 --- /dev/null +++ b/zeppelin-web/src/app/notebook/note-serving-status/note-serving-status.component.js @@ -0,0 +1,108 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import noteServingStatusTemplate from './note-serving-status.html'; +import './note-serving-status.css'; + +class NoteServingStatusController { + constructor($http, $location, baseUrlSrv) { + 'ngInject'; + this.servingStatus = '...'; // one of 'Loading', 'Not running', 'Running', 'Error' + this.apiBaseAddr = baseUrlSrv.getRestApiBase(); + this.location = $location; + this.http = $http; + } + + $onInit() { + this.updateStatus(); + } + + $onDestroy() { + if (this._t) { + clearTimeout(this._t); + } + } + + updateStatus() { + let self = this; + + if (this.noteid && this.revid) { + this.http.get(this.apiBaseAddr + '/notebook/serving/' + this.noteid + '/' + this.revid) + .success(function(data, status) { + if (data.body.task) { + if (Object.keys(data.body.task).length === 0) { + self.servingStatus = 'Not running'; + } else { + self.servingStatus = 'Running'; + } + } else { + self.servingStatus = 'Not running'; + } + }) + .error(function(data, status) { + if (self.noteid && self.revid) { + self.servingStatus = 'Not running'; + } + }); + } + + self._t = setTimeout(function() { + self.updateStatus(); + }, 3000); + } + + showNote() { + console.info('show note'); + this.location.path('/notebook/' + this.noteid + '/task/' + this.revid); + } + + startServing() { + console.info('start serving'); + let self = this; + this.http.post(this.apiBaseAddr + '/notebook/serving/' + this.noteid + '/' + this.revid) + .success(function(data, status) { + self.updateStatus(); + }) + .error(function(data, status) { + self.testStatus = 'Error'; + }); + } + + stopServing() { + console.info('stop serving'); + let self = this; + this.http.delete(this.apiBaseAddr + '/notebook/serving/' + this.noteid + '/' + this.revid) + .success(function(data, status) { + self.updateStatus(); + }) + .error(function(data, status) { + self.testStatus = 'Error'; + }); + } +} + +export const NoteServingStatusComponent = { + template: noteServingStatusTemplate, + controller: NoteServingStatusController, + bindings: { + noteid: '<', + revid: '<', + textonly: '<', + }, +}; + +export const NoteServingStatusModule = angular + .module('zeppelinWebApp') + .component('noteServingStatus', NoteServingStatusComponent) + .name; diff --git a/zeppelin-web/src/app/notebook/note-serving-status/note-serving-status.css b/zeppelin-web/src/app/notebook/note-serving-status/note-serving-status.css new file mode 100644 index 00000000000..450138b3231 --- /dev/null +++ b/zeppelin-web/src/app/notebook/note-serving-status/note-serving-status.css @@ -0,0 +1,26 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.note-serving-status-component { + display: inline-block; +} + +.note-serving-status-component-buttons span { + cursor: pointer; +} + +.note-serving-status-component-infos { + font-size: 12px; + margin-top: 10px; +} diff --git a/zeppelin-web/src/app/notebook/note-serving-status/note-serving-status.html b/zeppelin-web/src/app/notebook/note-serving-status/note-serving-status.html new file mode 100644 index 00000000000..f1ee99a0e7a --- /dev/null +++ b/zeppelin-web/src/app/notebook/note-serving-status/note-serving-status.html @@ -0,0 +1,44 @@ + + + + +
+ {{$ctrl.servingStatus}} + + + + + + + + + + + +
+
+
diff --git a/zeppelin-web/src/app/notebook/note-test-status/note-test-status.component.js b/zeppelin-web/src/app/notebook/note-test-status/note-test-status.component.js new file mode 100644 index 00000000000..e6a2432b391 --- /dev/null +++ b/zeppelin-web/src/app/notebook/note-test-status/note-test-status.component.js @@ -0,0 +1,107 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import noteTestStatusTemplate from './note-test-status.html'; +import './note-test-status.css'; + +class NoteTestStatusController { + constructor($http, $location, baseUrlSrv) { + 'ngInject'; + this.testStatus = 'notStarted'; // one of 'notStarted', 'running', 'fail', 'pass' + this.apiBaseAddr = baseUrlSrv.getRestApiBase(); + this.location = $location; + this.http = $http; + } + + $onInit() { + this.updateStatus(); + } + + $onDestroy() { + if (this._t) { + clearTimeout(this._t); + } + } + + updateStatus() { + let self = this; + this.http.get(this.apiBaseAddr + '/notebook/test/' + this.noteid + '/' + this.revid) + .success(function(data, status) { + if (data.body.running) { + self.testStatus = 'running'; + self._t = setTimeout(function() { + self.updateStatus(); + }, 3000); + } else { + if (data.body.result && data.body.result.fail > 0) { + self.testStatus = 'fail'; + } else { + self.testStatus = 'pass'; + } + if (self._t) { + clearTimeout(self._t); + } + } + }) + .error(function(data, status) { + self.testStatus = 'notStarted'; + if (self._t) { + clearTimeout(self._t); + } + }); + } + + showNote() { + console.info('show note'); + this.location.path('/notebook/' + this.noteid + '/task/' + this.revid); + } + + startTest() { + console.info('start test'); + let self = this; + this.http.post(this.apiBaseAddr + '/notebook/test/' + this.noteid + '/' + this.revid) + .success(function(data, status) { + self.updateStatus(); + }) + .error(function(data, status) { + self.testStatus = 'fail'; + }); + } + + stopTest() { + console.info('stop test'); + let self = this; + this.http.delete(this.apiBaseAddr + '/notebook/test/' + this.noteid + '/' + this.revid) + .success(function(data, status) { + self.updateStatus(); + }) + .error(function(data, status) { + self.testStatus = 'fail'; + }); + } +} + +export const NoteTestStatusComponent = { + template: noteTestStatusTemplate, + controller: NoteTestStatusController, + bindings: { + noteid: '<', + revid: '<', + }, +}; + +export const NoteTestStatusModule = angular + .module('zeppelinWebApp') + .component('noteTestStatus', NoteTestStatusComponent) + .name; diff --git a/zeppelin-web/src/app/notebook/note-test-status/note-test-status.css b/zeppelin-web/src/app/notebook/note-test-status/note-test-status.css new file mode 100644 index 00000000000..2b68c3ff6b9 --- /dev/null +++ b/zeppelin-web/src/app/notebook/note-test-status/note-test-status.css @@ -0,0 +1,8 @@ +.note-test-status-component { + padding: 3px 3px 3px 3px; + margin-right: 7px; +} + +.note-test-status-component span:hover { + cursor: pointer; +} diff --git a/zeppelin-web/src/app/notebook/note-test-status/note-test-status.html b/zeppelin-web/src/app/notebook/note-test-status/note-test-status.html new file mode 100644 index 00000000000..61a0175b4a8 --- /dev/null +++ b/zeppelin-web/src/app/notebook/note-test-status/note-test-status.html @@ -0,0 +1,29 @@ + + +
+ + + + + + + + + + + + + +
diff --git a/zeppelin-web/src/app/notebook/notebook-actionBar.html b/zeppelin-web/src/app/notebook/notebook-actionBar.html index 668697bdcb9..842d9a5b946 100644 --- a/zeppelin-web/src/app/notebook/notebook-actionBar.html +++ b/zeppelin-web/src/app/notebook/notebook-actionBar.html @@ -28,203 +28,204 @@

- - - - - - - - - - - - - -
+ + -
    -
  • -
    -
    - - -
    -
    -
    • -
+ -
-
- - - -
- + + - - -
    -
  • -
    - Find - - - {{search.currentOccurrence}} of {{search.occurrencesCount}} - -
    - -
    -
    - -
    -
    -
    - Replace - -
    - + +
    + + +
      +
    • +
      +
      + + +
      +
      +
      • +
    + +
    +
    + + + +
    +     + + + +
      +
    • +
      + Find + + + {{search.currentOccurrence}} of {{search.occurrencesCount}} + +
      + +
      +
      + +
      -
      - +
      + Replace + +
      + +
      +
      + +
      -
      -
      • -
    -     +
    • +
+ - +
+ diff --git a/zeppelin-web/src/app/notebook/notebook.controller.js b/zeppelin-web/src/app/notebook/notebook.controller.js index 085c94e53da..4bf53fb3904 100644 --- a/zeppelin-web/src/app/notebook/notebook.controller.js +++ b/zeppelin-web/src/app/notebook/notebook.controller.js @@ -62,13 +62,20 @@ function NotebookCtrl($scope, $route, $routeParams, $location, $rootScope, let connectedOnce = false; let isRevisionPath = function(path) { - let pattern = new RegExp('^.*\/notebook\/[a-zA-Z0-9_]*\/revision\/[a-zA-Z0-9_]*'); + let pattern = new RegExp('^.*\/notebook\/[a-zA-Z0-9_]*\/(revision|task)\/[a-zA-Z0-9_]*'); + return pattern.test(path); + }; + $scope.revisionId = $routeParams.revisionId; + + let isTaskPath = function(path) { + let pattern = new RegExp('^.*\/notebook\/[a-zA-Z0-9_]*\/task\/[a-zA-Z0-9_]*'); return pattern.test(path); }; $scope.noteRevisions = []; $scope.currentRevision = 'Head'; $scope.revisionView = isRevisionPath($location.path()); + $scope.taskView = isTaskPath($location.path()); $scope.search = { searchText: '', @@ -163,11 +170,26 @@ function NotebookCtrl($scope, $route, $routeParams, $location, $rootScope, const initNotebook = function() { noteVarShareService.clear(); if ($routeParams.revisionId) { - websocketMsgSrv.getNoteByRevision($routeParams.noteId, $routeParams.revisionId); + if ($scope.taskView) { + $http.get( + baseUrlSrv.getRestApiBase() + + '/notebook/test/' + + $routeParams.noteId + '/' + + $routeParams.revisionId + '/note') + .success(function(data, status) { + console.info('Test task note', data.body); + setNoteContent(data.body.note); + }) + .error(function(data, status) { + + }); + } else { + websocketMsgSrv.getNoteByRevision($routeParams.noteId, $routeParams.test); + } } else { websocketMsgSrv.getNote($routeParams.noteId); } - websocketMsgSrv.listRevisionHistory($routeParams.noteId); + let currentRoute = $route.current; if (currentRoute) { setTimeout( @@ -344,6 +366,7 @@ function NotebookCtrl($scope, $route, $routeParams, $location, $rootScope, if (data.note) { $scope.note = data.note; initializeLookAndFeel(); + websocketMsgSrv.listRevisionHistory($routeParams.noteId); } else { $location.path('/'); } @@ -1464,6 +1487,10 @@ function NotebookCtrl($scope, $route, $routeParams, $location, $rootScope, }); $scope.$on('setNoteContent', function(event, note) { + setNoteContent(note); + }); + + let setNoteContent = function(note) { if (note === undefined) { $location.path('/'); } @@ -1472,6 +1499,7 @@ function NotebookCtrl($scope, $route, $routeParams, $location, $rootScope, $scope.paragraphUrl = $routeParams.paragraphId; $scope.asIframe = $routeParams.asIframe; + if ($scope.paragraphUrl) { $scope.note = cleanParagraphExcept($scope.paragraphUrl, $scope.note); $scope.$broadcast('$unBindKeyEvent', $scope.$unBindKeyEvent); @@ -1483,12 +1511,16 @@ function NotebookCtrl($scope, $route, $routeParams, $location, $rootScope, initializeLookAndFeel(); // open interpreter binding setting when there're none selected - getInterpreterBindings(); - getPermissions(); + if (!$scope.taskView) { + getInterpreterBindings(); + getPermissions(); + } + websocketMsgSrv.listRevisionHistory($routeParams.noteId); + let isPersonalized = $scope.note.config.personalizedMode; isPersonalized = isPersonalized === undefined ? 'false' : isPersonalized; $scope.note.config.personalizedMode = isPersonalized; - }); + }; $scope.$on('$routeChangeStart', function(event, next, current) { if (!$scope.note || !$scope.note.paragraphs) { diff --git a/zeppelin-web/src/app/notebook/paragraph/result/result.controller.js b/zeppelin-web/src/app/notebook/paragraph/result/result.controller.js index f4e5b825f4f..1be3bbb324c 100644 --- a/zeppelin-web/src/app/notebook/paragraph/result/result.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/result/result.controller.js @@ -688,7 +688,7 @@ function ResultCtrl($scope, $rootScope, $route, $window, $routeParams, $location }; } else if (refresh) { // when graph options or data are changed - console.log('Refresh data %o', tableData); + // console.log('Refresh data %o', tableData); afterLoaded = function(loadedElem) { const transformationSettingTargetEl = getTrSettingElem($scope.id, graphMode); diff --git a/zeppelin-web/src/app/serving/metric/serving-metric-chart.component.js b/zeppelin-web/src/app/serving/metric/serving-metric-chart.component.js new file mode 100644 index 00000000000..e69de29bb2d diff --git a/zeppelin-web/src/app/serving/metric/serving-metric.component.js b/zeppelin-web/src/app/serving/metric/serving-metric.component.js new file mode 100644 index 00000000000..83bbef9ee95 --- /dev/null +++ b/zeppelin-web/src/app/serving/metric/serving-metric.component.js @@ -0,0 +1,178 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import servingMetricTemplate from './serving-metric.html'; +import './serving-metric.css'; +import Linechart from '../../visualization/builtins/visualization-linechart'; + +class ServingMetricController { + constructor($http, $location, baseUrlSrv, $element) { + 'ngInject'; + this.apiBaseAddr = baseUrlSrv.getRestApiBase(); + this.http = $http; + this.element = $element; + } + + $onInit() { + this.refresh(); + } + + $onDestroy() { + } + + refresh() { + let self = this; + this.http.get(this.apiBaseAddr + '/notebook/serving/' + + this.noteid + '/' + this.revid + '/' + this.endpoint) + .success(function(data, status) { + self.metrics = data.body; + self.drawChart(); + }) + .error(function(data, status) { + self.drawChart(); + }); + } + + newTable() { + return { + columns: [ + { + name: 'date', + index: 0, + aggr: 'sum', + }, + { + name: 'group', + index: 2, + aggr: 'sum', + }, + { + name: 'value', + index: 1, + aggr: 'sum', + } + ], + rows: [], + } + } + + drawChart() { + let tables = {}; + let allDates = []; + this.metrics.forEach((m) => { + let keys = Object.keys(m.metric); + let date = new Date(m.timestamp * 1000); + let dateString = date.getHours() + ':' + date.getMinutes(); + allDates.push(dateString); + + keys.forEach((key) => { + let keys = key.split('.'); + let tableName = keys[0]; + let groupName = (keys.length === 1) ? '' : key.substring(keys[0].length + 1); + + // create new table + if (tableName === "elapsed") { + if (!tables["numRequest"]) { + tables["numRequest"] = this.newTable(); + tables["avgLatencyMsec"] = this.newTable(); + } + + let value = m.metric[key]; + tables["numRequest"].rows.push([ + dateString, + value.count, + groupName, + ]); + + tables["avgLatencyMsec"].rows.push([ + dateString, + value.sum / value.count, + groupName, + ]); + } else { + if (!tables[tableName]) { + tables[tableName] = this.newTable(); + } + + let value = m.metric[key]; + tables[tableName].rows.push([ + dateString, + (tableName.match(/num/i)) ? value.count + : (tableName.match(/avg/i)) ? value.sum / value.count + : value.sum, + groupName, + ]); + } + }); + }); + + this.tableNames = Object.keys(tables); + let self = this; + Object.keys(tables).forEach((tableName) => { + // fill zero in table data row, on missing value + let tableData = tables[tableName]; + let rowCur = 0; + let zeroFilledRows = allDates.map((dateString) => { + if (rowCur < tableData.rows.length) { + if (tableData.rows[rowCur][0] === dateString) { + return tableData.rows[rowCur++]; + } + } + + return [dateString, 0, tableName]; + }); + tableData.rows = zeroFilledRows; + + setTimeout(() => { + let chartEl = this.element.find('#chart_' + + self.noteid + self.revid + self.endpoint + tableName); + let chart = new Linechart(chartEl, {}); + let pivot = chart.getTransformation(); + pivot.config = {common: {}}; + pivot.config.common.pivot = { + keys: [{ + name: 'date', + index: 0, + aggr: 'sum', + }], + groups: [], + values: [{ + name: 'value', + index: 1, + aggr: (tableName.match(/num/i)) ? 'sum' + : (tableName.match(/avg/i)) ? 'avg' + : 'sum', + }], + }; + chart.render(pivot.transform(tableData)); + }, 500); + }); + } +} + + +export const ServingMetricComponent = { + template: servingMetricTemplate, + controller: ServingMetricController, + bindings: { + noteid: '<', + revid: '<', + endpoint: '<', + }, +}; + +export const ServingMetricModule = angular + .module('zeppelinWebApp') + .component('servingMetric', ServingMetricComponent) + .name; diff --git a/zeppelin-web/src/app/serving/metric/serving-metric.css b/zeppelin-web/src/app/serving/metric/serving-metric.css new file mode 100644 index 00000000000..1e216a9715d --- /dev/null +++ b/zeppelin-web/src/app/serving/metric/serving-metric.css @@ -0,0 +1,17 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + .serving-metric-chart { + height: 300px; + } diff --git a/zeppelin-web/src/app/serving/metric/serving-metric.html b/zeppelin-web/src/app/serving/metric/serving-metric.html new file mode 100644 index 00000000000..6c42dd60d4a --- /dev/null +++ b/zeppelin-web/src/app/serving/metric/serving-metric.html @@ -0,0 +1,23 @@ + + +
+
+ Metric: {{tableName}} +
+ +
+
+
diff --git a/zeppelin-web/src/app/serving/serving.controller.js b/zeppelin-web/src/app/serving/serving.controller.js new file mode 100644 index 00000000000..f16a03af37a --- /dev/null +++ b/zeppelin-web/src/app/serving/serving.controller.js @@ -0,0 +1,62 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import './serving.html'; +import './serving.css'; + +class ServingController { + constructor($scope, $http, baseUrlSrv) { + 'ngInject'; + this.$http = $http; + this.$scope = $scope; + this.apiBaseAddr = baseUrlSrv.getRestApiBase(); + this.listServings(); + + let self = this; + $scope.refresh = function() { + self.listServings(); + }; + $scope.loading = false; + } + + listServings() { + console.log('list servings'); + let $scope = this.$scope; + $scope.loading = true; + this.$http.get(this.apiBaseAddr + '/notebook/serving') + .success(function(data, status) { + $scope.loading = false; + let servings = {}; + console.log(data.body); + data.body.forEach((s) => { + if (!servings[s.note.id]) { + servings[s.note.id] = []; + } + servings[s.note.id].push(s); + }); + $scope.servings = servings; + if (Object.keys(servings).length === 0) { + $scope.noServings = true; + } + console.log($scope.servings); + }) + .error(function(data, status) { + $scope.loading = false; + // error + }); + } +} + +angular.module('zeppelinWebApp') + .controller('ServingCtrl', ServingController); diff --git a/zeppelin-web/src/app/serving/serving.css b/zeppelin-web/src/app/serving/serving.css new file mode 100644 index 00000000000..72036d569e3 --- /dev/null +++ b/zeppelin-web/src/app/serving/serving.css @@ -0,0 +1,26 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + .serving-page-header { + margin: -10px -10px 20px; + padding: 10px 15px 15px 15px; + background: white; + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.15); + border-bottom: 1px solid #E5E5E5; + } + + .serving-page-panel { + background-color: white; + padding: 15px 15px 15px 15px; + margin-bottom: 10px; + } diff --git a/zeppelin-web/src/app/serving/serving.html b/zeppelin-web/src/app/serving/serving.html new file mode 100644 index 00000000000..7967fffea67 --- /dev/null +++ b/zeppelin-web/src/app/serving/serving.html @@ -0,0 +1,62 @@ + + +
+
+
+
+

+ Serving +

+
+
+
+
+
You can monitor the status of notebook and navigate to note or paragraph.
+
+ Reload +
+
+
+
+
+ +
+
+ +
+
+
Endpoint: "{{endpoint}}" (/serving/{{noteId}}/{{serving.note.revision}}/{{endpoint}})
+ +
+
+
+
+
+ No note serving found. +
+
diff --git a/zeppelin-web/src/app/tabledata/pivot.js b/zeppelin-web/src/app/tabledata/pivot.js index 2baa6b5c8d8..ca5f8ce2c29 100644 --- a/zeppelin-web/src/app/tabledata/pivot.js +++ b/zeppelin-web/src/app/tabledata/pivot.js @@ -27,7 +27,6 @@ export default class PivotTransformation extends Transformation { let self = this; let configObj = self.config; - console.log('getSetting', configObj); return { template: 'app/tabledata/pivot_settings.html', scope: { @@ -74,6 +73,7 @@ export default class PivotTransformation extends Transformation { if (firstTime) { this.selectDefault(); } + return this.pivot( tableData, config.keys, diff --git a/zeppelin-web/src/components/navbar/navbar.html b/zeppelin-web/src/components/navbar/navbar.html index 59d65c9936b..247498fcba4 100644 --- a/zeppelin-web/src/components/navbar/navbar.html +++ b/zeppelin-web/src/components/navbar/navbar.html @@ -65,6 +65,11 @@ Job +
  • + + Serving + +
      • diff --git a/zeppelin-web/src/index.js b/zeppelin-web/src/index.js index 19d2d3e1ef8..66cc74c6bc4 100644 --- a/zeppelin-web/src/index.js +++ b/zeppelin-web/src/index.js @@ -36,6 +36,8 @@ import './app/visualization/builtins/visualization-linechart.js'; import './app/visualization/builtins/visualization-scatterchart.js'; import './app/jobmanager/jobmanager.component.js'; +import './app/serving/serving.controller.js'; +import './app/serving/metric/serving-metric.component.js'; import './app/interpreter/interpreter.controller.js'; import './app/interpreter/interpreter.filter.js'; import './app/interpreter/interpreter-item.directive.js'; @@ -53,6 +55,8 @@ import './app/notebook/save-as/browser-detect.service.js'; import './app/notebook/elastic-input/elastic-input.controller.js'; import './app/notebook/dropdown-input/dropdown-input.directive.js'; import './app/notebook/note-var-share.service.js'; +import './app/notebook/note-test-status/note-test-status.component'; +import './app/notebook/note-serving-status/note-serving-status.component'; import './app/notebook-repository/notebook-repository.controller.js'; import './app/search/result-list.controller.js'; import './app/search/search.service.js'; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/background/BackgroundTaskLifecycleListener.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/background/BackgroundTaskLifecycleListener.java new file mode 100644 index 00000000000..aedc5a68daa --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/background/BackgroundTaskLifecycleListener.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.background; + +public interface BackgroundTaskLifecycleListener { + void onBackgroundTaskStart(String taskId); + void onBackgroundTaskUpdate(String taskId); + void onBackgroundTaskError(String taskId); + void onBackgroundTaskStop(String taskId); +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/background/FileSystemTaskContextStorage.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/background/FileSystemTaskContextStorage.java new file mode 100644 index 00000000000..3546b2d90a5 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/background/FileSystemTaskContextStorage.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.background; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.commons.io.FileUtils; +import org.apache.zeppelin.notebook.Note; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Local filesystem implementation of TaskContextStorage. + * + * Stores context under + * + * // + */ +public class FileSystemTaskContextStorage implements TaskContextStorage { + private static final Logger LOGGER = LoggerFactory.getLogger(FileSystemTaskContextStorage.class); + private final File contextRootDir; + + public FileSystemTaskContextStorage(String contextRoot) { + contextRootDir = new File(contextRoot); + } + + @Override + public void save(TaskContext context) throws IOException { + delete(context.getId()); + File contextDir = new File(contextRootDir, context.getId()); + File notebookDir = new File(contextDir, "notebook"); + File revFile = new File(contextDir, "rev"); + notebookDir.mkdirs(); + + Note note = context.getNote(); + File noteFile = new File(notebookDir, note.getName() + "_" + note.getId() + ".zpln"); + FileUtils.writeStringToFile(noteFile, note.toJson()); + FileUtils.writeStringToFile(revFile, context.getRevId()); + } + + @Override + public TaskContext load(String taskId) throws IOException { + File contextDir = new File(contextRootDir, taskId); + File notebookDir = new File(contextDir, "notebook"); + File revFile = new File(contextDir, "rev"); + + if (!contextDir.isDirectory()) { + return null; + } + + Collection files = FileUtils.listFiles(notebookDir, new String[]{"zpln"}, false); + File noteFile = files.iterator().next(); + Note note = Note.fromJson(FileUtils.readFileToString(noteFile)); + + String revId = FileUtils.readFileToString(revFile); + return new TaskContext(note, revId); + } + + @Override + public void delete(String taskId) throws IOException { + File contextDir = new File(contextRootDir, taskId); + FileUtils.deleteDirectory(contextDir); + } + + @Override + public List list() { + File[] files = contextRootDir.listFiles(); + if (files == null) { + return new LinkedList<>(); + } + + return Arrays.stream(files).map(f -> { + try { + return load(f.getName()); + } catch (IOException e) { + LOGGER.error("Can't load task context", e); + return null; + } + }).collect(Collectors.toList()); + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/background/NoteBackgroundTask.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/background/NoteBackgroundTask.java new file mode 100644 index 00000000000..2fa7d1ed356 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/background/NoteBackgroundTask.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.background; + +import java.io.IOException; +import java.util.Map; + +/** + * Note test task + */ +public abstract class NoteBackgroundTask { + private TaskContext taskContext; + + public NoteBackgroundTask(TaskContext taskContext) { + this.taskContext = taskContext; + } + + public TaskContext getTaskContext() { + return taskContext; + } + + public abstract void start() throws IOException; + public abstract void stop() throws IOException; + public abstract boolean isRunning() throws IOException; + public abstract Map getInfo() throws IOException; +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/background/NoteBackgroundTaskManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/background/NoteBackgroundTaskManager.java new file mode 100644 index 00000000000..ac576072134 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/background/NoteBackgroundTaskManager.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.background; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.notebook.Note; + +/** + * Manage note background task. such as test task, serving task. + */ +public abstract class NoteBackgroundTaskManager { + private final ZeppelinConfiguration zConf; + BackgroundTaskLifecycleListener listener; + + public NoteBackgroundTaskManager(ZeppelinConfiguration zConf) { + this.zConf = zConf; + } + + public NoteBackgroundTask start(Note note, String revId) throws IOException { + // create task + TaskContext taskContext = new TaskContext(note, revId); + getTaskContextStorage().save(taskContext); + NoteBackgroundTask task = createOrGetBackgroundTask(taskContext); + + // start test + boolean running = false; + try { + running = task.isRunning(); + } catch (IOException e) { + // task not exists. ignore exception here + } + if (!running) { + task.start(); + } + return task; + } + + public NoteBackgroundTask stop(String noteId, String revId) throws IOException { + NoteBackgroundTask task = get(noteId, revId); + if (task == null) { + return null; + } + + // stop test + task.stop(); + + getTaskContextStorage().delete(task.getTaskContext().getId()); + return task; + } + + public void delete(String noteId, String revId) throws IOException { + getTaskContextStorage().delete(TaskContext.getTaskId(noteId, revId)); + } + + public NoteBackgroundTask get(String noteId, String revId) throws IOException { + TaskContext taskContext = getTaskContextStorage().load(TaskContext.getTaskId(noteId, revId)); + if (taskContext == null) { + return null; + } + + NoteBackgroundTask task = createOrGetBackgroundTask(taskContext); + return task; + } + + public List list() { + List contexts = getTaskContextStorage().list(); + return contexts.stream().map(c -> createOrGetBackgroundTask(c)).collect(Collectors.toList()); + } + + public ZeppelinConfiguration getConf() { + return zConf; + } + + protected abstract TaskContextStorage getTaskContextStorage(); + protected abstract NoteBackgroundTask createOrGetBackgroundTask(TaskContext taskContext); + + public BackgroundTaskLifecycleListener getListener() { + return listener; + } + + public void setListener(BackgroundTaskLifecycleListener listener) { + this.listener = listener; + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/background/TaskContext.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/background/TaskContext.java new file mode 100644 index 00000000000..1dda474e6ce --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/background/TaskContext.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.background; + +import org.apache.zeppelin.notebook.Note; + +/** + * TaskContext + */ +public class TaskContext { + private final String revId; + private final Note note; + private String id; + + public TaskContext(Note note, String revId) { + id = getTaskId(note.getId(), revId); + this.note = note; + this.revId = revId; + } + + public String getRevId() { + return revId; + } + + public Note getNote() { + return note; + } + + public String getId() { + return id; + } + + public static String getTaskId(String noteId, String revId) { + return (noteId + "-" + revId).toLowerCase(); + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/background/TaskContextStorage.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/background/TaskContextStorage.java new file mode 100644 index 00000000000..579783f60c5 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/background/TaskContextStorage.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.background; + +import java.io.IOException; +import java.util.List; + +/** + * Storage interface for task context + */ +public interface TaskContextStorage { + void save(TaskContext context) throws IOException; + TaskContext load(String taskId) throws IOException; + void delete(String taskId) throws IOException; + List list(); +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java index 44d99be2173..467470d442d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java @@ -47,10 +47,12 @@ import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.NoteEventListener; import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.plugin.PluginManager; import org.apache.zeppelin.resource.Resource; import org.apache.zeppelin.resource.ResourcePool; import org.apache.zeppelin.resource.ResourceSet; import org.apache.zeppelin.scheduler.Job; +import org.apache.zeppelin.serving.RestApiRouter; import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.util.ReflectionUtils; import org.apache.zeppelin.storage.ConfigStorage; @@ -131,8 +133,7 @@ public class InterpreterSettingManager implements NoteEventListener { @Inject public InterpreterSettingManager(ZeppelinConfiguration zeppelinConfiguration, AngularObjectRegistryListener angularObjectRegistryListener, - RemoteInterpreterProcessListener - remoteInterpreterProcessListener, + RemoteInterpreterProcessListener remoteInterpreterProcessListener, ApplicationEventListener appEventListener) throws IOException { this(zeppelinConfiguration, new InterpreterOption(), @@ -177,7 +178,11 @@ public InterpreterSettingManager(ZeppelinConfiguration conf, LOGGER.info("Using LifecycleManager: " + this.lifecycleManager.getClass().getName()); this.configStorage = configStorage; - this.interpreterEventServer = new RemoteInterpreterEventServer(conf, this); + + PluginManager pluginManager = PluginManager.get(); + RestApiRouter apiRouter = pluginManager.loadNoteServingRestApiRouter(); + + this.interpreterEventServer = new RemoteInterpreterEventServer(conf, this, apiRouter); this.interpreterEventServer.start(); init(); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/RemoteInterpreterEventServer.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/RemoteInterpreterEventServer.java index 7f009347903..aa1fcdda293 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/RemoteInterpreterEventServer.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/RemoteInterpreterEventServer.java @@ -19,6 +19,8 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import org.apache.thrift.TException; import org.apache.thrift.server.TThreadPoolServer; import org.apache.thrift.transport.TServerSocket; @@ -42,13 +44,16 @@ import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventService; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResultMessage; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; +import org.apache.zeppelin.interpreter.thrift.RestApiInfo; import org.apache.zeppelin.interpreter.thrift.RunParagraphsEvent; import org.apache.zeppelin.interpreter.thrift.ServiceException; +import org.apache.zeppelin.plugin.PluginManager; import org.apache.zeppelin.resource.RemoteResource; import org.apache.zeppelin.resource.Resource; import org.apache.zeppelin.resource.ResourceId; import org.apache.zeppelin.resource.ResourcePool; import org.apache.zeppelin.resource.ResourceSet; +import org.apache.zeppelin.serving.RestApiRouter; import org.apache.zeppelin.user.AuthenticationInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,6 +72,7 @@ public class RemoteInterpreterEventServer implements RemoteInterpreterEventServi private static final Logger LOGGER = LoggerFactory.getLogger(RemoteInterpreterEventServer.class); + private final ZeppelinConfiguration zConf; private String portRange; private int port; private String host; @@ -80,13 +86,18 @@ public class RemoteInterpreterEventServer implements RemoteInterpreterEventServi private final RemoteInterpreterProcessListener listener; private final ApplicationEventListener appListener; private final Gson gson = new Gson(); + private final RestApiRouter restApiRouter; + public RemoteInterpreterEventServer(ZeppelinConfiguration zConf, - InterpreterSettingManager interpreterSettingManager) { + InterpreterSettingManager interpreterSettingManager, + RestApiRouter restApiRouter) { + this.zConf = zConf; this.portRange = zConf.getZeppelinServerRPCPortRange(); this.interpreterSettingManager = interpreterSettingManager; this.listener = interpreterSettingManager.getRemoteInterpreterProcessListener(); this.appListener = interpreterSettingManager.getAppEventListener(); + this.restApiRouter = restApiRouter; } public void start() throws IOException { @@ -367,6 +378,30 @@ public List getParagraphList(String user, String noteId) } } + @Override + public void addRestApi(RestApiInfo restApiInfo) throws TException { + LOGGER.info("add rest api from remote interpreter"); + + ManagedInterpreterGroup interpreterGroup = interpreterSettingManager.getInterpreterGroupById( + restApiInfo.getIntpGroupId()); + RemoteInterpreterProcess intpProcess = interpreterGroup.getInterpreterProcess(); + String hostAddress = intpProcess.getHost(); + + String revId = zConf.getNotebookRunRev(); + + try { + restApiRouter.addRoute( + restApiInfo.getNoteId(), + revId, + hostAddress, + restApiInfo.getHostname(), + restApiInfo.getServerPort(), + restApiInfo.getEndpointName()); + } catch (IOException e) { + LOGGER.error("Error on adding route for serving {}", restApiInfo); + } + } + private Object invokeResourceMethod(String intpGroupId, final InvokeResourceMethodEventMessage message) { final ResourceId resourceId = message.resourceId; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java index 02e8fd0bf54..36c7a468e7a 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java @@ -48,6 +48,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess { private final String interpreterSettingName; private final String interpreterGroupId; private final boolean isUserImpersonated; + private int interpreterRestApiServerPort = 0; private Map env; @@ -56,6 +57,7 @@ public RemoteInterpreterManagedProcess( int zeppelinServerRPCPort, String zeppelinServerRPCHost, String interpreterPortRange, + int interpreterRestApiServerPort, String intpDir, String localRepoDir, Map env, @@ -74,6 +76,7 @@ public RemoteInterpreterManagedProcess( this.interpreterSettingName = interpreterSettingName; this.interpreterGroupId = interpreterGroupId; this.isUserImpersonated = isUserImpersonated; + this.interpreterRestApiServerPort = interpreterRestApiServerPort; } @Override @@ -108,6 +111,8 @@ public void start(String userName) throws IOException { cmdLine.addArgument(localRepoDir, false); cmdLine.addArgument("-g", false); cmdLine.addArgument(interpreterSettingName, false); + cmdLine.addArgument("-s", false); + cmdLine.addArgument(String.valueOf(interpreterRestApiServerPort), false); Map procEnv = EnvironmentUtils.getProcEnvironment(); procEnv.putAll(env); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java index 6f9b0e9445f..a78b6f13d5e 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java @@ -18,11 +18,17 @@ package org.apache.zeppelin.plugin; import com.google.common.annotations.VisibleForTesting; +import org.apache.zeppelin.background.NoteBackgroundTaskManager; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.launcher.InterpreterLauncher; import org.apache.zeppelin.interpreter.recovery.RecoveryStorage; import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.apache.zeppelin.notebook.repo.OldNotebookRepo; +import org.apache.zeppelin.serving.DummyNoteBackgroundTaskManager; +import org.apache.zeppelin.serving.DummyRestApiRouter; +import org.apache.zeppelin.serving.MetricStorage; +import org.apache.zeppelin.serving.RedisMetricStorage; +import org.apache.zeppelin.serving.RestApiRouter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,6 +54,8 @@ public class PluginManager { private String pluginsDir = zConf.getPluginsDir(); private Map cachedLaunchers = new HashMap<>(); + private Map cachedBackgroundTaskManagers = new HashMap<>(); + private Map cachedRestApiRouter = new HashMap<>(); public static synchronized PluginManager get() { if (instance == null) { @@ -162,6 +170,119 @@ public synchronized InterpreterLauncher loadInterpreterLauncher(String launcherP return launcher; } + public synchronized NoteBackgroundTaskManager loadNoteBackgroundTaskManager() throws IOException { + if (zConf.getRunMode() == ZeppelinConfiguration.RUN_MODE.K8S) { + /** + * For now, class name is hardcoded here. + * Later, we can make it configurable if necessary. + */ + return loadNoteBackgroundTaskManager( + "K8sStandardInterpreterLauncher", + "org.apache.zeppelin.serving.K8sNoteServingTaskManager" + ); + } else { + return new DummyNoteBackgroundTaskManager(zConf); + } + } + + public synchronized NoteBackgroundTaskManager loadNoteTestTaskManager() throws IOException { + if (zConf.getRunMode() == ZeppelinConfiguration.RUN_MODE.K8S) { + /** + * For now, class name is hardcoded here. + * Later, we can make it configurable if necessary. + */ + return loadNoteBackgroundTaskManager( + "K8sStandardInterpreterLauncher", + "org.apache.zeppelin.test.K8sNoteTestTaskManager" + ); + } else { + return new DummyNoteBackgroundTaskManager(zConf); + } + } + + public synchronized NoteBackgroundTaskManager loadNoteBackgroundTaskManager(String launcherPlugin, + String pluginClass) throws IOException { + + if (cachedBackgroundTaskManagers.containsKey(pluginClass)) { + return cachedBackgroundTaskManagers.get(pluginClass); + } + LOGGER.info("Loading Interpreter Launcher Plugin: " + launcherPlugin); + URLClassLoader pluginClassLoader = getPluginClassLoader(pluginsDir, "Launcher", launcherPlugin); + NoteBackgroundTaskManager taskManager = null; + try { + taskManager = (NoteBackgroundTaskManager) (Class.forName(pluginClass, true, pluginClassLoader)) + .getConstructor(ZeppelinConfiguration.class) + .newInstance(zConf); + } catch (InstantiationException | IllegalAccessException | ClassNotFoundException + | NoSuchMethodException | InvocationTargetException e) { + LOGGER.warn("Fail to instantiate Launcher from plugin classpath:" + launcherPlugin, e); + } + + if (taskManager == null) { + throw new IOException("Fail to load plugin: " + launcherPlugin); + } + cachedBackgroundTaskManagers.put(pluginClass, taskManager); + return taskManager; + } + + public synchronized RestApiRouter loadNoteServingRestApiRouter() throws IOException { + if (zConf.getRunMode() == ZeppelinConfiguration.RUN_MODE.K8S) { + String backgroundTaskType = System.getenv("ZEPPELIN_BACKGROUND_TYPE"); + if ("test".equalsIgnoreCase(backgroundTaskType)) { + // if NoteTestTask shouldn't add route during test. + return new DummyRestApiRouter(); + } + + /** + * For now, class name is hardcoded here. + * Later, we can make it configurable if necessary. + */ + return loadNoteServingRestApiRouter( + "K8sStandardInterpreterLauncher", + "org.apache.zeppelin.serving.K8sRestApiRouter" + ); + } else { + return new DummyRestApiRouter(); + } + } + + public synchronized RestApiRouter loadNoteServingRestApiRouter(String launcherPlugin, + String pluginClass) throws IOException { + + if (cachedRestApiRouter.containsKey(launcherPlugin)) { + return cachedRestApiRouter.get(launcherPlugin); + } + LOGGER.info("Loading Interpreter Launcher Plugin: " + launcherPlugin); + URLClassLoader pluginClassLoader = getPluginClassLoader(pluginsDir, "Launcher", launcherPlugin); + RestApiRouter apiRouter = null; + try { + apiRouter = (RestApiRouter) (Class.forName(pluginClass, true, pluginClassLoader)) + .getConstructor(ZeppelinConfiguration.class) + .newInstance(zConf); + } catch (InstantiationException | IllegalAccessException | ClassNotFoundException + | NoSuchMethodException | InvocationTargetException e) { + LOGGER.warn("Fail to instantiate Launcher from plugin classpath:" + launcherPlugin, e); + } + + if (apiRouter == null) { + throw new IOException("Fail to load plugin: " + launcherPlugin); + } + cachedRestApiRouter.put(launcherPlugin, apiRouter); + return apiRouter; + } + + public synchronized MetricStorage loadNoteServingMetricStorage() throws IOException { + String redisAddr = zConf.getInterpreterMetricRedisAddr(); + if (redisAddr != null) { + return new RedisMetricStorage( + redisAddr, + "", + "", + RedisMetricStorage.DEFAULT_METRIC_EXPIRE_SEC); + } + return null; + } + private URLClassLoader getPluginClassLoader(String pluginsDir, String pluginType, String pluginName) throws IOException { @@ -190,4 +311,5 @@ private URLClassLoader getPluginClassLoader(String pluginsDir, public static void reset() { instance = null; } + } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/serving/DummyNoteBackgroundTaskManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/serving/DummyNoteBackgroundTaskManager.java new file mode 100644 index 00000000000..f7780b7f275 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/serving/DummyNoteBackgroundTaskManager.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.serving; + +import org.apache.zeppelin.background.NoteBackgroundTask; +import org.apache.zeppelin.background.NoteBackgroundTaskManager; +import org.apache.zeppelin.background.TaskContext; +import org.apache.zeppelin.background.TaskContextStorage; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Dummy implementation of note serving/test task manager. + * Used when zeppelin is running in an environment where serving/test is not supported. + */ +public class DummyNoteBackgroundTaskManager extends NoteBackgroundTaskManager { + private static Logger LOGGER = LoggerFactory.getLogger(DummyNoteBackgroundTaskManager.class); + + public DummyNoteBackgroundTaskManager(ZeppelinConfiguration zConf) { + super(zConf); + } + + @Override + protected TaskContextStorage getTaskContextStorage() { + LOGGER.info("No note serving task manager is configured"); + return null; + } + + @Override + protected NoteBackgroundTask createOrGetBackgroundTask(TaskContext taskContext) { + LOGGER.info("No note serving task manager is configured"); + return null; + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/serving/DummyRestApiRouter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/serving/DummyRestApiRouter.java new file mode 100644 index 00000000000..5dd901507d9 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/serving/DummyRestApiRouter.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.serving; + +import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Dummy rest api router for serving. + * Used when zeppelin is running in an environment where serving is not supported. + */ +public class DummyRestApiRouter implements RestApiRouter { + private static Logger LOGGER = LoggerFactory.getLogger(DummyRestApiRouter.class); + + @Override + public void addRoute(String noteId, String revId, String dnsname, String hostname, int port, String endpoint) throws IOException { + LOGGER.info("No api router is configured"); + } + + @Override + public void removeRoute(String noteId, String revId) throws IOException { + LOGGER.info("No api router is configured"); + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/serving/RestApiRouter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/serving/RestApiRouter.java new file mode 100644 index 00000000000..e1476f1309c --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/serving/RestApiRouter.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.serving; + +import java.io.IOException; + +/** + * RestApiRouter configure routing table for note serving. + */ +public interface RestApiRouter { + void addRoute(String noteId, String revId, String dnsName, String hostname, int port, String endpoint) throws IOException; + void removeRoute(String noteId, String revId) throws IOException; +} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/background/FileSystemTaskContextStorageTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/background/FileSystemTaskContextStorageTest.java new file mode 100644 index 00000000000..830c7051927 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/background/FileSystemTaskContextStorageTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.background; + +import com.google.common.io.Files; +import java.io.File; +import java.io.IOException; +import java.util.List; +import org.apache.commons.io.FileUtils; +import org.apache.zeppelin.background.FileSystemTaskContextStorage; +import org.apache.zeppelin.background.TaskContext; +import org.apache.zeppelin.notebook.Note; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * FileSystemTaskContextStorageTest. + */ +public class FileSystemTaskContextStorageTest { + private File contextRoot; + + @Before + public void setUp() { + contextRoot = Files.createTempDir(); + } + + @After + public void tearDown() throws IOException { + FileUtils.deleteDirectory(contextRoot); + } + + @Test + public void testSaveAndLoad() throws IOException { + // given + FileSystemTaskContextStorage contextStorage = new FileSystemTaskContextStorage(contextRoot.getAbsolutePath()); + Note note = new Note(); + TaskContext context = new TaskContext(note, "rev1"); + + // when + contextStorage.save(context); + TaskContext loaded = contextStorage.load(context.getId()); + + // then + assertEquals(loaded.getId(), context.getId()); + assertEquals(loaded.getRevId(), context.getRevId()); + assertEquals(loaded.getNote(), context.getNote()); + } + + @Test + public void testSaveOverwrite() throws IOException { + // given + FileSystemTaskContextStorage contextStorage = new FileSystemTaskContextStorage(contextRoot.getAbsolutePath()); + Note note = new Note(); + contextStorage.save(new TaskContext(note, "rev1")); + + // when + note.setName("hello"); + contextStorage.save(new TaskContext(note, "rev1")); + TaskContext loaded = contextStorage.load(TaskContext.getTaskId(note.getId(), "rev1")); + + // then + assertEquals("hello", loaded.getNote().getName()); + } + + @Test + public void testDelete() throws IOException { + // given + FileSystemTaskContextStorage contextStorage = new FileSystemTaskContextStorage(contextRoot.getAbsolutePath()); + Note note = new Note(); + TaskContext context = new TaskContext(note, "rev1"); + contextStorage.save(context); + assertEquals(1, contextStorage.list().size()); + + // when + contextStorage.delete(context.getId()); + + // then + assertEquals(0, contextStorage.list().size()); + } + + @Test + public void testList() throws IOException { + // given + FileSystemTaskContextStorage contextStorage = new FileSystemTaskContextStorage(contextRoot.getAbsolutePath()); + Note note = new Note(); + TaskContext context = new TaskContext(note, "rev1"); + contextStorage.save(context); + + // when + List list = contextStorage.list(); + + // then + assertEquals(1, list.size()); + assertEquals("rev1", list.get(0).getRevId()); + } +}