diff --git a/.gitignore b/.gitignore index 4086a4bb43e..618075d1de7 100644 --- a/.gitignore +++ b/.gitignore @@ -94,6 +94,7 @@ Thumbs.db **/.settings/ .project .settings/ +.factorypath # intelliJ IDEA project files .idea/ diff --git a/docs/setup/operation/configuration.md b/docs/setup/operation/configuration.md index 65129fffec1..3c509b00b54 100644 --- a/docs/setup/operation/configuration.md +++ b/docs/setup/operation/configuration.md @@ -418,7 +418,13 @@ If both are defined, then the **environment variables** will take priority.
zeppelin.k8s.template.dir
k8s Kubernetes yaml spec files - + + +
ZEPPELIN_K8S_SERVICE_NAME
+
zeppelin.k8s.service.name
+ zeppelin-server + Name of the Zeppelin server service resources + diff --git a/k8s/interpreter/100-interpreter-spec.yaml b/k8s/interpreter/100-interpreter-spec.yaml index c857ff2cb80..c331a9969aa 100644 --- a/k8s/interpreter/100-interpreter-spec.yaml +++ b/k8s/interpreter/100-interpreter-spec.yaml @@ -43,7 +43,7 @@ spec: containers: - name: {{zeppelin.k8s.interpreter.container.name}} image: {{zeppelin.k8s.interpreter.container.image}} - command: ["sh", "-c", "$(ZEPPELIN_HOME)/bin/interpreter.sh -d $(ZEPPELIN_HOME)/interpreter/{{zeppelin.k8s.interpreter.group.name}} -r {{zeppelin.k8s.interpreter.rpc.portRange}} -c {{zeppelin.k8s.server.rpc.host}} -p {{zeppelin.k8s.server.rpc.portRange}} -i {{zeppelin.k8s.interpreter.group.id}} -l {{zeppelin.k8s.interpreter.localRepo}} -g {{zeppelin.k8s.interpreter.setting.name}}"] + command: ["sh", "-c", "$(ZEPPELIN_HOME)/bin/interpreter.sh -d $(ZEPPELIN_HOME)/interpreter/{{zeppelin.k8s.interpreter.group.name}} -r {{zeppelin.k8s.interpreter.rpc.portRange}} -c {{zeppelin.k8s.server.rpc.service}} -p {{zeppelin.k8s.server.rpc.portRange}} -i {{zeppelin.k8s.interpreter.group.id}} -l {{zeppelin.k8s.interpreter.localRepo}} -g {{zeppelin.k8s.interpreter.setting.name}}"] lifecycle: preStop: exec: diff --git a/k8s/zeppelin-server.yaml b/k8s/zeppelin-server.yaml index 08b3c03cdd1..aefd07c0c18 100644 --- a/k8s/zeppelin-server.yaml +++ b/k8s/zeppelin-server.yaml @@ -186,7 +186,7 @@ spec: kind: Service apiVersion: v1 metadata: - name: zeppelin-server # keep Service name the same to Pod name. + name: zeppelin-server spec: ports: - name: http diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 6e667cfe696..8ce9ac377ac 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -836,6 +836,10 @@ public String getK8sTemplatesDir() { return getRelativeDir(ConfVars.ZEPPELIN_K8S_TEMPLATE_DIR); } + public String getK8sServiceName() { + return getString(ConfVars.ZEPPELIN_K8S_SERVICE_NAME); + } + public String getDockerContainerImage() { return getString(ConfVars.ZEPPELIN_DOCKER_CONTAINER_IMAGE); } @@ -1019,6 +1023,7 @@ public enum ConfVars { ZEPPELIN_K8S_CONTAINER_IMAGE("zeppelin.k8s.container.image", "apache/zeppelin:" + Util.getVersion()), ZEPPELIN_K8S_SPARK_CONTAINER_IMAGE("zeppelin.k8s.spark.container.image", "apache/spark:latest"), ZEPPELIN_K8S_TEMPLATE_DIR("zeppelin.k8s.template.dir", "k8s"), + ZEPPELIN_K8S_SERVICE_NAME("zeppelin.k8s.service.name", "zeppelin-server"), ZEPPELIN_DOCKER_CONTAINER_IMAGE("zeppelin.docker.container.image", "apache/zeppelin:" + Util.getVersion()), diff --git a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sRemoteInterpreterProcess.java b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sRemoteInterpreterProcess.java index 5f872cc4762..c167ae75372 100644 --- a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sRemoteInterpreterProcess.java +++ b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sRemoteInterpreterProcess.java @@ -14,7 +14,6 @@ import org.apache.commons.exec.ExecuteWatchdog; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; import org.slf4j.Logger; @@ -31,7 +30,7 @@ public class K8sRemoteInterpreterProcess extends RemoteInterpreterProcess { private final String containerImage; private final Properties properties; private final Map envs; - private final String zeppelinServiceHost; + private final String zeppelinService; private final String zeppelinServiceRpcPort; private final Gson gson = new Gson(); @@ -55,7 +54,7 @@ public K8sRemoteInterpreterProcess( String interpreterSettingName, Properties properties, Map envs, - String zeppelinServiceHost, + String zeppelinService, String zeppelinServiceRpcPort, boolean portForward, String sparkImage, @@ -71,7 +70,7 @@ public K8sRemoteInterpreterProcess( this.interpreterSettingName = interpreterSettingName; this.properties = properties; this.envs = new HashMap<>(envs); - this.zeppelinServiceHost = zeppelinServiceHost; + this.zeppelinService = zeppelinService; this.zeppelinServiceRpcPort = zeppelinServiceRpcPort; this.portForward = portForward; this.sparkImage = sparkImage; @@ -266,7 +265,7 @@ Properties getTemplateBindings() { k8sProperties.put("zeppelin.k8s.interpreter.setting.name", interpreterSettingName); k8sProperties.put("zeppelin.k8s.interpreter.localRepo", "/tmp/local-repo"); k8sProperties.put("zeppelin.k8s.interpreter.rpc.portRange", String.format("%d:%d", getPort(), getPort())); - k8sProperties.put("zeppelin.k8s.server.rpc.host", zeppelinServiceHost); + k8sProperties.put("zeppelin.k8s.server.rpc.service", zeppelinService); k8sProperties.put("zeppelin.k8s.server.rpc.portRange", zeppelinServiceRpcPort); if (ownerUID() != null && ownerName() != null) { k8sProperties.put("zeppelin.k8s.server.uid", ownerUID()); @@ -287,7 +286,7 @@ Properties getTemplateBindings() { // configure interpreter property "zeppelin.spark.uiWebUrl" if not defined, to enable spark ui through reverse proxy String webUrl = (String) properties.get("zeppelin.spark.uiWebUrl"); - if (webUrl == null || webUrl.trim().isEmpty()) { + if (StringUtils.isBlank(webUrl)) { webUrl = "//{{PORT}}-{{SERVICE_NAME}}.{{SERVICE_DOMAIN}}"; } properties.put("zeppelin.spark.uiWebUrl", diff --git a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sStandardInterpreterLauncher.java b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sStandardInterpreterLauncher.java index 80bd4aa4274..d4b03da5570 100644 --- a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sStandardInterpreterLauncher.java +++ b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sStandardInterpreterLauncher.java @@ -98,14 +98,14 @@ String getHostname() { } /** - * get Zeppelin server host dns. - * return ..svc + * get Zeppelin service. + * return ..svc * @throws IOException */ - private String getZeppelinServiceHost() throws IOException { + private String getZeppelinService() throws IOException { if (isRunningOnKubernetes()) { return String.format("%s.%s.svc", - getHostname(), // service name and pod name should be the same + zConf.getK8sServiceName(), getNamespace()); } else { return context.getZeppelinServerHost(); @@ -154,7 +154,7 @@ public InterpreterClient launch(InterpreterLaunchContext context) throws IOExcep context.getInterpreterSettingName(), properties, buildEnvFromProperties(context), - getZeppelinServiceHost(), + getZeppelinService(), getZeppelinServiceRpcPort(), zConf.getK8sPortForward(), zConf.getK8sSparkContainerImage(), diff --git a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/Kubectl.java b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/Kubectl.java index 39e7e920f27..a0ceb0a012d 100644 --- a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/Kubectl.java +++ b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/Kubectl.java @@ -25,6 +25,7 @@ import org.apache.commons.io.IOUtils; import java.io.*; +import java.nio.charset.StandardCharsets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -106,7 +107,7 @@ String execAndGet(String [] args) throws IOException { @VisibleForTesting String execAndGet(String [] args, String stdin) throws IOException { - InputStream ins = IOUtils.toInputStream(stdin); + InputStream ins = IOUtils.toInputStream(stdin, StandardCharsets.UTF_8); ByteArrayOutputStream stdout = new ByteArrayOutputStream(); ByteArrayOutputStream stderr = new ByteArrayOutputStream(); ArrayList argsToOverride = new ArrayList<>(Arrays.asList(args)); diff --git a/zeppelin-plugins/launcher/k8s-standard/src/test/java/org/apache/zeppelin/interpreter/launcher/K8sRemoteInterpreterProcessTest.java b/zeppelin-plugins/launcher/k8s-standard/src/test/java/org/apache/zeppelin/interpreter/launcher/K8sRemoteInterpreterProcessTest.java index bc0489ead31..52c562186f3 100644 --- a/zeppelin-plugins/launcher/k8s-standard/src/test/java/org/apache/zeppelin/interpreter/launcher/K8sRemoteInterpreterProcessTest.java +++ b/zeppelin-plugins/launcher/k8s-standard/src/test/java/org/apache/zeppelin/interpreter/launcher/K8sRemoteInterpreterProcessTest.java @@ -115,7 +115,7 @@ public void testGetTemplateBindings() throws IOException { "shell", properties, envs, - "zeppelin.server.hostname", + "zeppelin.server.service", "12320", false, "spark-container:1.0", @@ -135,7 +135,7 @@ public void testGetTemplateBindings() throws IOException { assertEquals("shell", p.get("zeppelin.k8s.interpreter.setting.name")); assertEquals(true , p.containsKey("zeppelin.k8s.interpreter.localRepo")); assertEquals("12321:12321" , p.get("zeppelin.k8s.interpreter.rpc.portRange")); - assertEquals("zeppelin.server.hostname" , p.get("zeppelin.k8s.server.rpc.host")); + assertEquals("zeppelin.server.service" , p.get("zeppelin.k8s.server.rpc.service")); assertEquals("12320" , p.get("zeppelin.k8s.server.rpc.portRange")); assertEquals("v1", p.get("my.key1")); assertEquals("V1", envs.get("MY_ENV1")); @@ -168,7 +168,7 @@ public void testGetTemplateBindingsForSpark() throws IOException { "myspark", properties, envs, - "zeppelin.server.hostname", + "zeppelin.server.service", "12320", false, "spark-container:1.0", @@ -221,7 +221,7 @@ public void testGetTemplateBindingsForSparkWithProxyUser() throws IOException { "myspark", properties, envs, - "zeppelin.server.hostname", + "zeppelin.server.service", "12320", false, "spark-container:1.0", @@ -273,7 +273,7 @@ public void testGetTemplateBindingsForSparkWithProxyUserAnonymous() throws IOExc "myspark", properties, envs, - "zeppelin.server.hostname", + "zeppelin.server.service", "12320", false, "spark-container:1.0", @@ -314,7 +314,7 @@ public void testSparkUiWebUrlTemplate() { "myspark", properties, envs, - "zeppelin.server.hostname", + "zeppelin.server.service", "12320", false, "spark-container:1.0", diff --git a/zeppelin-plugins/launcher/k8s-standard/src/test/java/org/apache/zeppelin/interpreter/launcher/K8sStandardInterpreterLauncherTest.java b/zeppelin-plugins/launcher/k8s-standard/src/test/java/org/apache/zeppelin/interpreter/launcher/K8sStandardInterpreterLauncherTest.java index b25373dd5ac..c580a4357d8 100644 --- a/zeppelin-plugins/launcher/k8s-standard/src/test/java/org/apache/zeppelin/interpreter/launcher/K8sStandardInterpreterLauncherTest.java +++ b/zeppelin-plugins/launcher/k8s-standard/src/test/java/org/apache/zeppelin/interpreter/launcher/K8sStandardInterpreterLauncherTest.java @@ -54,8 +54,6 @@ public void testK8sLauncher() throws IOException { Properties properties = new Properties(); properties.setProperty("ENV_1", "VALUE_1"); properties.setProperty("property_1", "value_1"); - properties.setProperty("CALLBACK_HOST", "zeppelin-server.default.svc"); - properties.setProperty("CALLBACK_PORT", "12320"); InterpreterOption option = new InterpreterOption(); option.setUserImpersonate(true); InterpreterLaunchContext context = new InterpreterLaunchContext( @@ -88,8 +86,6 @@ public void testK8sLauncherWithSparkAndUserImpersonate() throws IOException { Properties properties = new Properties(); properties.setProperty("ENV_1", "VALUE_1"); properties.setProperty("property_1", "value_1"); - properties.setProperty("CALLBACK_HOST", "zeppelin-server.default.svc"); - properties.setProperty("CALLBACK_PORT", "12320"); properties.setProperty("SERVICE_DOMAIN", "example.com"); properties.setProperty("zeppelin.interpreter.connect.timeout", "60"); InterpreterOption option = new InterpreterOption(); @@ -131,8 +127,6 @@ public void testK8sLauncherWithSparkAndWithoutUserImpersonate() throws IOExcepti Properties properties = new Properties(); properties.setProperty("ENV_1", "VALUE_1"); properties.setProperty("property_1", "value_1"); - properties.setProperty("CALLBACK_HOST", "zeppelin-server.default.svc"); - properties.setProperty("CALLBACK_PORT", "12320"); properties.setProperty("SERVICE_DOMAIN", "example.com"); properties.setProperty("zeppelin.interpreter.connect.timeout", "60"); InterpreterOption option = new InterpreterOption();