diff --git a/.travis.yml b/.travis.yml index 11c3d91c82bd0..adc0cae118e10 100644 --- a/.travis.yml +++ b/.travis.yml @@ -100,7 +100,7 @@ script: - | if [[ -v PRODUCT_TESTS_SPECIFIC_ENVIRONMENT ]]; then presto-product-tests/bin/run_on_docker.sh \ - multinode-tls -g smoke,cli,group-by,join,tls + multinode-tls-kerberos -g smoke,cli,group-by,join,tls -x stats_client fi - | if [[ -v HIVE_TESTS ]]; then diff --git a/presto-docs/src/main/sphinx/security/internal-communication.rst b/presto-docs/src/main/sphinx/security/internal-communication.rst index e3186a45cc426..2a53894854f15 100644 --- a/presto-docs/src/main/sphinx/security/internal-communication.rst +++ b/presto-docs/src/main/sphinx/security/internal-communication.rst @@ -114,6 +114,20 @@ To enable SSL/TLS for Presto internal communication, do the following: internal-communication.https.keystore.key= +Internal SSL/TLS communication with Kerberos +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If the :doc:`Kerberos` authentication is enabled, specify valid Kerberos +credentials for the internal communication, in addition to the SSL/TLS properties. + + .. code-block:: none + + internal-communication.authentication.kerberos.enabled=true + internal-communication.authentication.krb5.principal= + internal-communication.authentication.krb5.service-name= + internal-communication.authentication.krb5.config= + internal-communication.authentication.krb5.keytab= + Performance with SSL/TLS enabled -------------------------------- diff --git a/presto-main/src/main/java/com/facebook/presto/server/InternalCommunicationConfig.java b/presto-main/src/main/java/com/facebook/presto/server/InternalCommunicationConfig.java index c471f2cd64454..454258e66f210 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/InternalCommunicationConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/server/InternalCommunicationConfig.java @@ -15,12 +15,22 @@ import io.airlift.configuration.Config; +import java.io.File; + public class InternalCommunicationConfig { private boolean httpsRequired; private String keyStorePath; private String keyStorePassword; + private String kerberosPrincipal; + private String kerberosServiceName; + private boolean kerberosEnabled; + private File kerberosKeytab; + private File kerberosConfig; + private boolean kerberosUseCanonicalHostname = true; + private File kerberosCredentialCache; + public boolean isHttpsRequired() { return httpsRequired; @@ -56,4 +66,88 @@ public InternalCommunicationConfig setKeyStorePassword(String keyStorePassword) this.keyStorePassword = keyStorePassword; return this; } + + public boolean isKerberosEnabled() + { + return kerberosEnabled; + } + + @Config("internal-communication.authentication.kerberos.enabled") + public InternalCommunicationConfig setKerberosEnabled(boolean kerberosEnabled) + { + this.kerberosEnabled = kerberosEnabled; + return this; + } + + public String getKerberosPrincipal() + { + return kerberosPrincipal; + } + + @Config("internal-communication.authentication.krb5.principal") + public InternalCommunicationConfig setKerberosPrincipal(String kerberosPrincipal) + { + this.kerberosPrincipal = kerberosPrincipal; + return this; + } + + public String getKerberosServiceName() + { + return kerberosServiceName; + } + + @Config("internal-communication.authentication.krb5.service-name") + public InternalCommunicationConfig setKerberosServiceName(String kerberosServiceName) + { + this.kerberosServiceName = kerberosServiceName; + return this; + } + + public File getKerberosKeytab() + { + return kerberosKeytab; + } + + @Config("internal-communication.authentication.krb5.keytab") + public InternalCommunicationConfig setKerberosKeytab(File kerberosKeytab) + { + this.kerberosKeytab = kerberosKeytab; + return this; + } + + public File getKerberosConfig() + { + return kerberosConfig; + } + + @Config("internal-communication.authentication.krb5.config") + public InternalCommunicationConfig setKerberosConfig(File kerberosConfig) + { + this.kerberosConfig = kerberosConfig; + return this; + } + + public boolean isKerberosUseCanonicalHostname() + { + return kerberosUseCanonicalHostname; + } + + @Config("internal-communication.authentication.krb5.use-canonical-hostname") + public InternalCommunicationConfig setKerberosUseCanonicalHostname(boolean kerberosUseCanonicalHostname) + { + this.kerberosUseCanonicalHostname = kerberosUseCanonicalHostname; + return this; + } + + public File getKerberosCredentialCache() + { + return kerberosCredentialCache; + } + + @Config("internal-communication.authentication.krb5.credential-cache") + public InternalCommunicationConfig setKerberosCredentialCache(File kerberosCredentialCache) + { + this.kerberosCredentialCache = kerberosCredentialCache; + return this; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java index 4ca3eedb67f64..d3a68840b7872 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java @@ -125,6 +125,7 @@ import com.facebook.presto.type.TypeDeserializer; import com.facebook.presto.type.TypeRegistry; import com.facebook.presto.util.FinalizerService; +import com.facebook.presto.util.KerberosPrincipal; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.inject.Binder; @@ -136,6 +137,7 @@ import io.airlift.configuration.AbstractConfigurationAwareModule; import io.airlift.discovery.client.ServiceDescriptor; import io.airlift.http.client.HttpClientConfig; +import io.airlift.http.client.spnego.KerberosConfig; import io.airlift.slice.Slice; import io.airlift.stats.PauseMeter; import io.airlift.units.DataSize; @@ -145,6 +147,7 @@ import javax.inject.Inject; import javax.inject.Singleton; +import java.io.File; import java.util.List; import java.util.Optional; import java.util.Set; @@ -153,6 +156,7 @@ import static com.facebook.presto.execution.scheduler.NodeSchedulerConfig.NetworkTopologyType.FLAT; import static com.facebook.presto.execution.scheduler.NodeSchedulerConfig.NetworkTopologyType.LEGACY; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Strings.nullToEmpty; import static com.google.common.reflect.Reflection.newProxy; @@ -209,6 +213,28 @@ protected void setup(Binder binder) config.setKeyStorePassword(internalCommunicationConfig.getKeyStorePassword()); }); + if (internalCommunicationConfig.isKerberosEnabled()) { + File kerberosConfig = internalCommunicationConfig.getKerberosConfig(); + File kerberosKeytab = internalCommunicationConfig.getKerberosKeytab(); + KerberosPrincipal principal = KerberosPrincipal.valueOf(internalCommunicationConfig.getKerberosPrincipal()); + String kerberosServiceName = internalCommunicationConfig.getKerberosServiceName(); + checkArgument(kerberosConfig != null, "kerberos config must be set"); + checkArgument(kerberosKeytab != null, "kerberos keytab must be set"); + checkArgument(kerberosServiceName != null, "kerberos service name must be set"); + + configBinder(binder).bindConfigGlobalDefaults(KerberosConfig.class, config -> { + config.setConfig(kerberosConfig); + config.setKeytab(kerberosKeytab); + config.setUseCanonicalHostname(internalCommunicationConfig.isKerberosUseCanonicalHostname()); + config.setCredentialCache(internalCommunicationConfig.getKerberosCredentialCache()); + }); + configBinder(binder).bindConfigGlobalDefaults(HttpClientConfig.class, config -> { + config.setAuthenticationEnabled(true); + config.setKerberosPrincipal(principal.substituteHostnamePlaceholder().toString()); + config.setKerberosRemoteServiceName(kerberosServiceName); + }); + } + configBinder(binder).bindConfig(FeaturesConfig.class); binder.bind(SqlParser.class).in(Scopes.SINGLETON); diff --git a/presto-main/src/main/java/com/facebook/presto/util/KerberosPrincipal.java b/presto-main/src/main/java/com/facebook/presto/util/KerberosPrincipal.java new file mode 100644 index 0000000000000..305a10ec930bb --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/util/KerberosPrincipal.java @@ -0,0 +1,201 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.util; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public class KerberosPrincipal +{ + private static final char REALM_SEPARATOR = '@'; + private static final char PARTS_SEPARATOR = '/'; + private static final char ESCAPE_CHARACTER = '\\'; + + private static final String HOSTNAME_PLACEHOLDER = "_HOST"; + + private final String username; + private final Optional hostname; + private final Optional realm; + + public static KerberosPrincipal valueOf(String value) + { + requireNonNull(value, "value is null"); + + int hostnameSeparatorIndex = -1; + int realmSeparatorIndex = -1; + + boolean escape = false; + for (int index = 0; index < value.length(); index++) { + char c = value.charAt(index); + switch (c) { + case ESCAPE_CHARACTER: + escape = !escape; + break; + case PARTS_SEPARATOR: + if (escape) { + escape = false; + } + else { + hostnameSeparatorIndex = index; + } + break; + case REALM_SEPARATOR: + if (escape) { + escape = false; + } + else { + realmSeparatorIndex = index; + } + break; + default: + escape = false; + break; + } + } + + if (hostnameSeparatorIndex >= 0 && realmSeparatorIndex >= 0 && hostnameSeparatorIndex > realmSeparatorIndex) { + throw invalidKerberosPrincipal(value); + } + + String usernameAndHostname; + Optional realm; + if (realmSeparatorIndex >= 0) { + // usernameAndHostname cannot be empty + if (realmSeparatorIndex == 0) { + throw invalidKerberosPrincipal(value); + } + + // realm cannot be empty + if (realmSeparatorIndex == value.length() - 1) { + throw invalidKerberosPrincipal(value); + } + + usernameAndHostname = value.substring(0, realmSeparatorIndex); + realm = Optional.of(value.substring(realmSeparatorIndex + 1, value.length())); + } + else { + usernameAndHostname = value; + realm = Optional.empty(); + } + + String username; + Optional hostname; + if (hostnameSeparatorIndex >= 0) { + // username cannot be empty + if (hostnameSeparatorIndex == 0) { + throw invalidKerberosPrincipal(value); + } + + // hostname cannot be empty + if (hostnameSeparatorIndex == usernameAndHostname.length() - 1) { + throw invalidKerberosPrincipal(value); + } + + username = usernameAndHostname.substring(0, hostnameSeparatorIndex); + hostname = Optional.of(usernameAndHostname.substring(hostnameSeparatorIndex + 1, usernameAndHostname.length())); + } + else { + username = usernameAndHostname; + hostname = Optional.empty(); + } + + return new KerberosPrincipal(username, hostname, realm); + } + + private static IllegalArgumentException invalidKerberosPrincipal(String value) + { + return new IllegalArgumentException("Invalid Kerberos principal: " + value); + } + + public KerberosPrincipal(String userName, Optional hostName, Optional realm) + { + this.username = requireNonNull(userName, "username is null"); + this.hostname = requireNonNull(hostName, "hostname is null"); + this.realm = requireNonNull(realm, "realm is null"); + } + + public String getUserName() + { + return username; + } + + public Optional getHostName() + { + return hostname; + } + + public Optional getRealm() + { + return realm; + } + + public KerberosPrincipal substituteHostnamePlaceholder() + { + return substituteHostnamePlaceholder(getLocalCanonicalHostName()); + } + + public KerberosPrincipal substituteHostnamePlaceholder(String hostname) + { + if (this.hostname.isPresent() && this.hostname.get().equals(HOSTNAME_PLACEHOLDER)) { + return new KerberosPrincipal(username, Optional.of(hostname), realm); + } + return this; + } + + private static String getLocalCanonicalHostName() + { + try { + return InetAddress.getLocalHost().getCanonicalHostName().toLowerCase(Locale.US); + } + catch (UnknownHostException e) { + throw new RuntimeException(e); + } + } + + @Override + public String toString() + { + StringBuilder builder = new StringBuilder(); + builder.append(username); + hostname.ifPresent(hostname -> builder.append(PARTS_SEPARATOR).append(hostname)); + realm.ifPresent(realm -> builder.append(REALM_SEPARATOR).append(realm)); + return builder.toString(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + KerberosPrincipal that = (KerberosPrincipal) o; + return Objects.equals(username, that.username) && + Objects.equals(hostname, that.hostname) && + Objects.equals(realm, that.realm); + } + + @Override + public int hashCode() + { + return Objects.hash(username, hostname, realm); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/util/TestKerberosPrincipal.java b/presto-main/src/test/java/com/facebook/presto/util/TestKerberosPrincipal.java new file mode 100644 index 0000000000000..695d4b91524b0 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/util/TestKerberosPrincipal.java @@ -0,0 +1,64 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.util; + +import org.testng.annotations.Test; + +import java.util.Optional; + +import static org.testng.Assert.assertEquals; + +public class TestKerberosPrincipal +{ + @Test + public void testPrincipalParsing() + throws Exception + { + assertEquals(KerberosPrincipal.valueOf("user"), new KerberosPrincipal("user", Optional.empty(), Optional.empty())); + assertEquals(KerberosPrincipal.valueOf("user@REALM.COM"), new KerberosPrincipal("user", Optional.empty(), Optional.of("REALM.COM"))); + assertEquals(KerberosPrincipal.valueOf("user/hostname@REALM.COM"), new KerberosPrincipal("user", Optional.of("hostname"), Optional.of("REALM.COM"))); + assertEquals(KerberosPrincipal.valueOf("user\\@host.com@REALM.COM"), new KerberosPrincipal("user\\@host.com", Optional.empty(), Optional.of("REALM.COM"))); + assertEquals(KerberosPrincipal.valueOf("user/_HOST@REALM.COM"), new KerberosPrincipal("user", Optional.of("_HOST"), Optional.of("REALM.COM"))); + assertEquals(KerberosPrincipal.valueOf("user/part1/part2/_HOST@REALM.COM"), new KerberosPrincipal("user/part1/part2", Optional.of("_HOST"), Optional.of("REALM.COM"))); + assertEquals(KerberosPrincipal.valueOf("user\\/part1\\/part2\\/_HOST@REALM.COM"), new KerberosPrincipal("user\\/part1\\/part2\\/_HOST", Optional.empty(), Optional.of("REALM.COM"))); + assertEquals(KerberosPrincipal.valueOf("user\\/part1\\/part2\\/_HOST\\@REALM.COM"), new KerberosPrincipal("user\\/part1\\/part2\\/_HOST\\@REALM.COM", Optional.empty(), Optional.empty())); + } + + @Test + public void testSubstituteHostnamePlaceholder() + throws Exception + { + assertEquals( + KerberosPrincipal.valueOf("user").substituteHostnamePlaceholder("localhost"), + new KerberosPrincipal("user", Optional.empty(), Optional.empty())); + assertEquals( + KerberosPrincipal.valueOf("user/_HOST").substituteHostnamePlaceholder("localhost"), + new KerberosPrincipal("user", Optional.of("localhost"), Optional.empty())); + assertEquals( + KerberosPrincipal.valueOf("user/_HOST@REALM.COM").substituteHostnamePlaceholder("localhost"), + new KerberosPrincipal("user", Optional.of("localhost"), Optional.of("REALM.COM"))); + assertEquals( + KerberosPrincipal.valueOf("user@REALM.COM").substituteHostnamePlaceholder("localhost"), + new KerberosPrincipal("user", Optional.empty(), Optional.of("REALM.COM"))); + } + + @Test + public void testToString() + throws Exception + { + assertEquals(new KerberosPrincipal("user", Optional.empty(), Optional.empty()).toString(), "user"); + assertEquals(new KerberosPrincipal("user", Optional.of("host"), Optional.empty()).toString(), "user/host"); + assertEquals(new KerberosPrincipal("user", Optional.of("host"), Optional.of("REALM.COM")).toString(), "user/host@REALM.COM"); + } +} diff --git a/presto-product-tests/bin/run_on_docker.sh b/presto-product-tests/bin/run_on_docker.sh index e14fd30da603e..0342154382787 100755 --- a/presto-product-tests/bin/run_on_docker.sh +++ b/presto-product-tests/bin/run_on_docker.sh @@ -182,13 +182,13 @@ shift 1 PRESTO_SERVICES="presto-master" if [[ "$ENVIRONMENT" == "multinode" ]]; then PRESTO_SERVICES="${PRESTO_SERVICES} presto-worker" -elif [[ "$ENVIRONMENT" == "multinode-tls" ]]; then +elif [[ "$ENVIRONMENT" == "multinode-tls-kerberos" ]]; then PRESTO_SERVICES="${PRESTO_SERVICES} presto-worker-1 presto-worker-2" fi CLI_ARGUMENTS="--server presto-master:8080" -if [[ "$ENVIRONMENT" == "multinode-tls" ]]; then - CLI_ARGUMENTS="--server https://presto-master.docker.cluster:7778 --keystore-path /docker/volumes/conf/presto/etc/docker.cluster.jks --keystore-password 123456" +if [[ "$ENVIRONMENT" == "multinode-tls-kerberos" ]]; then + CLI_ARGUMENTS="--server https://presto-master.docker.cluster:7778 --keystore-path /docker/volumes/conf/presto/etc/docker.cluster.jks --keystore-password 123456 --enable-authentication --krb5-config-path /etc/krb5.conf --krb5-principal presto-client/presto-master.docker.cluster@LABS.TERADATA.COM --krb5-keytab-path /etc/presto/conf/presto-client.keytab --krb5-remote-service-name presto-server --krb5-disable-remote-service-hostname-canonicalization" fi # check docker and docker compose installation diff --git a/presto-product-tests/conf/docker/multinode-tls/compose.sh b/presto-product-tests/conf/docker/multinode-tls-kerberos/compose.sh similarity index 85% rename from presto-product-tests/conf/docker/multinode-tls/compose.sh rename to presto-product-tests/conf/docker/multinode-tls-kerberos/compose.sh index 94b6c88939b11..b6cbf470eeca1 100755 --- a/presto-product-tests/conf/docker/multinode-tls/compose.sh +++ b/presto-product-tests/conf/docker/multinode-tls-kerberos/compose.sh @@ -6,6 +6,7 @@ source ${SCRIPT_DIRECTORY}/../common/compose-commons.sh docker-compose \ -f ${SCRIPT_DIRECTORY}/../common/standard.yml \ +-f ${SCRIPT_DIRECTORY}/../common/kerberos.yml \ -f ${SCRIPT_DIRECTORY}/../common/jdbc_db.yml \ -f ${SCRIPT_DIRECTORY}/docker-compose.yml \ "$@" diff --git a/presto-product-tests/conf/docker/multinode-tls/docker-compose.yml b/presto-product-tests/conf/docker/multinode-tls-kerberos/docker-compose.yml similarity index 62% rename from presto-product-tests/conf/docker/multinode-tls/docker-compose.yml rename to presto-product-tests/conf/docker/multinode-tls-kerberos/docker-compose.yml index 21c2ceb457310..eff9795eb791e 100644 --- a/presto-product-tests/conf/docker/multinode-tls/docker-compose.yml +++ b/presto-product-tests/conf/docker/multinode-tls-kerberos/docker-compose.yml @@ -4,13 +4,16 @@ services: presto-master: domainname: docker.cluster hostname: presto-master - command: /docker/volumes/conf/docker/files/presto-launcher-wrapper.sh multinode-tls-master run + image: 'teradatalabs/cdh5-hive-kerberized:${DOCKER_IMAGES_VERSION}' + command: /docker/volumes/conf/docker/files/presto-launcher-wrapper.sh multinode-tls-kerberos-master run ports: - '7778:7778' networks: default: aliases: - presto-master.docker.cluster + volumes: + - ../../../conf/presto/etc/environment-specific-catalogs/singlenode-kerberos-hdfs-no-impersonation/hive.properties:/docker/volumes/conf/presto/etc/catalog/hive.properties presto-worker-1: domainname: docker.cluster @@ -18,7 +21,8 @@ services: extends: file: ../common/standard.yml service: java-8-base - command: /docker/volumes/conf/docker/files/presto-launcher-wrapper.sh multinode-tls-worker run + image: 'teradatalabs/cdh5-hive-kerberized:${DOCKER_IMAGES_VERSION}' + command: /docker/volumes/conf/docker/files/presto-launcher-wrapper.sh multinode-tls-kerberos-worker run networks: default: aliases: @@ -34,7 +38,8 @@ services: extends: file: ../common/standard.yml service: java-8-base - command: /docker/volumes/conf/docker/files/presto-launcher-wrapper.sh multinode-tls-worker run + image: 'teradatalabs/cdh5-hive-kerberized:${DOCKER_IMAGES_VERSION}' + command: /docker/volumes/conf/docker/files/presto-launcher-wrapper.sh multinode-tls-kerberos-worker run networks: default: aliases: @@ -45,5 +50,6 @@ services: - presto-master application-runner: + image: 'teradatalabs/cdh5-hive-kerberized:${DOCKER_IMAGES_VERSION}' volumes: - - ../../../conf/tempto/tempto-configuration-for-docker-tls.yaml:/docker/volumes/tempto/tempto-configuration-local.yaml + - ../../../conf/tempto/tempto-configuration-for-docker-tls-kerberos.yaml:/docker/volumes/tempto/tempto-configuration-local.yaml diff --git a/presto-product-tests/conf/presto/etc/multinode-tls-kerberos-master.properties b/presto-product-tests/conf/presto/etc/multinode-tls-kerberos-master.properties new file mode 100644 index 0000000000000..da33227bc5113 --- /dev/null +++ b/presto-product-tests/conf/presto/etc/multinode-tls-kerberos-master.properties @@ -0,0 +1,54 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# WARNING +# ^^^^^^^ +# This configuration file is for development only and should NOT be used be +# used in production. For example configuration, see the Presto documentation. +# + +node.id=will-be-overwritten +node.environment=test +node.internal-address-source=FQDN + +coordinator=true +node-scheduler.include-coordinator=true +discovery-server.enabled=true +discovery.uri=https://presto-master.docker.cluster:7778 + +query.max-memory=1GB +query.max-memory-per-node=512MB + +http-server.http.enabled=false +http-server.https.enabled=true +http-server.https.port=7778 +http-server.https.keystore.path=/docker/volumes/conf/presto/etc/docker.cluster.jks +http-server.https.keystore.key=123456 + +http.authentication.krb5.config=/etc/krb5.conf +http-server.authentication.type=KERBEROS +http.server.authentication.krb5.service-name=presto-server +http.server.authentication.krb5.keytab=/etc/presto/conf/presto-server.keytab + +internal-communication.https.required=true +internal-communication.https.keystore.path=/docker/volumes/conf/presto/etc/docker.cluster.jks +internal-communication.https.keystore.key=123456 + +internal-communication.authentication.kerberos.enabled=true +internal-communication.authentication.krb5.config=/etc/krb5.conf +internal-communication.authentication.krb5.service-name=presto-server +internal-communication.authentication.krb5.principal=presto-server/_HOST@LABS.TERADATA.COM +internal-communication.authentication.krb5.keytab=/etc/presto/conf/presto-server.keytab +internal-communication.authentication.krb5.use-canonical-hostname=false diff --git a/presto-product-tests/conf/presto/etc/multinode-tls-kerberos-worker.properties b/presto-product-tests/conf/presto/etc/multinode-tls-kerberos-worker.properties new file mode 100644 index 0000000000000..24ab8bda556aa --- /dev/null +++ b/presto-product-tests/conf/presto/etc/multinode-tls-kerberos-worker.properties @@ -0,0 +1,67 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# WARNING +# ^^^^^^^ +# This configuration file is for development only and should NOT be used be +# used in production. For example configuration, see the Presto documentation. +# + +node.id=will-be-overwritten +node.environment=test +node.internal-address-source=FQDN + +coordinator=false +discovery-server.enabled=false +discovery.uri=https://presto-master.docker.cluster:7778 + +query.max-memory=1GB +query.max-memory-per-node=512MB + +http-server.http.enabled=false +http-server.https.enabled=true +http-server.https.port=7778 +http-server.https.keystore.path=/docker/volumes/conf/presto/etc/docker.cluster.jks +http-server.https.keystore.key=123456 + +http.authentication.krb5.config=/etc/krb5.conf +http-server.authentication.type=KERBEROS +http.server.authentication.krb5.service-name=presto-server +http.server.authentication.krb5.keytab=/etc/presto/conf/presto-server.keytab + +internal-communication.https.required=true +internal-communication.https.keystore.path=/docker/volumes/conf/presto/etc/docker.cluster.jks +internal-communication.https.keystore.key=123456 + +internal-communication.authentication.kerberos.enabled=true +internal-communication.authentication.krb5.config=/etc/krb5.conf +internal-communication.authentication.krb5.service-name=presto-server +internal-communication.authentication.krb5.principal=presto-server/_HOST@LABS.TERADATA.COM +internal-communication.authentication.krb5.keytab=/etc/presto/conf/presto-server.keytab +internal-communication.authentication.krb5.use-canonical-hostname=false diff --git a/presto-product-tests/conf/presto/etc/multinode-tls-master.properties b/presto-product-tests/conf/presto/etc/multinode-tls-master.properties deleted file mode 100644 index 47a45a7e9ad3b..0000000000000 --- a/presto-product-tests/conf/presto/etc/multinode-tls-master.properties +++ /dev/null @@ -1,29 +0,0 @@ -# -# WARNING -# ^^^^^^^ -# This configuration file is for development only and should NOT be used be -# used in production. For example configuration, see the Presto documentation. -# - -node.id=will-be-overwritten -node.environment=test -node.internal-address-source=FQDN - -coordinator=true -node-scheduler.include-coordinator=true -discovery-server.enabled=true -discovery.uri=https://presto-master.docker.cluster:7778 - -query.max-memory=1GB -query.max-memory-per-node=512MB - -http-server.http.enabled=false -http-server.http.port=8080 -http-server.https.enabled=true -http-server.https.port=7778 -http-server.https.keystore.path=/docker/volumes/conf/presto/etc/docker.cluster.jks -http-server.https.keystore.key=123456 - -internal-communication.https.required=true -internal-communication.https.keystore.path=/docker/volumes/conf/presto/etc/docker.cluster.jks -internal-communication.https.keystore.key=123456 diff --git a/presto-product-tests/conf/presto/etc/multinode-tls-worker.properties b/presto-product-tests/conf/presto/etc/multinode-tls-worker.properties deleted file mode 100644 index dbe128f62bd9a..0000000000000 --- a/presto-product-tests/conf/presto/etc/multinode-tls-worker.properties +++ /dev/null @@ -1,28 +0,0 @@ -# -# WARNING -# ^^^^^^^ -# This configuration file is for development only and should NOT be used be -# used in production. For example configuration, see the Presto documentation. -# - -node.id=will-be-overwritten -node.environment=test -node.internal-address-source=FQDN - -coordinator=false -discovery-server.enabled=false -discovery.uri=https://presto-master.docker.cluster:7778 - -query.max-memory=1GB -query.max-memory-per-node=512MB - -http-server.http.enabled=false -http-server.http.port=8080 -http-server.https.enabled=true -http-server.https.port=7778 -http-server.https.keystore.path=/docker/volumes/conf/presto/etc/docker.cluster.jks -http-server.https.keystore.key=123456 - -internal-communication.https.required=true -internal-communication.https.keystore.path=/docker/volumes/conf/presto/etc/docker.cluster.jks -internal-communication.https.keystore.key=123456 diff --git a/presto-product-tests/conf/tempto/tempto-configuration-for-docker-tls-kerberos.yaml b/presto-product-tests/conf/tempto/tempto-configuration-for-docker-tls-kerberos.yaml new file mode 100644 index 0000000000000..46df8a66df87a --- /dev/null +++ b/presto-product-tests/conf/tempto/tempto-configuration-for-docker-tls-kerberos.yaml @@ -0,0 +1,46 @@ +hdfs: + username: hdfs/hadoop-master@LABS.TERADATA.COM + webhdfs: + authentication: SPNEGO + keytab: /etc/hadoop/conf/hdfs.keytab + spnego_use_canonical_hostname: false + +databases: + hive: + host: hadoop-master + jdbc_url: jdbc:hive2://${databases.hive.host}:10000/default;principal=hive/hadoop-master@LABS.TERADATA.COM;auth=kerberos;kerberosAuthType=fromSubject; + jdbc_user: hdfs + jdbc_password: na + kerberos_principal: hdfs/hadoop-master@LABS.TERADATA.COM + kerberos_keytab: /etc/hadoop/conf/hdfs.keytab + + presto: + host: presto-master.docker.cluster + port: 7778 + http_port: 8080 + https_port: ${databases.presto.port} + server_address: https://${databases.presto.host}:${databases.presto.https_port} + jdbc_url: "jdbc:presto://${databases.presto.host}:${databases.presto.port}/hive/${databases.hive.schema}?\ + SSL=true&\ + SSLTrustStorePath=${databases.presto.https_keystore_path}&\ + SSLTrustStorePassword=${databases.presto.https_keystore_password}&\ + user=presto-client&\ + KerberosRemoteServiceName=${databases.presto.kerberos_service_name}&\ + KerberosPrincipal=${databases.presto.kerberos_principal}&\ + KerberosKeytabPath=${databases.presto.kerberos_keytab}&\ + KerberosConfigPath=${databases.presto.kerberos_config_path}&\ + KerberosUseCanonicalHostname=${databases.presto.kerberos_use_canonical_hostname}" + configured_hdfs_user: hdfs + https_keystore_path: /docker/volumes/conf/presto/etc/docker.cluster.jks + https_keystore_password: '123456' + kerberos_principal: presto-client/presto-master.docker.cluster@LABS.TERADATA.COM + kerberos_keytab: /etc/presto/conf/presto-client.keytab + kerberos_config_path: /etc/krb5.conf + kerberos_service_name: presto-server + kerberos_use_canonical_hostname: false + cli_kerberos_authentication: true + cli_kerberos_principal: ${databases.presto.kerberos_principal} + cli_kerberos_keytab: ${databases.presto.kerberos_keytab} + cli_kerberos_config_path: ${databases.presto.kerberos_config_path} + cli_kerberos_service_name: ${databases.presto.kerberos_service_name} + cli_kerberos_use_canonical_hostname: ${databases.presto.kerberos_use_canonical_hostname} diff --git a/presto-product-tests/conf/tempto/tempto-configuration-for-docker-tls.yaml b/presto-product-tests/conf/tempto/tempto-configuration-for-docker-tls.yaml deleted file mode 100644 index a86a12f00efa6..0000000000000 --- a/presto-product-tests/conf/tempto/tempto-configuration-for-docker-tls.yaml +++ /dev/null @@ -1,16 +0,0 @@ -databases: - hive: - host: hadoop-master - presto: - host: presto-master.docker.cluster - port: 7778 - http_port: 8080 - https_port: ${databases.presto.port} - server_address: https://${databases.presto.host}:${databases.presto.port} - jdbc_url: "jdbc:presto://${databases.presto.host}:${databases.presto.port}/hive/${databases.hive.schema}?\ - SSL=true&\ - SSLTrustStorePath=${databases.presto.https_keystore_path}&\ - SSLTrustStorePassword=${databases.presto.https_keystore_password}" - configured_hdfs_user: hive - https_keystore_path: /docker/volumes/conf/presto/etc/docker.cluster.jks - https_keystore_password: '123456' diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/TestGroups.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/TestGroups.java index bfa83472587fb..ea18c1c3a1e1b 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/TestGroups.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/TestGroups.java @@ -59,6 +59,7 @@ public final class TestGroups public static final String LDAP_CLI = "ldap_cli"; public static final String SKIP_ON_CDH = "skip_on_cdh"; public static final String TLS = "tls"; + public static final String STATS_CLIENT = "stats_client"; private TestGroups() {} } diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestTablePartitioningInsertInto.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestTablePartitioningInsertInto.java index ea79ef60afcb2..329aff5d8a213 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestTablePartitioningInsertInto.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestTablePartitioningInsertInto.java @@ -21,6 +21,7 @@ import static com.facebook.presto.tests.TestGroups.HIVE_CONNECTOR; import static com.facebook.presto.tests.TestGroups.SMOKE; +import static com.facebook.presto.tests.TestGroups.STATS_CLIENT; import static com.facebook.presto.tests.hive.HiveTableDefinitions.NATION_PARTITIONED_BY_REGIONKEY; import static com.facebook.presto.tests.hive.HiveTableDefinitions.NATION_PARTITIONED_BY_REGIONKEY_NUMBER_OF_LINES_PER_SPLIT; import static com.teradata.tempto.Requirements.compose; @@ -46,7 +47,7 @@ public Requirement getRequirements(Configuration configuration) mutableTable(NATION, TARGET_NATION_NAME, CREATED)); } - @Test(groups = {HIVE_CONNECTOR, SMOKE}) + @Test(groups = {HIVE_CONNECTOR, SMOKE, STATS_CLIENT}) public void selectFromPartitionedNation() throws Exception {