diff --git a/lib/trino-hdfs/src/main/java/io/trino/hdfs/authentication/AuthenticationModules.java b/lib/trino-hdfs/src/main/java/io/trino/hdfs/authentication/AuthenticationModules.java index 6a2f14314f90..78f3c0c2f90f 100644 --- a/lib/trino-hdfs/src/main/java/io/trino/hdfs/authentication/AuthenticationModules.java +++ b/lib/trino-hdfs/src/main/java/io/trino/hdfs/authentication/AuthenticationModules.java @@ -20,6 +20,7 @@ import com.google.inject.Singleton; import io.trino.hdfs.HdfsConfigurationInitializer; import io.trino.plugin.base.authentication.KerberosAuthentication; +import io.trino.plugin.base.authentication.KerberosConfiguration; import javax.inject.Inject; @@ -69,8 +70,11 @@ public void configure(Binder binder) HadoopAuthentication createHadoopAuthentication(HdfsKerberosConfig config, HdfsConfigurationInitializer updater) { String principal = config.getHdfsTrinoPrincipal(); - String keytabLocation = config.getHdfsTrinoKeytab(); - return createCachingKerberosHadoopAuthentication(principal, keytabLocation, updater); + KerberosConfiguration.Builder builder = new KerberosConfiguration.Builder() + .withKerberosPrincipal(principal); + config.getHdfsTrinoKeytab().ifPresent(builder::withKeytabLocation); + config.getHdfsTrinoCredentialCacheLocation().ifPresent(builder::withCredentialCacheLocation); + return createCachingKerberosHadoopAuthentication(builder.build(), updater); } }; } @@ -99,15 +103,18 @@ public void configure(Binder binder) HadoopAuthentication createHadoopAuthentication(HdfsKerberosConfig config, HdfsConfigurationInitializer updater) { String principal = config.getHdfsTrinoPrincipal(); - String keytabLocation = config.getHdfsTrinoKeytab(); - return createCachingKerberosHadoopAuthentication(principal, keytabLocation, updater); + KerberosConfiguration.Builder builder = new KerberosConfiguration.Builder() + .withKerberosPrincipal(principal); + config.getHdfsTrinoKeytab().ifPresent(builder::withKeytabLocation); + config.getHdfsTrinoCredentialCacheLocation().ifPresent(builder::withCredentialCacheLocation); + return createCachingKerberosHadoopAuthentication(builder.build(), updater); } }; } - public static HadoopAuthentication createCachingKerberosHadoopAuthentication(String principal, String keytabLocation, HdfsConfigurationInitializer updater) + public static HadoopAuthentication createCachingKerberosHadoopAuthentication(KerberosConfiguration kerberosConfiguration, HdfsConfigurationInitializer updater) { - KerberosAuthentication kerberosAuthentication = new KerberosAuthentication(principal, keytabLocation); + KerberosAuthentication kerberosAuthentication = new KerberosAuthentication(kerberosConfiguration); KerberosHadoopAuthentication kerberosHadoopAuthentication = createKerberosHadoopAuthentication(kerberosAuthentication, updater); return new CachingKerberosHadoopAuthentication(kerberosHadoopAuthentication); } diff --git a/lib/trino-hdfs/src/main/java/io/trino/hdfs/authentication/HdfsKerberosConfig.java b/lib/trino-hdfs/src/main/java/io/trino/hdfs/authentication/HdfsKerberosConfig.java index 652559fd4a17..a068f6e73fda 100644 --- a/lib/trino-hdfs/src/main/java/io/trino/hdfs/authentication/HdfsKerberosConfig.java +++ b/lib/trino-hdfs/src/main/java/io/trino/hdfs/authentication/HdfsKerberosConfig.java @@ -18,12 +18,16 @@ import io.airlift.configuration.LegacyConfig; import io.airlift.configuration.validation.FileExists; +import javax.validation.constraints.AssertTrue; import javax.validation.constraints.NotNull; +import java.util.Optional; + public class HdfsKerberosConfig { private String hdfsTrinoPrincipal; private String hdfsTrinoKeytab; + private String hdfsTrinoCredentialCacheLocation; @NotNull public String getHdfsTrinoPrincipal() @@ -41,10 +45,9 @@ public HdfsKerberosConfig setHdfsTrinoPrincipal(String hdfsTrinoPrincipal) } @NotNull - @FileExists - public String getHdfsTrinoKeytab() + public Optional<@FileExists String> getHdfsTrinoKeytab() { - return hdfsTrinoKeytab; + return Optional.ofNullable(hdfsTrinoKeytab); } @Config("hive.hdfs.trino.keytab") @@ -55,4 +58,24 @@ public HdfsKerberosConfig setHdfsTrinoKeytab(String hdfsTrinoKeytab) this.hdfsTrinoKeytab = hdfsTrinoKeytab; return this; } + + @NotNull + public Optional<@FileExists String> getHdfsTrinoCredentialCacheLocation() + { + return Optional.ofNullable(hdfsTrinoCredentialCacheLocation); + } + + @Config("hive.hdfs.trino.credential-cache.location") + @ConfigDescription("Trino credential-cache location used to access HDFS") + public HdfsKerberosConfig setHdfsTrinoCredentialCacheLocation(String hdfsTrinoCredentialCacheLocation) + { + this.hdfsTrinoCredentialCacheLocation = hdfsTrinoCredentialCacheLocation; + return this; + } + + @AssertTrue(message = "Exactly one of `hive.hdfs.trino.keytab` or `hive.hdfs.trino.credential-cache.location` must be specified") + public boolean isConfigValid() + { + return getHdfsTrinoKeytab().isPresent() ^ getHdfsTrinoCredentialCacheLocation().isPresent(); + } } diff --git a/lib/trino-hdfs/src/test/java/io/trino/hdfs/authentication/TestHdfsKerberosConfig.java b/lib/trino-hdfs/src/test/java/io/trino/hdfs/authentication/TestHdfsKerberosConfig.java index efd9aa2e335c..18648d8939e6 100644 --- a/lib/trino-hdfs/src/test/java/io/trino/hdfs/authentication/TestHdfsKerberosConfig.java +++ b/lib/trino-hdfs/src/test/java/io/trino/hdfs/authentication/TestHdfsKerberosConfig.java @@ -14,16 +14,19 @@ package io.trino.hdfs.authentication; import com.google.common.collect.ImmutableMap; +import io.airlift.configuration.ConfigurationFactory; import org.testng.annotations.Test; -import java.io.IOException; +import javax.validation.constraints.AssertTrue; + import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static io.airlift.testing.ValidationAssertions.assertFailsValidation; +import static org.assertj.core.api.Assertions.assertThat; public class TestHdfsKerberosConfig { @@ -32,12 +35,13 @@ public void testDefaults() { assertRecordedDefaults(recordDefaults(HdfsKerberosConfig.class) .setHdfsTrinoPrincipal(null) - .setHdfsTrinoKeytab(null)); + .setHdfsTrinoKeytab(null) + .setHdfsTrinoCredentialCacheLocation(null)); } @Test - public void testExplicitPropertyMappings() - throws IOException + public void testExplicitPropertyMappingsForKeytab() + throws Exception { Path keytab = Files.createTempFile(null, null); @@ -46,10 +50,64 @@ public void testExplicitPropertyMappings() .put("hive.hdfs.trino.keytab", keytab.toString()) .buildOrThrow(); + ConfigurationFactory configurationFactory = new ConfigurationFactory(properties); + HdfsKerberosConfig config = configurationFactory.build(HdfsKerberosConfig.class); + HdfsKerberosConfig expected = new HdfsKerberosConfig() .setHdfsTrinoPrincipal("trino@EXAMPLE.COM") .setHdfsTrinoKeytab(keytab.toString()); - assertFullMapping(properties, expected); + assertThat(config.getHdfsTrinoPrincipal()) + .isEqualTo(expected.getHdfsTrinoPrincipal()); + assertThat(config.getHdfsTrinoKeytab()) + .isEqualTo(expected.getHdfsTrinoKeytab()); + } + + @Test + public void testExplicitPropertyMappingsForCredentialCache() + throws Exception + { + Path credentialCacheLocation = Files.createTempFile("credentialCache", null); + + Map properties = ImmutableMap.builder() + .put("hive.hdfs.trino.principal", "trino@EXAMPLE.COM") + .put("hive.hdfs.trino.credential-cache.location", credentialCacheLocation.toString()) + .buildOrThrow(); + + ConfigurationFactory configurationFactory = new ConfigurationFactory(properties); + HdfsKerberosConfig config = configurationFactory.build(HdfsKerberosConfig.class); + + HdfsKerberosConfig expected = new HdfsKerberosConfig() + .setHdfsTrinoPrincipal("trino@EXAMPLE.COM") + .setHdfsTrinoCredentialCacheLocation(credentialCacheLocation.toString()); + + assertThat(config.getHdfsTrinoPrincipal()) + .isEqualTo(expected.getHdfsTrinoPrincipal()); + assertThat(config.getHdfsTrinoCredentialCacheLocation()) + .isEqualTo(expected.getHdfsTrinoCredentialCacheLocation()); + } + + @Test + public void testValidation() + throws Exception + { + assertFailsValidation( + new HdfsKerberosConfig() + .setHdfsTrinoPrincipal("trino@EXAMPLE.COM"), + "configValid", + "Exactly one of `hive.hdfs.trino.keytab` or `hive.hdfs.trino.credential-cache.location` must be specified", + AssertTrue.class); + + Path keytab = Files.createTempFile(null, null); + Path credentialCacheLocation = Files.createTempFile("credentialCache", null); + + assertFailsValidation( + new HdfsKerberosConfig() + .setHdfsTrinoPrincipal("trino@EXAMPLE.COM") + .setHdfsTrinoKeytab(keytab.toString()) + .setHdfsTrinoCredentialCacheLocation(credentialCacheLocation.toString()), + "configValid", + "Exactly one of `hive.hdfs.trino.keytab` or `hive.hdfs.trino.credential-cache.location` must be specified", + AssertTrue.class); } } diff --git a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/authentication/KerberosAuthentication.java b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/authentication/KerberosAuthentication.java index 9d17ac25192a..dce7ba4efba4 100644 --- a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/authentication/KerberosAuthentication.java +++ b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/authentication/KerberosAuthentication.java @@ -13,51 +13,33 @@ */ package io.trino.plugin.base.authentication; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import io.airlift.log.Logger; import javax.security.auth.Subject; import javax.security.auth.kerberos.KerberosPrincipal; -import javax.security.auth.login.AppConfigurationEntry; import javax.security.auth.login.Configuration; import javax.security.auth.login.LoginContext; import javax.security.auth.login.LoginException; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.net.InetAddress; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Map; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.lang.String.format; -import static java.nio.file.Files.exists; -import static java.nio.file.Files.isReadable; import static java.util.Collections.emptySet; -import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; public class KerberosAuthentication { private static final Logger log = Logger.get(KerberosAuthentication.class); - private static final String KERBEROS_LOGIN_MODULE = "com.sun.security.auth.module.Krb5LoginModule"; - - private static final String HOSTNAME_PATTERN = "_HOST"; private final KerberosPrincipal principal; private final Configuration configuration; - public KerberosAuthentication(String principal, String keytabLocation) + public KerberosAuthentication(KerberosConfiguration kerberosConfiguration) { - requireNonNull(principal, "principal is null"); - requireNonNull(keytabLocation, "keytabLocation is null"); - Path keytabPath = Paths.get(keytabLocation); - checkArgument(exists(keytabPath), "keytab does not exist: %s", keytabLocation); - checkArgument(isReadable(keytabPath), "keytab is not readable: %s", keytabLocation); - this.principal = createKerberosPrincipal(principal); - this.configuration = createConfiguration(this.principal.getName(), keytabLocation); + requireNonNull(kerberosConfiguration, "kerberosConfiguration is null"); + this.principal = kerberosConfiguration.kerberosPrincipal(); + if (log.isDebugEnabled()) { + kerberosConfiguration = kerberosConfiguration.withDebug(); + } + this.configuration = kerberosConfiguration.getConfiguration(); } public Subject getSubject() @@ -83,53 +65,4 @@ public void attemptLogin(Subject subject) throw new RuntimeException(e); } } - - private static KerberosPrincipal createKerberosPrincipal(String principal) - { - try { - return new KerberosPrincipal(getServerPrincipal(principal, InetAddress.getLocalHost().getCanonicalHostName())); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private static Configuration createConfiguration(String principal, String keytabLocation) - { - ImmutableMap.Builder optionsBuilder = ImmutableMap.builder() - .put("useKeyTab", "true") - .put("storeKey", "true") - .put("doNotPrompt", "true") - .put("isInitiator", "true") - .put("principal", principal) - .put("keyTab", keytabLocation); - - if (log.isDebugEnabled()) { - optionsBuilder.put("debug", "true"); - } - - Map options = optionsBuilder.buildOrThrow(); - - return new Configuration() - { - @Override - public AppConfigurationEntry[] getAppConfigurationEntry(String name) - { - return new AppConfigurationEntry[] { - new AppConfigurationEntry( - KERBEROS_LOGIN_MODULE, - AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, - options)}; - } - }; - } - - private static String getServerPrincipal(String principal, String hostname) - { - String[] components = principal.split("[/@]"); - if (components.length != 3 || !components[1].equals(HOSTNAME_PATTERN)) { - return principal; - } - return format("%s/%s@%s", components[0], hostname.toLowerCase(ENGLISH), components[2]); - } } diff --git a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/authentication/KerberosConfiguration.java b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/authentication/KerberosConfiguration.java new file mode 100644 index 000000000000..7df9280473f7 --- /dev/null +++ b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/authentication/KerberosConfiguration.java @@ -0,0 +1,152 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.base.authentication; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; + +import javax.security.auth.kerberos.KerberosPrincipal; +import javax.security.auth.login.AppConfigurationEntry; +import javax.security.auth.login.Configuration; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.InetAddress; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.nio.file.Files.exists; +import static java.nio.file.Files.isReadable; +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; + +public record KerberosConfiguration(KerberosPrincipal kerberosPrincipal, Map options) +{ + private static final String KERBEROS_LOGIN_MODULE = "com.sun.security.auth.module.Krb5LoginModule"; + + private static final Pattern PRINCIPAL_NAME_PATTERN = Pattern.compile("(.*/)_HOST(@.*)?"); + + public KerberosConfiguration + { + requireNonNull(kerberosPrincipal, "kerberosPrincipal is null"); + options = ImmutableMap.copyOf(requireNonNull(options, "options is null")); + } + + public KerberosConfiguration withDebug() + { + ImmutableMap.Builder optionsBuilder = ImmutableMap.builder(); + optionsBuilder.putAll(options) + .put("debug", "true"); + return new KerberosConfiguration(kerberosPrincipal, optionsBuilder.buildOrThrow()); + } + + public Configuration getConfiguration() + { + return new Configuration() + { + @Override + public AppConfigurationEntry[] getAppConfigurationEntry(String name) + { + return new AppConfigurationEntry[] { + new AppConfigurationEntry( + KERBEROS_LOGIN_MODULE, + AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, + options)}; + } + }; + } + + public static class Builder + { + private KerberosPrincipal kerberosPrincipal; + private Optional keytabLocation = Optional.empty(); + private Optional credentialCacheLocation = Optional.empty(); + + public Builder withKerberosPrincipal(String kerberosPrincipal) + { + this.kerberosPrincipal = createKerberosPrincipal(kerberosPrincipal); + return this; + } + + public Builder withKeytabLocation(String keytabLocation) + { + verifyFile(keytabLocation); + this.keytabLocation = Optional.of(keytabLocation); + return this; + } + + public Builder withCredentialCacheLocation(String credentialCacheLocation) + { + verifyFile(credentialCacheLocation); + this.credentialCacheLocation = Optional.of(credentialCacheLocation); + return this; + } + + public KerberosConfiguration build() + { + ImmutableMap.Builder optionsBuilder = ImmutableMap.builder() + .put("doNotPrompt", "true") + .put("isInitiator", "true") + .put("principal", kerberosPrincipal.getName()); + + checkArgument(keytabLocation.isPresent() ^ credentialCacheLocation.isPresent(), "Either keytab or credential cache must be specified"); + + keytabLocation.ifPresent( + keytab -> optionsBuilder + .put("useKeyTab", "true") + .put("storeKey", "true") + .put("keyTab", keytab)); + + credentialCacheLocation.ifPresent( + credentialCache -> optionsBuilder + .put("useTicketCache", "true") + .put("renewTGT", "true") + .put("ticketCache", credentialCache)); + + return new KerberosConfiguration(kerberosPrincipal, optionsBuilder.buildOrThrow()); + } + + private static KerberosPrincipal createKerberosPrincipal(String principal) + { + try { + return new KerberosPrincipal(getServerPrincipal(principal, InetAddress.getLocalHost().getCanonicalHostName())); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @VisibleForTesting + static String getServerPrincipal(String principal, String hostname) + { + Matcher matcher = PRINCIPAL_NAME_PATTERN.matcher(principal); + if (matcher.matches()) { + return matcher.replaceAll("$1" + hostname.toLowerCase(ENGLISH) + "$2"); + } + return principal; + } + + private static void verifyFile(String fileLocation) + { + Path filePath = Paths.get(fileLocation); + checkArgument(exists(filePath), "File does not exist: %s", fileLocation); + checkArgument(isReadable(filePath), "File is not readable: %s", fileLocation); + } + } +} diff --git a/lib/trino-plugin-toolkit/src/test/java/io/trino/plugin/base/authentication/TestKerberosConfiguration.java b/lib/trino-plugin-toolkit/src/test/java/io/trino/plugin/base/authentication/TestKerberosConfiguration.java new file mode 100644 index 000000000000..c5c3748333f7 --- /dev/null +++ b/lib/trino-plugin-toolkit/src/test/java/io/trino/plugin/base/authentication/TestKerberosConfiguration.java @@ -0,0 +1,44 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.base.authentication; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import static io.trino.plugin.base.authentication.KerberosConfiguration.Builder.getServerPrincipal; +import static org.assertj.core.api.Assertions.assertThat; + +public class TestKerberosConfiguration +{ + private static final String HOST_NAME = "host_name"; + + @Test(dataProvider = "kerberosPrincipalPattern") + public void testHostnameSubstitution(String actual, String expected) + { + assertThat(getServerPrincipal(actual, HOST_NAME)).isEqualTo(expected); + } + + @DataProvider(name = "kerberosPrincipalPattern") + public Object[][] kerberosPrincipalPattern() + { + return new Object[][] { + {"server/_HOST@REALM.COM", "server/host_name@REALM.COM"}, + {"server/_HOST", "server/host_name"}, + {"server/trino-worker@REALM.COM", "server/trino-worker@REALM.COM"}, + {"server/trino-worker", "server/trino-worker"}, + {"SERVER_HOST/_HOST@REALM.COM", "SERVER_HOST/host_name@REALM.COM"}, + {"SERVER_HOST/_HOST", "SERVER_HOST/host_name"}, + }; + } +} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/MetastoreKerberosConfig.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/MetastoreKerberosConfig.java index 1983938bf288..1aa1ae1eb971 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/MetastoreKerberosConfig.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/MetastoreKerberosConfig.java @@ -17,13 +17,17 @@ import io.airlift.configuration.ConfigDescription; import io.airlift.configuration.validation.FileExists; +import javax.validation.constraints.AssertTrue; import javax.validation.constraints.NotNull; +import java.util.Optional; + public class MetastoreKerberosConfig { private String hiveMetastoreServicePrincipal; private String hiveMetastoreClientPrincipal; private String hiveMetastoreClientKeytab; + private String hiveMetastoreCredentialCachePath; @NotNull public String getHiveMetastoreServicePrincipal() @@ -54,10 +58,9 @@ public MetastoreKerberosConfig setHiveMetastoreClientPrincipal(String hiveMetast } @NotNull - @FileExists - public String getHiveMetastoreClientKeytab() + public Optional<@FileExists String> getHiveMetastoreClientKeytab() { - return hiveMetastoreClientKeytab; + return Optional.ofNullable(hiveMetastoreClientKeytab); } @Config("hive.metastore.client.keytab") @@ -67,4 +70,24 @@ public MetastoreKerberosConfig setHiveMetastoreClientKeytab(String hiveMetastore this.hiveMetastoreClientKeytab = hiveMetastoreClientKeytab; return this; } + + @NotNull + public Optional<@FileExists String> getHiveMetastoreClientCredentialCacheLocation() + { + return Optional.ofNullable(hiveMetastoreCredentialCachePath); + } + + @Config("hive.metastore.client.credential-cache.location") + @ConfigDescription("Hive Metastore client credential cache location") + public MetastoreKerberosConfig setHiveMetastoreClientCredentialCacheLocation(String hiveMetastoreCredentialCachePath) + { + this.hiveMetastoreCredentialCachePath = hiveMetastoreCredentialCachePath; + return this; + } + + @AssertTrue(message = "Exactly one of `hive.metastore.client.keytab` or `hive.metastore.client.credential-cache.location` must be specified") + public boolean isConfigValid() + { + return getHiveMetastoreClientKeytab().isPresent() ^ getHiveMetastoreClientCredentialCacheLocation().isPresent(); + } } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreAuthenticationModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreAuthenticationModule.java index 36e8f9c41db8..fe8114888cbf 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreAuthenticationModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreAuthenticationModule.java @@ -20,6 +20,7 @@ import io.airlift.configuration.AbstractConfigurationAwareModule; import io.trino.hdfs.HdfsConfigurationInitializer; import io.trino.hdfs.authentication.HadoopAuthentication; +import io.trino.plugin.base.authentication.KerberosConfiguration; import io.trino.plugin.hive.ForHiveMetastore; import static com.google.inject.Scopes.SINGLETON; @@ -69,8 +70,11 @@ public void configure(Binder binder) public HadoopAuthentication createHadoopAuthentication(MetastoreKerberosConfig config, HdfsConfigurationInitializer updater) { String principal = config.getHiveMetastoreClientPrincipal(); - String keytabLocation = config.getHiveMetastoreClientKeytab(); - return createCachingKerberosHadoopAuthentication(principal, keytabLocation, updater); + KerberosConfiguration.Builder builder = new KerberosConfiguration.Builder() + .withKerberosPrincipal(principal); + config.getHiveMetastoreClientKeytab().ifPresent(builder::withKeytabLocation); + config.getHiveMetastoreClientCredentialCacheLocation().ifPresent(builder::withCredentialCacheLocation); + return createCachingKerberosHadoopAuthentication(builder.build(), updater); } } } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestMetastoreKerberosConfig.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestMetastoreKerberosConfig.java index bd10dea053f1..e4ecb48cfd4c 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestMetastoreKerberosConfig.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestMetastoreKerberosConfig.java @@ -14,16 +14,19 @@ package io.trino.plugin.hive.metastore.thrift; import com.google.common.collect.ImmutableMap; +import io.airlift.configuration.ConfigurationFactory; import org.testng.annotations.Test; -import java.io.IOException; +import javax.validation.constraints.AssertTrue; + import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static io.airlift.testing.ValidationAssertions.assertFailsValidation; +import static org.assertj.core.api.Assertions.assertThat; public class TestMetastoreKerberosConfig { @@ -33,12 +36,13 @@ public void testDefaults() assertRecordedDefaults(recordDefaults(MetastoreKerberosConfig.class) .setHiveMetastoreServicePrincipal(null) .setHiveMetastoreClientPrincipal(null) - .setHiveMetastoreClientKeytab(null)); + .setHiveMetastoreClientKeytab(null) + .setHiveMetastoreClientCredentialCacheLocation(null)); } @Test - public void testExplicitPropertyMappings() - throws IOException + public void testExplicitPropertyMappingsForKeytab() + throws Exception { Path clientKeytabFile = Files.createTempFile(null, null); @@ -48,11 +52,73 @@ public void testExplicitPropertyMappings() .put("hive.metastore.client.keytab", clientKeytabFile.toString()) .buildOrThrow(); + ConfigurationFactory configurationFactory = new ConfigurationFactory(properties); + MetastoreKerberosConfig config = configurationFactory.build(MetastoreKerberosConfig.class); + MetastoreKerberosConfig expected = new MetastoreKerberosConfig() .setHiveMetastoreServicePrincipal("hive/_HOST@EXAMPLE.COM") .setHiveMetastoreClientPrincipal("metastore@EXAMPLE.COM") .setHiveMetastoreClientKeytab(clientKeytabFile.toString()); - assertFullMapping(properties, expected); + assertThat(config.getHiveMetastoreServicePrincipal()) + .isEqualTo(expected.getHiveMetastoreServicePrincipal()); + assertThat(config.getHiveMetastoreClientPrincipal()) + .isEqualTo(expected.getHiveMetastoreClientPrincipal()); + assertThat(config.getHiveMetastoreClientKeytab()) + .isEqualTo(expected.getHiveMetastoreClientKeytab()); + } + + @Test + public void testExplicitPropertyMappingsForCredentialCache() + throws Exception + { + Path credentialCacheLocation = Files.createTempFile("credentialCache", null); + + Map properties = ImmutableMap.builder() + .put("hive.metastore.service.principal", "hive/_HOST@EXAMPLE.COM") + .put("hive.metastore.client.principal", "metastore@EXAMPLE.COM") + .put("hive.metastore.client.credential-cache.location", credentialCacheLocation.toString()) + .buildOrThrow(); + + ConfigurationFactory configurationFactory = new ConfigurationFactory(properties); + MetastoreKerberosConfig config = configurationFactory.build(MetastoreKerberosConfig.class); + + MetastoreKerberosConfig expected = new MetastoreKerberosConfig() + .setHiveMetastoreServicePrincipal("hive/_HOST@EXAMPLE.COM") + .setHiveMetastoreClientPrincipal("metastore@EXAMPLE.COM") + .setHiveMetastoreClientCredentialCacheLocation(credentialCacheLocation.toString()); + + assertThat(config.getHiveMetastoreServicePrincipal()) + .isEqualTo(expected.getHiveMetastoreServicePrincipal()); + assertThat(config.getHiveMetastoreClientPrincipal()) + .isEqualTo(expected.getHiveMetastoreClientPrincipal()); + assertThat(config.getHiveMetastoreClientCredentialCacheLocation()) + .isEqualTo(expected.getHiveMetastoreClientCredentialCacheLocation()); + } + + @Test + public void testValidation() + throws Exception + { + assertFailsValidation( + new MetastoreKerberosConfig() + .setHiveMetastoreServicePrincipal("hive/_HOST@EXAMPLE.COM") + .setHiveMetastoreClientPrincipal("metastore@EXAMPLE.COM"), + "configValid", + "Exactly one of `hive.metastore.client.keytab` or `hive.metastore.client.credential-cache.location` must be specified", + AssertTrue.class); + + Path clientKeytabFile = Files.createTempFile(null, null); + Path credentialCacheLocation = Files.createTempFile("credentialCache", null); + + assertFailsValidation( + new MetastoreKerberosConfig() + .setHiveMetastoreServicePrincipal("hive/_HOST@EXAMPLE.COM") + .setHiveMetastoreClientPrincipal("metastore@EXAMPLE.COM") + .setHiveMetastoreClientKeytab(clientKeytabFile.toString()) + .setHiveMetastoreClientCredentialCacheLocation(credentialCacheLocation.toString()), + "configValid", + "Exactly one of `hive.metastore.client.keytab` or `hive.metastore.client.credential-cache.location` must be specified", + AssertTrue.class); } } diff --git a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduSecurityModule.java b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduSecurityModule.java index 0df9f4517721..4a98b22f5bab 100644 --- a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduSecurityModule.java +++ b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduSecurityModule.java @@ -18,6 +18,7 @@ import io.airlift.configuration.AbstractConfigurationAwareModule; import io.trino.plugin.base.authentication.CachingKerberosAuthentication; import io.trino.plugin.base.authentication.KerberosAuthentication; +import io.trino.plugin.base.authentication.KerberosConfiguration; import io.trino.plugin.kudu.schema.NoSchemaEmulation; import io.trino.plugin.kudu.schema.SchemaEmulation; import io.trino.plugin.kudu.schema.SchemaEmulationByTableNameConvention; @@ -88,7 +89,11 @@ public static KuduClientSession createKuduClientSession(KuduClientConfig config, builder -> { kuduKerberosConfig.getKuduPrincipalPrimary().ifPresent(builder::saslProtocolName); setJavaSecurityKrb5Conf(kuduKerberosConfig.getConfig().getAbsolutePath()); - KerberosAuthentication kerberosAuthentication = new KerberosAuthentication(kuduKerberosConfig.getClientPrincipal(), kuduKerberosConfig.getClientKeytab().getAbsolutePath()); + KerberosAuthentication kerberosAuthentication = new KerberosAuthentication( + new KerberosConfiguration.Builder() + .withKerberosPrincipal(kuduKerberosConfig.getClientPrincipal()) + .withKeytabLocation(kuduKerberosConfig.getClientKeytab().getAbsolutePath()) + .build()); CachingKerberosAuthentication cachingKerberosAuthentication = new CachingKerberosAuthentication(kerberosAuthentication); return new KerberizedKuduClient(builder, cachingKerberosAuthentication); }); diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/HadoopKerberos.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/HadoopKerberos.java index e37410c55fbd..7320ff69b960 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/HadoopKerberos.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/HadoopKerberos.java @@ -69,7 +69,10 @@ public void extendEnvironment(Environment.Builder builder) portBinder.exposePort(container, 7778); container .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withDomainName("docker.cluster")) - .withCopyFileToContainer(forHostPath(configDir.getPath("config.properties")), CONTAINER_PRESTO_CONFIG_PROPERTIES); + .withCopyFileToContainer(forHostPath(configDir.getPath("config.properties")), CONTAINER_PRESTO_CONFIG_PROPERTIES) + .withCopyFileToContainer( + forHostPath(configDir.getPath("create_kerberos_credential_cache_files.sh")), + "/docker/presto-init.d/create_kerberos_credentials.sh"); }); builder.configureContainer(TESTS, container -> { container.setDockerImageName(dockerImageName); diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosHiveImpersonationWithCredentialCache.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosHiveImpersonationWithCredentialCache.java new file mode 100644 index 000000000000..0dc3be3b0a42 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosHiveImpersonationWithCredentialCache.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.env.environment; + +import com.google.common.collect.ImmutableList; +import io.trino.tests.product.launcher.docker.DockerFiles; +import io.trino.tests.product.launcher.docker.DockerFiles.ResourceProvider; +import io.trino.tests.product.launcher.env.Environment; +import io.trino.tests.product.launcher.env.EnvironmentProvider; +import io.trino.tests.product.launcher.env.common.HadoopKerberos; +import io.trino.tests.product.launcher.env.common.Standard; +import io.trino.tests.product.launcher.env.common.TestsEnvironment; + +import javax.inject.Inject; + +import static java.util.Objects.requireNonNull; +import static org.testcontainers.utility.MountableFile.forHostPath; + +@TestsEnvironment +public final class EnvSinglenodeKerberosHiveImpersonationWithCredentialCache + extends EnvironmentProvider +{ + private final ResourceProvider configDir; + + @Inject + public EnvSinglenodeKerberosHiveImpersonationWithCredentialCache(DockerFiles dockerFiles, Standard standard, HadoopKerberos hadoopKerberos) + { + super(ImmutableList.of(standard, hadoopKerberos)); + configDir = requireNonNull(dockerFiles, "dockerFiles is null").getDockerFilesHostDirectory("conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache"); + } + + @Override + @SuppressWarnings("resource") + public void extendEnvironment(Environment.Builder builder) + { + builder.addConnector("hive", forHostPath(configDir.getPath("hive.properties"))); + builder.addConnector("iceberg", forHostPath(configDir.getPath("iceberg.properties"))); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosHiveNoImpersonationWithCredentialCache.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosHiveNoImpersonationWithCredentialCache.java new file mode 100644 index 000000000000..dd4de06080d6 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosHiveNoImpersonationWithCredentialCache.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.env.environment; + +import com.google.common.collect.ImmutableList; +import io.trino.tests.product.launcher.docker.DockerFiles; +import io.trino.tests.product.launcher.docker.DockerFiles.ResourceProvider; +import io.trino.tests.product.launcher.env.Environment; +import io.trino.tests.product.launcher.env.EnvironmentProvider; +import io.trino.tests.product.launcher.env.common.HadoopKerberos; +import io.trino.tests.product.launcher.env.common.Standard; +import io.trino.tests.product.launcher.env.common.TestsEnvironment; + +import javax.inject.Inject; + +import static java.util.Objects.requireNonNull; +import static org.testcontainers.utility.MountableFile.forHostPath; + +@TestsEnvironment +public final class EnvSinglenodeKerberosHiveNoImpersonationWithCredentialCache + extends EnvironmentProvider +{ + private final ResourceProvider configDir; + + @Inject + public EnvSinglenodeKerberosHiveNoImpersonationWithCredentialCache(DockerFiles dockerFiles, Standard standard, HadoopKerberos hadoopKerberos) + { + super(ImmutableList.of(standard, hadoopKerberos)); + configDir = requireNonNull(dockerFiles, "dockerFiles is null").getDockerFilesHostDirectory("conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache"); + } + + @Override + @SuppressWarnings("resource") + public void extendEnvironment(Environment.Builder builder) + { + builder.addConnector("hive", forHostPath(configDir.getPath("hive.properties"))); + builder.addConnector("iceberg", forHostPath(configDir.getPath("iceberg.properties"))); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosKmsHdfsImpersonationWithCredentialCache.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosKmsHdfsImpersonationWithCredentialCache.java new file mode 100644 index 000000000000..caf896851375 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosKmsHdfsImpersonationWithCredentialCache.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.env.environment; + +import com.google.common.collect.ImmutableList; +import io.trino.tests.product.launcher.docker.DockerFiles; +import io.trino.tests.product.launcher.docker.DockerFiles.ResourceProvider; +import io.trino.tests.product.launcher.env.Environment; +import io.trino.tests.product.launcher.env.EnvironmentProvider; +import io.trino.tests.product.launcher.env.common.HadoopKerberosKmsWithImpersonation; +import io.trino.tests.product.launcher.env.common.Standard; +import io.trino.tests.product.launcher.env.common.TestsEnvironment; + +import javax.inject.Inject; + +import static java.util.Objects.requireNonNull; +import static org.testcontainers.utility.MountableFile.forHostPath; + +@TestsEnvironment +public final class EnvSinglenodeKerberosKmsHdfsImpersonationWithCredentialCache + extends EnvironmentProvider +{ + private final ResourceProvider configDir; + + @Inject + public EnvSinglenodeKerberosKmsHdfsImpersonationWithCredentialCache(DockerFiles dockerFiles, Standard standard, HadoopKerberosKmsWithImpersonation hadoopKerberosKmsWithImpersonation) + { + super(ImmutableList.of(standard, hadoopKerberosKmsWithImpersonation)); + configDir = requireNonNull(dockerFiles, "dockerFiles is null").getDockerFilesHostDirectory("conf/environment/singlenode-kerberos-kms-hdfs-impersonation-with-credential-cache"); + } + + @Override + @SuppressWarnings("resource") + public void extendEnvironment(Environment.Builder builder) + { + builder.addConnector("hive", forHostPath(configDir.getPath("hive.properties"))); + builder.addConnector("iceberg", forHostPath(configDir.getPath("iceberg.properties"))); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosKmsHdfsNoImpersonationWithCredentialCache.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosKmsHdfsNoImpersonationWithCredentialCache.java new file mode 100644 index 000000000000..3b5347b8342f --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosKmsHdfsNoImpersonationWithCredentialCache.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.env.environment; + +import com.google.common.collect.ImmutableList; +import io.trino.tests.product.launcher.docker.DockerFiles; +import io.trino.tests.product.launcher.docker.DockerFiles.ResourceProvider; +import io.trino.tests.product.launcher.env.Environment; +import io.trino.tests.product.launcher.env.EnvironmentProvider; +import io.trino.tests.product.launcher.env.common.HadoopKerberosKms; +import io.trino.tests.product.launcher.env.common.Standard; +import io.trino.tests.product.launcher.env.common.TestsEnvironment; + +import javax.inject.Inject; + +import static java.util.Objects.requireNonNull; +import static org.testcontainers.utility.MountableFile.forHostPath; + +@TestsEnvironment +public final class EnvSinglenodeKerberosKmsHdfsNoImpersonationWithCredentialCache + extends EnvironmentProvider +{ + private final ResourceProvider configDir; + + @Inject + public EnvSinglenodeKerberosKmsHdfsNoImpersonationWithCredentialCache(DockerFiles dockerFiles, Standard standard, HadoopKerberosKms hadoopKerberosKms) + { + super(ImmutableList.of(standard, hadoopKerberosKms)); + configDir = requireNonNull(dockerFiles, "dockerFiles is null").getDockerFilesHostDirectory("conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation-with-credential-cache"); + } + + @Override + @SuppressWarnings("resource") + public void extendEnvironment(Environment.Builder builder) + { + builder.addConnector("hive", forHostPath(configDir.getPath("hive.properties"))); + builder.addConnector("iceberg", forHostPath(configDir.getPath("iceberg.properties"))); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite2.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite2.java index b19764c72a12..1d410d7ec104 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite2.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite2.java @@ -19,6 +19,7 @@ import io.trino.tests.product.launcher.env.environment.EnvSinglenodeHdfsImpersonation; import io.trino.tests.product.launcher.env.environment.EnvSinglenodeKerberosHdfsImpersonation; import io.trino.tests.product.launcher.env.environment.EnvSinglenodeKerberosHdfsNoImpersonation; +import io.trino.tests.product.launcher.env.environment.EnvSinglenodeKerberosHiveNoImpersonationWithCredentialCache; import io.trino.tests.product.launcher.suite.Suite; import io.trino.tests.product.launcher.suite.SuiteTestRun; @@ -39,6 +40,9 @@ public List getTestRuns(EnvironmentConfig config) testOnEnvironment(EnvSinglenodeKerberosHdfsNoImpersonation.class) .withGroups("configured_features", "storage_formats", "hdfs_no_impersonation") .build(), + testOnEnvironment(EnvSinglenodeKerberosHiveNoImpersonationWithCredentialCache.class) + .withGroups("configured_features", "storage_formats", "hdfs_no_impersonation") + .build(), testOnEnvironment(EnvSinglenodeHdfsImpersonation.class) .withGroups("configured_features", "storage_formats", "cli", "hdfs_impersonation") .build(), diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite5.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite5.java index 90c89177d943..5ebb6ba4177a 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite5.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite5.java @@ -18,6 +18,7 @@ import io.trino.tests.product.launcher.env.environment.EnvMultinodeHiveCaching; import io.trino.tests.product.launcher.env.environment.EnvSinglenodeHiveImpersonation; import io.trino.tests.product.launcher.env.environment.EnvSinglenodeKerberosHiveImpersonation; +import io.trino.tests.product.launcher.env.environment.EnvSinglenodeKerberosHiveImpersonationWithCredentialCache; import io.trino.tests.product.launcher.suite.Suite; import io.trino.tests.product.launcher.suite.SuiteTestRun; @@ -38,6 +39,9 @@ public List getTestRuns(EnvironmentConfig config) testOnEnvironment(EnvSinglenodeKerberosHiveImpersonation.class) .withGroups("configured_features", "storage_formats", "hdfs_impersonation", "authorization") .build(), + testOnEnvironment(EnvSinglenodeKerberosHiveImpersonationWithCredentialCache.class) + .withGroups("configured_features", "storage_formats", "hdfs_impersonation", "authorization") + .build(), testOnEnvironment(EnvMultinodeHiveCaching.class) .withGroups("configured_features", "hive_caching", "storage_formats") .withExcludedGroups("iceberg") diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite6NonGeneric.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite6NonGeneric.java index 013d98051323..164bcd0de949 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite6NonGeneric.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite6NonGeneric.java @@ -22,7 +22,9 @@ import io.trino.tests.product.launcher.env.environment.EnvMultinodePhoenix5; import io.trino.tests.product.launcher.env.environment.EnvSinglenodeCassandra; import io.trino.tests.product.launcher.env.environment.EnvSinglenodeKerberosKmsHdfsImpersonation; +import io.trino.tests.product.launcher.env.environment.EnvSinglenodeKerberosKmsHdfsImpersonationWithCredentialCache; import io.trino.tests.product.launcher.env.environment.EnvSinglenodeKerberosKmsHdfsNoImpersonation; +import io.trino.tests.product.launcher.env.environment.EnvSinglenodeKerberosKmsHdfsNoImpersonationWithCredentialCache; import io.trino.tests.product.launcher.suite.Suite; import io.trino.tests.product.launcher.suite.SuiteTestRun; @@ -43,9 +45,15 @@ public List getTestRuns(EnvironmentConfig config) testOnEnvironment(EnvSinglenodeKerberosKmsHdfsNoImpersonation.class) .withGroups("configured_features", "storage_formats") .build(), + testOnEnvironment(EnvSinglenodeKerberosKmsHdfsNoImpersonationWithCredentialCache.class) + .withGroups("configured_features", "storage_formats") + .build(), testOnEnvironment(EnvSinglenodeKerberosKmsHdfsImpersonation.class) .withGroups("configured_features", "storage_formats") .build(), + testOnEnvironment(EnvSinglenodeKerberosKmsHdfsImpersonationWithCredentialCache.class) + .withGroups("configured_features", "storage_formats") + .build(), testOnEnvironment(EnvSinglenodeCassandra.class) .withGroups("configured_features", "cassandra") .build(), diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop-kerberos/create_kerberos_credential_cache_files.sh b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop-kerberos/create_kerberos_credential_cache_files.sh new file mode 100644 index 000000000000..e74d7fd60260 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop-kerberos/create_kerberos_credential_cache_files.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -exuo pipefail + +kinit -f -c /etc/trino/conf/presto-server-krbcc \ + -kt /etc/trino/conf/presto-server.keytab presto-server/$(hostname -f)@LABS.TERADATA.COM + +kinit -f -c /etc/trino/conf/hive-presto-master-krbcc \ + -kt /etc/trino/conf/hive-presto-master.keytab hive/$(hostname -f)@LABS.TERADATA.COM + + +kinit -f -c /etc/trino/conf/hdfs-krbcc \ + -kt /etc/hadoop/conf/hdfs.keytab hdfs/hadoop-master@LABS.TERADATA.COM + +kinit -f -c /etc/trino/conf/hive-krbcc \ + -kt /etc/hive/conf/hive.keytab hive/hadoop-master@LABS.TERADATA.COM diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/hive.properties new file mode 100644 index 000000000000..6b136719230e --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/hive.properties @@ -0,0 +1,25 @@ +connector.name=hive +hive.metastore.uri=thrift://hadoop-master:9083 +hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml +hive.metastore-cache-ttl=0s + +hive.metastore.authentication.type=KERBEROS +hive.metastore.thrift.impersonation.enabled=true +hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.principal=hive/_HOST@LABS.TERADATA.COM +hive.metastore.client.credential-cache.location=/etc/trino/conf/hive-presto-master-krbcc + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=true +hive.hdfs.trino.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.hdfs.trino.credential-cache.location=/etc/trino/conf/presto-server-krbcc +# Restrictive new directory permissions make sense when HDFS and metastore impersonation is enabled. +# We cannot enable them here, because 'authorization' tests require that one user is able to write to other user's table. +#hive.fs.new-directory-permissions=0700 +hive.fs.cache.max-size=10 +hive.max-partitions-per-scan=100 + +#required for testGrantRevoke() product test +hive.security=sql-standard +#required for testAccessControlSetHiveViewAuthorization() product test +hive.hive-views.enabled=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/iceberg.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/iceberg.properties new file mode 100644 index 000000000000..c376c7179c0c --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/iceberg.properties @@ -0,0 +1,16 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 +hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml + +hive.metastore.authentication.type=KERBEROS +hive.metastore.thrift.impersonation.enabled=true +hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.principal=hive/_HOST@LABS.TERADATA.COM +hive.metastore.client.credential-cache.location=/etc/trino/conf/hive-presto-master-krbcc + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=true +hive.hdfs.trino.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.hdfs.trino.credential-cache.location=/etc/trino/conf/presto-server-krbcc + +iceberg.file-format=PARQUET diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/hive.properties new file mode 100644 index 000000000000..38fbd90a4c30 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/hive.properties @@ -0,0 +1,22 @@ +connector.name=hive +hive.metastore.uri=thrift://hadoop-master:9083 +hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml +hive.allow-drop-table=true +hive.allow-rename-table=true +hive.allow-comment-table=true +hive.metastore-cache-ttl=0s +hive.allow-add-column=true +hive.allow-drop-column=true +hive.allow-rename-column=true + +hive.metastore.authentication.type=KERBEROS +hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.credential-cache.location=/etc/trino/conf/hive-krbcc + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=false +hive.hdfs.trino.principal=hdfs/hadoop-master@LABS.TERADATA.COM +hive.hdfs.trino.credential-cache.location=/etc/trino/conf/hdfs-krbcc +hive.fs.cache.max-size=10 +hive.max-partitions-per-scan=100 diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/iceberg.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/iceberg.properties new file mode 100644 index 000000000000..8182b53463f6 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/iceberg.properties @@ -0,0 +1,15 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 +hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml + +hive.metastore.authentication.type=KERBEROS +hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.credential-cache.location=/etc/trino/conf/hive-krbcc + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=false +hive.hdfs.trino.principal=hdfs/hadoop-master@LABS.TERADATA.COM +hive.hdfs.trino.credential-cache.location=/etc/trino/conf/hdfs-krbcc + +iceberg.file-format=PARQUET diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation-with-credential-cache/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation-with-credential-cache/hive.properties new file mode 100644 index 000000000000..01add4b28b47 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation-with-credential-cache/hive.properties @@ -0,0 +1,24 @@ +connector.name=hive +hive.metastore.uri=thrift://hadoop-master:9083 +hive.allow-drop-table=true +hive.allow-rename-table=true +hive.allow-add-column=true +hive.allow-drop-column=true +hive.allow-rename-column=true + +hive.metastore.authentication.type=KERBEROS +hive.metastore.service.principal=hive/_HOST@LABS.TERADATA.COM +# When using HDFS impersonation we talk to HDFS as session user configured in tempto (which is 'hive') +# However, when DROP TABLE is issued, Metastore needs to be able to delete the table files on HDFS, +# otherwise they be silently left behind, potentially causing some further tests to fail. For this reason, +# `hive.metastore.client.principal` needs to match session user configured in tempto. +hive.metastore.client.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.credential-cache.location=/etc/trino/conf/hive-krbcc + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=true +hive.hdfs.trino.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.hdfs.trino.credential-cache.location=/etc/trino/conf/presto-server-krbcc +hive.fs.cache.max-size=10 +hive.max-partitions-per-scan=100 +hive.config.resources=/etc/hadoop/conf/core-site.xml,/etc/hadoop-kms/conf/hive-disable-key-provider-cache-site.xml diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation-with-credential-cache/iceberg.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation-with-credential-cache/iceberg.properties new file mode 100644 index 000000000000..ea0a276ecfb2 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation-with-credential-cache/iceberg.properties @@ -0,0 +1,21 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 + +hive.metastore.authentication.type=KERBEROS +hive.metastore.service.principal=hive/_HOST@LABS.TERADATA.COM +# When using HDFS impersonation we talk to HDFS as session user configured in tempto (which is 'hive') +# However, TODO we don't have Metastore impersonation yet; when DROP TABLE is issued, +# Metastore needs to be able to delete the table files on HDFS, otherwise they be silently left behind, +# potentially causing some further tests to fail. For this reason, `hive.metastore.client.principal` needs +# to match session user configured in tempto. +hive.metastore.client.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.credential-cache.location=/etc/trino/conf/hive-krbcc + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=true +hive.hdfs.trino.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.hdfs.trino.credential-cache.location=/etc/trino/conf/presto-server-krbcc +hive.fs.cache.max-size=10 +hive.config.resources=/etc/hadoop/conf/core-site.xml,/etc/hadoop-kms/conf/hive-disable-key-provider-cache-site.xml + +iceberg.file-format=PARQUET diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation-with-credential-cache/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation-with-credential-cache/hive.properties new file mode 100644 index 000000000000..15a8a7c69458 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation-with-credential-cache/hive.properties @@ -0,0 +1,20 @@ +connector.name=hive +hive.metastore.uri=thrift://hadoop-master:9083 +hive.allow-drop-table=true +hive.allow-rename-table=true +hive.allow-add-column=true +hive.allow-drop-column=true +hive.allow-rename-column=true + +hive.metastore.authentication.type=KERBEROS +hive.metastore.service.principal=hive/_HOST@LABS.TERADATA.COM +hive.metastore.client.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.metastore.client.credential-cache.location=/etc/trino/conf/presto-server-krbcc + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=false +hive.hdfs.trino.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.hdfs.trino.credential-cache.location=/etc/trino/conf/presto-server-krbcc +hive.fs.cache.max-size=10 +hive.max-partitions-per-scan=100 +hive.config.resources=/etc/hadoop/conf/core-site.xml,/etc/hadoop-kms/conf/hive-disable-key-provider-cache-site.xml diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation-with-credential-cache/iceberg.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation-with-credential-cache/iceberg.properties new file mode 100644 index 000000000000..76308bbc3125 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation-with-credential-cache/iceberg.properties @@ -0,0 +1,16 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 + +hive.metastore.authentication.type=KERBEROS +hive.metastore.service.principal=hive/_HOST@LABS.TERADATA.COM +hive.metastore.client.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.metastore.client.credential-cache.location=/etc/trino/conf/presto-server-krbcc + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=false +hive.hdfs.trino.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.hdfs.trino.credential-cache.location=/etc/trino/conf/presto-server-krbcc +hive.fs.cache.max-size=10 +hive.config.resources=/etc/hadoop/conf/core-site.xml,/etc/hadoop-kms/conf/hive-disable-key-provider-cache-site.xml + +iceberg.file-format=PARQUET