Skip to content

Allow configuring credential-cache for kerberized hive connector #13482

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Praveen2112 merged 3 commits into from
Aug 29, 2022
Merged

Allow configuring credential-cache for kerberized hive connector #13482

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c497254
Extract KerberosConfiguration from KerberosAuthentication
Praveen2112 Aug 12, 2022
325039c
Make hostname substitution resilient
Praveen2112 Aug 19, 2022
adf88dc
Allow configuring credential-cache for kerberized hive connector
Praveen2112 Aug 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 13 additions & 6 deletions lib/trino-hdfs/src/main/java/io/trino/hdfs/authentication/AuthenticationModules.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.google.inject.Singleton;
import io.trino.hdfs.HdfsConfigurationInitializer;
import io.trino.plugin.base.authentication.KerberosAuthentication;
import io.trino.plugin.base.authentication.KerberosConfiguration;

import javax.inject.Inject;

Expand Down Expand Up @@ -69,8 +70,11 @@ public void configure(Binder binder)
HadoopAuthentication createHadoopAuthentication(HdfsKerberosConfig config, HdfsConfigurationInitializer updater)
{
String principal = config.getHdfsTrinoPrincipal();
String keytabLocation = config.getHdfsTrinoKeytab();
return createCachingKerberosHadoopAuthentication(principal, keytabLocation, updater);
KerberosConfiguration.Builder builder = new KerberosConfiguration.Builder()
.withKerberosPrincipal(principal);
config.getHdfsTrinoKeytab().ifPresent(builder::withKeytabLocation);
config.getHdfsTrinoCredentialCacheLocation().ifPresent(builder::withCredentialCacheLocation);
return createCachingKerberosHadoopAuthentication(builder.build(), updater);
}
};
}
Expand Down Expand Up @@ -99,15 +103,18 @@ public void configure(Binder binder)
HadoopAuthentication createHadoopAuthentication(HdfsKerberosConfig config, HdfsConfigurationInitializer updater)
{
String principal = config.getHdfsTrinoPrincipal();
String keytabLocation = config.getHdfsTrinoKeytab();
return createCachingKerberosHadoopAuthentication(principal, keytabLocation, updater);
KerberosConfiguration.Builder builder = new KerberosConfiguration.Builder()
.withKerberosPrincipal(principal);
config.getHdfsTrinoKeytab().ifPresent(builder::withKeytabLocation);
config.getHdfsTrinoCredentialCacheLocation().ifPresent(builder::withCredentialCacheLocation);
return createCachingKerberosHadoopAuthentication(builder.build(), updater);
}
};
}

public static HadoopAuthentication createCachingKerberosHadoopAuthentication(String principal, String keytabLocation, HdfsConfigurationInitializer updater)
public static HadoopAuthentication createCachingKerberosHadoopAuthentication(KerberosConfiguration kerberosConfiguration, HdfsConfigurationInitializer updater)
{
KerberosAuthentication kerberosAuthentication = new KerberosAuthentication(principal, keytabLocation);
KerberosAuthentication kerberosAuthentication = new KerberosAuthentication(kerberosConfiguration);
KerberosHadoopAuthentication kerberosHadoopAuthentication = createKerberosHadoopAuthentication(kerberosAuthentication, updater);
return new CachingKerberosHadoopAuthentication(kerberosHadoopAuthentication);
}
Expand Down
29 changes: 26 additions & 3 deletions lib/trino-hdfs/src/main/java/io/trino/hdfs/authentication/HdfsKerberosConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,16 @@
import io.airlift.configuration.LegacyConfig;
import io.airlift.configuration.validation.FileExists;

import javax.validation.constraints.AssertTrue;
import javax.validation.constraints.NotNull;

import java.util.Optional;

public class HdfsKerberosConfig
{
private String hdfsTrinoPrincipal;
private String hdfsTrinoKeytab;
private String hdfsTrinoCredentialCacheLocation;

@NotNull
public String getHdfsTrinoPrincipal()
Expand All @@ -41,10 +45,9 @@ public HdfsKerberosConfig setHdfsTrinoPrincipal(String hdfsTrinoPrincipal)
}

@NotNull
@FileExists
public String getHdfsTrinoKeytab()
public Optional<@FileExists String> getHdfsTrinoKeytab()
Copy link
Copy Markdown
Member

@s2lomon s2lomon Aug 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wow, I've not known that we can do that. So it's either null, or path to existing file correct?

Copy link
Copy Markdown
Member Author

@Praveen2112 Praveen2112 Aug 17, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah. And it applies check if the file exists.

{
return hdfsTrinoKeytab;
return Optional.ofNullable(hdfsTrinoKeytab);
}

@Config("hive.hdfs.trino.keytab")
Expand All @@ -55,4 +58,24 @@ public HdfsKerberosConfig setHdfsTrinoKeytab(String hdfsTrinoKeytab)
this.hdfsTrinoKeytab = hdfsTrinoKeytab;
return this;
}

@NotNull
public Optional<@FileExists String> getHdfsTrinoCredentialCacheLocation()
{
return Optional.ofNullable(hdfsTrinoCredentialCacheLocation);
}

@Config("hive.hdfs.trino.credential-cache.location")
@ConfigDescription("Trino credential-cache location used to access HDFS")
public HdfsKerberosConfig setHdfsTrinoCredentialCacheLocation(String hdfsTrinoCredentialCacheLocation)
{
this.hdfsTrinoCredentialCacheLocation = hdfsTrinoCredentialCacheLocation;
return this;
}

@AssertTrue(message = "Exactly one of `hive.hdfs.trino.keytab` or `hive.hdfs.trino.credential-cache.location` must be specified")
public boolean isConfigValid()
{
return getHdfsTrinoKeytab().isPresent() ^ getHdfsTrinoCredentialCacheLocation().isPresent();
}
}
70 changes: 64 additions & 6 deletions lib/trino-hdfs/src/test/java/io/trino/hdfs/authentication/TestHdfsKerberosConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,19 @@
package io.trino.hdfs.authentication;

import com.google.common.collect.ImmutableMap;
import io.airlift.configuration.ConfigurationFactory;
import org.testng.annotations.Test;

import java.io.IOException;
import javax.validation.constraints.AssertTrue;

import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Map;

import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping;
import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults;
import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults;
import static io.airlift.testing.ValidationAssertions.assertFailsValidation;
import static org.assertj.core.api.Assertions.assertThat;

public class TestHdfsKerberosConfig
{
Expand All @@ -32,12 +35,13 @@ public void testDefaults()
{
assertRecordedDefaults(recordDefaults(HdfsKerberosConfig.class)
.setHdfsTrinoPrincipal(null)
.setHdfsTrinoKeytab(null));
.setHdfsTrinoKeytab(null)
.setHdfsTrinoCredentialCacheLocation(null));
}

@Test
public void testExplicitPropertyMappings()
throws IOException
public void testExplicitPropertyMappingsForKeytab()
throws Exception
{
Path keytab = Files.createTempFile(null, null);

Expand All @@ -46,10 +50,64 @@ public void testExplicitPropertyMappings()
.put("hive.hdfs.trino.keytab", keytab.toString())
.buildOrThrow();

ConfigurationFactory configurationFactory = new ConfigurationFactory(properties);
HdfsKerberosConfig config = configurationFactory.build(HdfsKerberosConfig.class);

HdfsKerberosConfig expected = new HdfsKerberosConfig()
.setHdfsTrinoPrincipal("trino@EXAMPLE.COM")
.setHdfsTrinoKeytab(keytab.toString());

assertFullMapping(properties, expected);
assertThat(config.getHdfsTrinoPrincipal())
.isEqualTo(expected.getHdfsTrinoPrincipal());
assertThat(config.getHdfsTrinoKeytab())
.isEqualTo(expected.getHdfsTrinoKeytab());
}

@Test
public void testExplicitPropertyMappingsForCredentialCache()
throws Exception
{
Path credentialCacheLocation = Files.createTempFile("credentialCache", null);

Map<String, String> properties = ImmutableMap.<String, String>builder()
.put("hive.hdfs.trino.principal", "trino@EXAMPLE.COM")
.put("hive.hdfs.trino.credential-cache.location", credentialCacheLocation.toString())
.buildOrThrow();

ConfigurationFactory configurationFactory = new ConfigurationFactory(properties);
HdfsKerberosConfig config = configurationFactory.build(HdfsKerberosConfig.class);

HdfsKerberosConfig expected = new HdfsKerberosConfig()
.setHdfsTrinoPrincipal("trino@EXAMPLE.COM")
.setHdfsTrinoCredentialCacheLocation(credentialCacheLocation.toString());

assertThat(config.getHdfsTrinoPrincipal())
.isEqualTo(expected.getHdfsTrinoPrincipal());
assertThat(config.getHdfsTrinoCredentialCacheLocation())
.isEqualTo(expected.getHdfsTrinoCredentialCacheLocation());
}

@Test
public void testValidation()
throws Exception
{
assertFailsValidation(
new HdfsKerberosConfig()
.setHdfsTrinoPrincipal("trino@EXAMPLE.COM"),
"configValid",
"Exactly one of `hive.hdfs.trino.keytab` or `hive.hdfs.trino.credential-cache.location` must be specified",
AssertTrue.class);

Path keytab = Files.createTempFile(null, null);
Path credentialCacheLocation = Files.createTempFile("credentialCache", null);

assertFailsValidation(
new HdfsKerberosConfig()
.setHdfsTrinoPrincipal("trino@EXAMPLE.COM")
.setHdfsTrinoKeytab(keytab.toString())
.setHdfsTrinoCredentialCacheLocation(credentialCacheLocation.toString()),
"configValid",
"Exactly one of `hive.hdfs.trino.keytab` or `hive.hdfs.trino.credential-cache.location` must be specified",
AssertTrue.class);
}
}
81 changes: 7 additions & 74 deletions ...gin-toolkit/src/main/java/io/trino/plugin/base/authentication/KerberosAuthentication.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,51 +13,33 @@
*/
package io.trino.plugin.base.authentication;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import io.airlift.log.Logger;

import javax.security.auth.Subject;
import javax.security.auth.kerberos.KerberosPrincipal;
import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.Configuration;
import javax.security.auth.login.LoginContext;
import javax.security.auth.login.LoginException;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.InetAddress;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Map;

import static com.google.common.base.Preconditions.checkArgument;
import static java.lang.String.format;
import static java.nio.file.Files.exists;
import static java.nio.file.Files.isReadable;
import static java.util.Collections.emptySet;
import static java.util.Locale.ENGLISH;
import static java.util.Objects.requireNonNull;

public class KerberosAuthentication
{
private static final Logger log = Logger.get(KerberosAuthentication.class);
private static final String KERBEROS_LOGIN_MODULE = "com.sun.security.auth.module.Krb5LoginModule";

private static final String HOSTNAME_PATTERN = "_HOST";

private final KerberosPrincipal principal;
private final Configuration configuration;

public KerberosAuthentication(String principal, String keytabLocation)
public KerberosAuthentication(KerberosConfiguration kerberosConfiguration)
{
requireNonNull(principal, "principal is null");
requireNonNull(keytabLocation, "keytabLocation is null");
Path keytabPath = Paths.get(keytabLocation);
checkArgument(exists(keytabPath), "keytab does not exist: %s", keytabLocation);
checkArgument(isReadable(keytabPath), "keytab is not readable: %s", keytabLocation);
this.principal = createKerberosPrincipal(principal);
this.configuration = createConfiguration(this.principal.getName(), keytabLocation);
requireNonNull(kerberosConfiguration, "kerberosConfiguration is null");
this.principal = kerberosConfiguration.kerberosPrincipal();
if (log.isDebugEnabled()) {
kerberosConfiguration = kerberosConfiguration.withDebug();
}
this.configuration = kerberosConfiguration.getConfiguration();
}

public Subject getSubject()
Expand All @@ -83,53 +65,4 @@ public void attemptLogin(Subject subject)
throw new RuntimeException(e);
}
}

private static KerberosPrincipal createKerberosPrincipal(String principal)
{
try {
return new KerberosPrincipal(getServerPrincipal(principal, InetAddress.getLocalHost().getCanonicalHostName()));
}
catch (IOException e) {
throw new UncheckedIOException(e);
}
}

private static Configuration createConfiguration(String principal, String keytabLocation)
{
ImmutableMap.Builder<String, String> optionsBuilder = ImmutableMap.<String, String>builder()
.put("useKeyTab", "true")
.put("storeKey", "true")
.put("doNotPrompt", "true")
.put("isInitiator", "true")
.put("principal", principal)
.put("keyTab", keytabLocation);

if (log.isDebugEnabled()) {
optionsBuilder.put("debug", "true");
}

Map<String, String> options = optionsBuilder.buildOrThrow();

return new Configuration()
{
@Override
public AppConfigurationEntry[] getAppConfigurationEntry(String name)
{
return new AppConfigurationEntry[] {
new AppConfigurationEntry(
KERBEROS_LOGIN_MODULE,
AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
options)};
}
};
}

private static String getServerPrincipal(String principal, String hostname)
{
String[] components = principal.split("[/@]");
if (components.length != 3 || !components[1].equals(HOSTNAME_PATTERN)) {
return principal;
}
return format("%s/%s@%s", components[0], hostname.toLowerCase(ENGLISH), components[2]);
}
}
Loading