diff --git a/core/trino-main/src/main/java/io/trino/server/security/KerberosAuthenticator.java b/core/trino-main/src/main/java/io/trino/server/security/KerberosAuthenticator.java index 9838868a5cd0..3d605e40a26d 100644 --- a/core/trino-main/src/main/java/io/trino/server/security/KerberosAuthenticator.java +++ b/core/trino-main/src/main/java/io/trino/server/security/KerberosAuthenticator.java @@ -41,11 +41,10 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; -import java.util.Objects; import java.util.Optional; -import static com.google.common.base.Preconditions.checkState; import static com.google.common.net.HttpHeaders.AUTHORIZATION; +import static io.trino.plugin.base.util.SystemProperties.setJavaSecurityKrb5Conf; import static io.trino.server.security.UserMapping.createUserMapping; import static java.util.Objects.requireNonNull; import static javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag.REQUIRED; @@ -70,14 +69,7 @@ public KerberosAuthenticator(KerberosConfig config) requireNonNull(config, "config is null"); this.userMapping = createUserMapping(config.getUserMappingPattern(), config.getUserMappingFile()); - String newValue = config.getKerberosConfig().getAbsolutePath(); - String currentValue = System.getProperty("java.security.krb5.conf"); - checkState( - currentValue == null || Objects.equals(currentValue, newValue), - "Refusing to set system property 'java.security.krb5.conf' to '%s', it is already set to '%s'", - newValue, - currentValue); - System.setProperty("java.security.krb5.conf", newValue); + setJavaSecurityKrb5Conf(config.getKerberosConfig().getAbsolutePath()); try { String hostname = Optional.ofNullable(config.getPrincipalHostname()) diff --git a/docs/src/main/sphinx/connector/kudu.rst b/docs/src/main/sphinx/connector/kudu.rst index 0b6671d28905..a0d399cfb490 100644 --- a/docs/src/main/sphinx/connector/kudu.rst +++ b/docs/src/main/sphinx/connector/kudu.rst @@ -26,6 +26,9 @@ replacing the properties as appropriate: connector.name=kudu + ## Defaults to NONE + kudu.authentication.type = NONE + ## List of Kudu master addresses, at least one is needed (comma separated) ## Supported formats: example.com, example.com:7051, 192.0.2.1, 192.0.2.1:7051, ## [2001:db8::1], [2001:db8::1]:7051, 2001:db8::1 @@ -54,6 +57,29 @@ replacing the properties as appropriate: ## Disable Kudu client's collection of statistics. #kudu.client.disable-statistics = false +Kerberos support +---------------- + +In order to connect to a kudu cluster that uses ``kerberos`` +authentication, you need to configure the following kudu properties: + +.. code-block:: properties + + kudu.authentication.type = KERBEROS + + ## The kerberos client principal name + kudu.authentication.client.principal = clientprincipalname + + ## The path to the kerberos keytab file + ## The configured client principal must exist in this keytab file + kudu.authentication.client.keytab = /path/to/keytab/file.keytab + + ## The path to the krb5.conf kerberos config file + kudu.authentication.config = /path/to/kerberos/krb5.conf + + ## Optional and defaults to "kudu" + ## If kudu is running with a custom SPN this needs to be configured + kudu.authentication.server.principal.primary = kudu Querying data ------------- @@ -588,4 +614,3 @@ Limitations ----------- - Only lower case table and column names in Kudu are supported. -- Using a secured Kudu cluster has not been tested. diff --git a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/authentication/CachingKerberosAuthentication.java b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/authentication/CachingKerberosAuthentication.java index 61571ab7efe0..c448a566d4ca 100644 --- a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/authentication/CachingKerberosAuthentication.java +++ b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/authentication/CachingKerberosAuthentication.java @@ -37,11 +37,26 @@ public CachingKerberosAuthentication(KerberosAuthentication kerberosAuthenticati public synchronized Subject getSubject() { - if (subject == null || nextRefreshTime < System.currentTimeMillis()) { + if (subject == null || ticketNeedsRefresh()) { subject = requireNonNull(kerberosAuthentication.getSubject(), "kerberosAuthentication.getSubject() is null"); KerberosTicket tgtTicket = getTicketGrantingTicket(subject); nextRefreshTime = KerberosTicketUtils.getRefreshTime(tgtTicket); } return subject; } + + public synchronized void reauthenticateIfSoonWillBeExpired() + { + requireNonNull(subject, "subject is null, getSubject() must be called before reauthenticate()"); + if (ticketNeedsRefresh()) { + kerberosAuthentication.attemptLogin(subject); + KerberosTicket tgtTicket = getTicketGrantingTicket(subject); + nextRefreshTime = KerberosTicketUtils.getRefreshTime(tgtTicket); + } + } + + private boolean ticketNeedsRefresh() + { + return nextRefreshTime < System.currentTimeMillis(); + } } diff --git a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/authentication/KerberosAuthentication.java b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/authentication/KerberosAuthentication.java index 75e5dd71c47f..9d17ac25192a 100644 --- a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/authentication/KerberosAuthentication.java +++ b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/authentication/KerberosAuthentication.java @@ -73,6 +73,17 @@ public Subject getSubject() } } + public void attemptLogin(Subject subject) + { + try { + LoginContext loginContext = new LoginContext("", subject, null, configuration); + loginContext.login(); + } + catch (LoginException e) { + throw new RuntimeException(e); + } + } + private static KerberosPrincipal createKerberosPrincipal(String principal) { try { diff --git a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/util/SystemProperties.java b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/util/SystemProperties.java new file mode 100644 index 000000000000..a03c4d80634c --- /dev/null +++ b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/util/SystemProperties.java @@ -0,0 +1,47 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.base.util; + +import java.util.Objects; + +import static com.google.common.base.Preconditions.checkState; + +public final class SystemProperties +{ + public static final String JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; + + private SystemProperties() {} + + public static synchronized void setJavaSecurityKrb5Conf(String value) + { + set(JAVA_SECURITY_KRB5_CONF, value); + } + + public static synchronized void set(String key, String newValue) + { + String currentValue = System.getProperty(key); + checkSameValues(key, newValue, currentValue); + System.setProperty(key, newValue); + } + + private static void checkSameValues(String key, String newValue, String currentValue) + { + checkState( + currentValue == null || Objects.equals(currentValue, newValue), + "Refusing to set system property '%s' to '%s', it is already set to '%s'", + key, + newValue, + currentValue); + } +} diff --git a/plugin/trino-kudu/pom.xml b/plugin/trino-kudu/pom.xml index 66358a75622c..3446cad73ae8 100644 --- a/plugin/trino-kudu/pom.xml +++ b/plugin/trino-kudu/pom.xml @@ -134,6 +134,13 @@ test + + io.trino + trino-spi + test-jar + test + + io.trino trino-testing diff --git a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/ForwardingKuduClient.java b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/ForwardingKuduClient.java new file mode 100644 index 000000000000..09ba08f5a1a2 --- /dev/null +++ b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/ForwardingKuduClient.java @@ -0,0 +1,116 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.kudu; + +import org.apache.kudu.Schema; +import org.apache.kudu.client.AlterTableOptions; +import org.apache.kudu.client.AlterTableResponse; +import org.apache.kudu.client.CreateTableOptions; +import org.apache.kudu.client.DeleteTableResponse; +import org.apache.kudu.client.KuduClient; +import org.apache.kudu.client.KuduException; +import org.apache.kudu.client.KuduScanToken; +import org.apache.kudu.client.KuduScanner; +import org.apache.kudu.client.KuduSession; +import org.apache.kudu.client.KuduTable; +import org.apache.kudu.client.ListTablesResponse; + +import java.io.IOException; + +public abstract class ForwardingKuduClient + implements KuduClientWrapper +{ + protected abstract KuduClient delegate(); + + @Override + public KuduTable createTable(String name, Schema schema, CreateTableOptions builder) + throws KuduException + { + return delegate().createTable(name, schema, builder); + } + + @Override + public DeleteTableResponse deleteTable(String name) + throws KuduException + { + return delegate().deleteTable(name); + } + + @Override + public AlterTableResponse alterTable(String name, AlterTableOptions ato) + throws KuduException + { + return delegate().alterTable(name, ato); + } + + @Override + public ListTablesResponse getTablesList() + throws KuduException + { + return delegate().getTablesList(); + } + + @Override + public ListTablesResponse getTablesList(String nameFilter) + throws KuduException + { + return delegate().getTablesList(nameFilter); + } + + @Override + public boolean tableExists(String name) + throws KuduException + { + return delegate().tableExists(name); + } + + @Override + public KuduTable openTable(final String name) + throws KuduException + { + return delegate().openTable(name); + } + + @Override + public KuduScanner.KuduScannerBuilder newScannerBuilder(KuduTable table) + { + return delegate().newScannerBuilder(table); + } + + @Override + public KuduScanToken.KuduScanTokenBuilder newScanTokenBuilder(KuduTable table) + { + return delegate().newScanTokenBuilder(table); + } + + @Override + public KuduSession newSession() + { + return delegate().newSession(); + } + + @Override + public KuduScanner deserializeIntoScanner(byte[] serializedScanToken) + throws IOException + { + return KuduScanToken.deserializeIntoScanner(serializedScanToken, delegate()); + } + + @Override + public void close() + throws KuduException + { + delegate().close(); + } +} diff --git a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KerberizedKuduClient.java b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KerberizedKuduClient.java new file mode 100644 index 000000000000..c82bb585ca97 --- /dev/null +++ b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KerberizedKuduClient.java @@ -0,0 +1,45 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.kudu; + +import io.trino.plugin.base.authentication.CachingKerberosAuthentication; +import org.apache.kudu.client.KuduClient; + +import javax.security.auth.Subject; + +import java.security.PrivilegedAction; + +import static java.util.Objects.requireNonNull; +import static org.apache.kudu.client.KuduClient.KuduClientBuilder; + +public class KerberizedKuduClient + extends ForwardingKuduClient +{ + private final KuduClient kuduClient; + private final CachingKerberosAuthentication cachingKerberosAuthentication; + + KerberizedKuduClient(KuduClientBuilder kuduClientBuilder, CachingKerberosAuthentication cachingKerberosAuthentication) + { + requireNonNull(kuduClientBuilder, "kuduClientBuilder is null"); + this.cachingKerberosAuthentication = requireNonNull(cachingKerberosAuthentication, "cachingKerberosAuthentication is null"); + kuduClient = Subject.doAs(cachingKerberosAuthentication.getSubject(), (PrivilegedAction) (kuduClientBuilder::build)); + } + + @Override + protected KuduClient delegate() + { + cachingKerberosAuthentication.reauthenticateIfSoonWillBeExpired(); + return kuduClient; + } +} diff --git a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduAuthenticationConfig.java b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduAuthenticationConfig.java new file mode 100644 index 000000000000..61ad2c4e4b42 --- /dev/null +++ b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduAuthenticationConfig.java @@ -0,0 +1,41 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.kudu; + +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigDescription; + +public class KuduAuthenticationConfig +{ + private KuduAuthenticationType authenticationType = KuduAuthenticationType.NONE; + + public enum KuduAuthenticationType + { + NONE, + KERBEROS; + } + + public KuduAuthenticationType getAuthenticationType() + { + return authenticationType; + } + + @Config("kudu.authentication.type") + @ConfigDescription("Kudu authentication type") + public KuduAuthenticationConfig setAuthenticationType(KuduAuthenticationType authenticationType) + { + this.authenticationType = authenticationType; + return this; + } +} diff --git a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduClientSession.java b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduClientSession.java index 785769553860..0762a7b2f6aa 100644 --- a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduClientSession.java +++ b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduClientSession.java @@ -45,7 +45,6 @@ import org.apache.kudu.Type; import org.apache.kudu.client.AlterTableOptions; import org.apache.kudu.client.CreateTableOptions; -import org.apache.kudu.client.KuduClient; import org.apache.kudu.client.KuduException; import org.apache.kudu.client.KuduPredicate; import org.apache.kudu.client.KuduScanToken; @@ -78,10 +77,10 @@ public class KuduClientSession { private static final Logger log = Logger.get(KuduClientSession.class); public static final String DEFAULT_SCHEMA = "default"; - private final KuduClient client; + private final KuduClientWrapper client; private final SchemaEmulation schemaEmulation; - public KuduClientSession(KuduClient client, SchemaEmulation schemaEmulation) + public KuduClientSession(KuduClientWrapper client, SchemaEmulation schemaEmulation) { this.client = client; this.schemaEmulation = schemaEmulation; @@ -219,7 +218,7 @@ public List buildKuduSplits(KuduTableHandle tableHandle, DynamicFilte public KuduScanner createScanner(KuduSplit kuduSplit) { try { - return KuduScanToken.deserializeIntoScanner(kuduSplit.getSerializedScanToken(), client); + return client.deserializeIntoScanner(kuduSplit.getSerializedScanToken()); } catch (IOException e) { throw new RuntimeException(e); diff --git a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduClientWrapper.java b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduClientWrapper.java new file mode 100644 index 000000000000..1fd8ebe171f9 --- /dev/null +++ b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduClientWrapper.java @@ -0,0 +1,57 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.kudu; + +import org.apache.kudu.Schema; +import org.apache.kudu.client.AlterTableOptions; +import org.apache.kudu.client.AlterTableResponse; +import org.apache.kudu.client.CreateTableOptions; +import org.apache.kudu.client.DeleteTableResponse; +import org.apache.kudu.client.KuduException; +import org.apache.kudu.client.KuduScanToken; +import org.apache.kudu.client.KuduScanner; +import org.apache.kudu.client.KuduSession; +import org.apache.kudu.client.KuduTable; +import org.apache.kudu.client.ListTablesResponse; + +import java.io.IOException; + +public interface KuduClientWrapper + extends AutoCloseable +{ + KuduTable createTable(String name, Schema schema, CreateTableOptions builder) throws KuduException; + + DeleteTableResponse deleteTable(String name) throws KuduException; + + AlterTableResponse alterTable(String name, AlterTableOptions ato) throws KuduException; + + ListTablesResponse getTablesList() throws KuduException; + + ListTablesResponse getTablesList(String nameFilter) throws KuduException; + + boolean tableExists(String name) throws KuduException; + + KuduTable openTable(String name) throws KuduException; + + KuduScanner.KuduScannerBuilder newScannerBuilder(KuduTable table); + + KuduScanToken.KuduScanTokenBuilder newScanTokenBuilder(KuduTable table); + + KuduSession newSession(); + + KuduScanner deserializeIntoScanner(byte[] serializedScanToken) throws IOException; + + @Override + void close() throws KuduException; +} diff --git a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduKerberosConfig.java b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduKerberosConfig.java new file mode 100644 index 000000000000..150ddeccefa1 --- /dev/null +++ b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduKerberosConfig.java @@ -0,0 +1,89 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.kudu; + +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigDescription; +import io.airlift.configuration.validation.FileExists; + +import javax.validation.constraints.NotNull; + +import java.io.File; +import java.util.Optional; + +public class KuduKerberosConfig +{ + private String clientPrincipal; + private File clientKeytab; + private File config; + // The kudu client defaults to using "kudu" if this is undefined + private Optional kuduPrincipalPrimary = Optional.empty(); + + @NotNull + public String getClientPrincipal() + { + return clientPrincipal; + } + + @Config("kudu.authentication.client.principal") + @ConfigDescription("Kudu Kerberos client principal") + public KuduKerberosConfig setClientPrincipal(String clientPrincipal) + { + this.clientPrincipal = clientPrincipal; + return this; + } + + @NotNull + @FileExists + public File getClientKeytab() + { + return clientKeytab; + } + + @Config("kudu.authentication.client.keytab") + @ConfigDescription("Kudu Kerberos client keytab location") + public KuduKerberosConfig setClientKeytab(File clientKeytab) + { + this.clientKeytab = clientKeytab; + return this; + } + + @NotNull + @FileExists + public File getConfig() + { + return config; + } + + @Config("kudu.authentication.config") + @ConfigDescription("Kudu Kerberos service configuration file") + public KuduKerberosConfig setConfig(File config) + { + this.config = config; + return this; + } + + public Optional getKuduPrincipalPrimary() + { + return kuduPrincipalPrimary; + } + + @Config("kudu.authentication.server.principal.primary") + @ConfigDescription("The 'primary' portion of the kudu service principal name") + public KuduKerberosConfig setKuduPrincipalPrimary(String kuduPrincipalPrimary) + { + this.kuduPrincipalPrimary = Optional.ofNullable(kuduPrincipalPrimary); + return this; + } +} diff --git a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduModule.java b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduModule.java index dd4ac2b9b117..6dd9a977e643 100755 --- a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduModule.java +++ b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduModule.java @@ -13,33 +13,27 @@ */ package io.trino.plugin.kudu; -import com.google.inject.AbstractModule; -import com.google.inject.Provides; +import com.google.inject.Binder; import com.google.inject.Scopes; import com.google.inject.multibindings.Multibinder; import com.google.inject.multibindings.ProvidesIntoSet; +import io.airlift.configuration.AbstractConfigurationAwareModule; import io.trino.plugin.base.classloader.ClassLoaderSafeNodePartitioningProvider; import io.trino.plugin.base.classloader.ForClassLoaderSafe; import io.trino.plugin.kudu.procedures.RangePartitionProcedures; import io.trino.plugin.kudu.properties.KuduTableProperties; -import io.trino.plugin.kudu.schema.NoSchemaEmulation; -import io.trino.plugin.kudu.schema.SchemaEmulation; -import io.trino.plugin.kudu.schema.SchemaEmulationByTableNameConvention; import io.trino.spi.connector.ConnectorNodePartitioningProvider; import io.trino.spi.connector.ConnectorPageSinkProvider; import io.trino.spi.connector.ConnectorPageSourceProvider; import io.trino.spi.connector.ConnectorSplitManager; import io.trino.spi.procedure.Procedure; import io.trino.spi.type.TypeManager; -import org.apache.kudu.client.KuduClient; - -import javax.inject.Singleton; import static io.airlift.configuration.ConfigBinder.configBinder; import static java.util.Objects.requireNonNull; public class KuduModule - extends AbstractModule + extends AbstractConfigurationAwareModule { private final TypeManager typeManager; @@ -49,25 +43,27 @@ public KuduModule(TypeManager typeManager) } @Override - protected void configure() + protected void setup(Binder binder) { - bind(TypeManager.class).toInstance(typeManager); + binder.bind(TypeManager.class).toInstance(typeManager); - bind(KuduConnector.class).in(Scopes.SINGLETON); - bind(KuduMetadata.class).in(Scopes.SINGLETON); - bind(KuduTableProperties.class).in(Scopes.SINGLETON); - bind(ConnectorSplitManager.class).to(KuduSplitManager.class).in(Scopes.SINGLETON); - bind(ConnectorPageSourceProvider.class).to(KuduPageSourceProvider.class) + binder.bind(KuduConnector.class).in(Scopes.SINGLETON); + binder.bind(KuduMetadata.class).in(Scopes.SINGLETON); + binder.bind(KuduTableProperties.class).in(Scopes.SINGLETON); + binder.bind(ConnectorSplitManager.class).to(KuduSplitManager.class).in(Scopes.SINGLETON); + binder.bind(ConnectorPageSourceProvider.class).to(KuduPageSourceProvider.class) .in(Scopes.SINGLETON); - bind(ConnectorPageSinkProvider.class).to(KuduPageSinkProvider.class).in(Scopes.SINGLETON); - bind(KuduSessionProperties.class).in(Scopes.SINGLETON); - bind(ConnectorNodePartitioningProvider.class).annotatedWith(ForClassLoaderSafe.class).to(KuduNodePartitioningProvider.class).in(Scopes.SINGLETON); - bind(ConnectorNodePartitioningProvider.class).to(ClassLoaderSafeNodePartitioningProvider.class).in(Scopes.SINGLETON); - bind(KuduRecordSetProvider.class).in(Scopes.SINGLETON); - configBinder(binder()).bindConfig(KuduClientConfig.class); + binder.bind(ConnectorPageSinkProvider.class).to(KuduPageSinkProvider.class).in(Scopes.SINGLETON); + binder.bind(KuduSessionProperties.class).in(Scopes.SINGLETON); + binder.bind(ConnectorNodePartitioningProvider.class).annotatedWith(ForClassLoaderSafe.class).to(KuduNodePartitioningProvider.class).in(Scopes.SINGLETON); + binder.bind(ConnectorNodePartitioningProvider.class).to(ClassLoaderSafeNodePartitioningProvider.class).in(Scopes.SINGLETON); + binder.bind(KuduRecordSetProvider.class).in(Scopes.SINGLETON); + configBinder(binder).bindConfig(KuduClientConfig.class); + + binder.bind(RangePartitionProcedures.class).in(Scopes.SINGLETON); + Multibinder.newSetBinder(binder, Procedure.class); - bind(RangePartitionProcedures.class).in(Scopes.SINGLETON); - Multibinder.newSetBinder(binder(), Procedure.class); + install(new KuduSecurityModule()); } @ProvidesIntoSet @@ -81,28 +77,4 @@ Procedure getDropRangePartitionProcedure(RangePartitionProcedures procedures) { return procedures.getDropPartitionProcedure(); } - - @Singleton - @Provides - KuduClientSession createKuduClientSession(KuduClientConfig config) - { - requireNonNull(config, "config is null"); - - KuduClient.KuduClientBuilder builder = new KuduClient.KuduClientBuilder(config.getMasterAddresses()); - builder.defaultAdminOperationTimeoutMs(config.getDefaultAdminOperationTimeout().toMillis()); - builder.defaultOperationTimeoutMs(config.getDefaultOperationTimeout().toMillis()); - if (config.isDisableStatistics()) { - builder.disableStatistics(); - } - KuduClient client = builder.build(); - - SchemaEmulation strategy; - if (config.isSchemaEmulationEnabled()) { - strategy = new SchemaEmulationByTableNameConvention(config.getSchemaEmulationPrefix()); - } - else { - strategy = new NoSchemaEmulation(); - } - return new KuduClientSession(client, strategy); - } } diff --git a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduSecurityModule.java b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduSecurityModule.java new file mode 100644 index 000000000000..0df9f4517721 --- /dev/null +++ b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduSecurityModule.java @@ -0,0 +1,119 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.kudu; + +import com.google.inject.Binder; +import com.google.inject.Provides; +import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.trino.plugin.base.authentication.CachingKerberosAuthentication; +import io.trino.plugin.base.authentication.KerberosAuthentication; +import io.trino.plugin.kudu.schema.NoSchemaEmulation; +import io.trino.plugin.kudu.schema.SchemaEmulation; +import io.trino.plugin.kudu.schema.SchemaEmulationByTableNameConvention; +import org.apache.kudu.client.KuduClient; + +import javax.inject.Singleton; + +import java.util.function.Function; + +import static io.airlift.configuration.ConditionalModule.conditionalModule; +import static io.airlift.configuration.ConfigBinder.configBinder; +import static io.trino.plugin.base.util.SystemProperties.setJavaSecurityKrb5Conf; +import static io.trino.plugin.kudu.KuduAuthenticationConfig.KuduAuthenticationType.KERBEROS; +import static io.trino.plugin.kudu.KuduAuthenticationConfig.KuduAuthenticationType.NONE; +import static java.util.Objects.requireNonNull; +import static org.apache.kudu.client.KuduClient.KuduClientBuilder; + +public class KuduSecurityModule + extends AbstractConfigurationAwareModule +{ + @Override + protected void setup(Binder binder) + { + configBinder(binder).bindConfig(KuduAuthenticationConfig.class); + + install(conditionalModule( + KuduAuthenticationConfig.class, + authenticationConfig -> authenticationConfig.getAuthenticationType() == NONE, + new NoneAuthenticationModule())); + + install(conditionalModule( + KuduAuthenticationConfig.class, + authenticationConfig -> authenticationConfig.getAuthenticationType() == KERBEROS, + new KerberosAuthenticationModule())); + } + + private static class NoneAuthenticationModule + extends AbstractConfigurationAwareModule + { + @Override + public void setup(Binder binder) + { + } + + @Provides + @Singleton + public static KuduClientSession createKuduClientSession(KuduClientConfig config) + { + return KuduSecurityModule.createKuduClientSession(config, + builder -> new PassthroughKuduClient(builder.build())); + } + } + + private static class KerberosAuthenticationModule + extends AbstractConfigurationAwareModule + { + @Override + public void setup(Binder binder) + { + configBinder(binder).bindConfig(KuduKerberosConfig.class); + } + + @Provides + @Singleton + public static KuduClientSession createKuduClientSession(KuduClientConfig config, KuduKerberosConfig kuduKerberosConfig) + { + return KuduSecurityModule.createKuduClientSession(config, + builder -> { + kuduKerberosConfig.getKuduPrincipalPrimary().ifPresent(builder::saslProtocolName); + setJavaSecurityKrb5Conf(kuduKerberosConfig.getConfig().getAbsolutePath()); + KerberosAuthentication kerberosAuthentication = new KerberosAuthentication(kuduKerberosConfig.getClientPrincipal(), kuduKerberosConfig.getClientKeytab().getAbsolutePath()); + CachingKerberosAuthentication cachingKerberosAuthentication = new CachingKerberosAuthentication(kerberosAuthentication); + return new KerberizedKuduClient(builder, cachingKerberosAuthentication); + }); + } + } + + private static KuduClientSession createKuduClientSession(KuduClientConfig config, Function kuduClientFactory) + { + requireNonNull(config, "config is null"); + + KuduClient.KuduClientBuilder builder = new KuduClientBuilder(config.getMasterAddresses()); + builder.defaultAdminOperationTimeoutMs(config.getDefaultAdminOperationTimeout().toMillis()); + builder.defaultOperationTimeoutMs(config.getDefaultOperationTimeout().toMillis()); + if (config.isDisableStatistics()) { + builder.disableStatistics(); + } + KuduClientWrapper client = kuduClientFactory.apply(builder); + + SchemaEmulation strategy; + if (config.isSchemaEmulationEnabled()) { + strategy = new SchemaEmulationByTableNameConvention(config.getSchemaEmulationPrefix()); + } + else { + strategy = new NoSchemaEmulation(); + } + return new KuduClientSession(client, strategy); + } +} diff --git a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/PassthroughKuduClient.java b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/PassthroughKuduClient.java new file mode 100644 index 000000000000..e9a1cdb39a57 --- /dev/null +++ b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/PassthroughKuduClient.java @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.kudu; + +import org.apache.kudu.client.KuduClient; + +import static java.util.Objects.requireNonNull; + +public class PassthroughKuduClient + extends ForwardingKuduClient +{ + private final KuduClient kuduClient; + + PassthroughKuduClient(KuduClient kuduClient) + { + this.kuduClient = requireNonNull(kuduClient, "kuduClient is null"); + } + + @Override + protected KuduClient delegate() + { + return kuduClient; + } +} diff --git a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/schema/NoSchemaEmulation.java b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/schema/NoSchemaEmulation.java index 51e1e1288468..692db5327d3f 100644 --- a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/schema/NoSchemaEmulation.java +++ b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/schema/NoSchemaEmulation.java @@ -14,10 +14,10 @@ package io.trino.plugin.kudu.schema; import com.google.common.collect.ImmutableList; +import io.trino.plugin.kudu.KuduClientWrapper; import io.trino.spi.TrinoException; import io.trino.spi.connector.SchemaNotFoundException; import io.trino.spi.connector.SchemaTableName; -import org.apache.kudu.client.KuduClient; import java.util.List; @@ -28,7 +28,7 @@ public class NoSchemaEmulation implements SchemaEmulation { @Override - public void createSchema(KuduClient client, String schemaName) + public void createSchema(KuduClientWrapper client, String schemaName) { if (DEFAULT_SCHEMA.equals(schemaName)) { throw new SchemaAlreadyExistsException(schemaName); @@ -39,7 +39,7 @@ public void createSchema(KuduClient client, String schemaName) } @Override - public void dropSchema(KuduClient client, String schemaName) + public void dropSchema(KuduClientWrapper client, String schemaName) { if (DEFAULT_SCHEMA.equals(schemaName)) { throw new TrinoException(GENERIC_USER_ERROR, "Deleting default schema not allowed."); @@ -50,13 +50,13 @@ public void dropSchema(KuduClient client, String schemaName) } @Override - public boolean existsSchema(KuduClient client, String schemaName) + public boolean existsSchema(KuduClientWrapper client, String schemaName) { return DEFAULT_SCHEMA.equals(schemaName); } @Override - public List listSchemaNames(KuduClient client) + public List listSchemaNames(KuduClientWrapper client) { return ImmutableList.of("default"); } diff --git a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/schema/SchemaEmulation.java b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/schema/SchemaEmulation.java index 6d1cd2d39394..ff983718fa5f 100644 --- a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/schema/SchemaEmulation.java +++ b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/schema/SchemaEmulation.java @@ -13,20 +13,20 @@ */ package io.trino.plugin.kudu.schema; +import io.trino.plugin.kudu.KuduClientWrapper; import io.trino.spi.connector.SchemaTableName; -import org.apache.kudu.client.KuduClient; import java.util.List; public interface SchemaEmulation { - void createSchema(KuduClient client, String schemaName); + void createSchema(KuduClientWrapper client, String schemaName); - void dropSchema(KuduClient client, String schemaName); + void dropSchema(KuduClientWrapper client, String schemaName); - boolean existsSchema(KuduClient client, String schemaName); + boolean existsSchema(KuduClientWrapper client, String schemaName); - List listSchemaNames(KuduClient client); + List listSchemaNames(KuduClientWrapper client); String toRawName(SchemaTableName schemaTableName); diff --git a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/schema/SchemaEmulationByTableNameConvention.java b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/schema/SchemaEmulationByTableNameConvention.java index b72172664261..e5009f52b325 100644 --- a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/schema/SchemaEmulationByTableNameConvention.java +++ b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/schema/SchemaEmulationByTableNameConvention.java @@ -14,6 +14,7 @@ package io.trino.plugin.kudu.schema; import com.google.common.collect.ImmutableList; +import io.trino.plugin.kudu.KuduClientWrapper; import io.trino.spi.TrinoException; import io.trino.spi.connector.SchemaNotFoundException; import io.trino.spi.connector.SchemaTableName; @@ -23,7 +24,6 @@ import org.apache.kudu.client.CreateTableOptions; import org.apache.kudu.client.Delete; import org.apache.kudu.client.Insert; -import org.apache.kudu.client.KuduClient; import org.apache.kudu.client.KuduException; import org.apache.kudu.client.KuduScanner; import org.apache.kudu.client.KuduSession; @@ -56,7 +56,7 @@ public SchemaEmulationByTableNameConvention(String commonPrefix) } @Override - public void createSchema(KuduClient client, String schemaName) + public void createSchema(KuduClientWrapper client, String schemaName) { if (DEFAULT_SCHEMA.equals(schemaName)) { throw new SchemaAlreadyExistsException(schemaName); @@ -81,7 +81,7 @@ public void createSchema(KuduClient client, String schemaName) } @Override - public boolean existsSchema(KuduClient client, String schemaName) + public boolean existsSchema(KuduClientWrapper client, String schemaName) { if (DEFAULT_SCHEMA.equals(schemaName)) { return true; @@ -93,7 +93,7 @@ public boolean existsSchema(KuduClient client, String schemaName) } @Override - public void dropSchema(KuduClient client, String schemaName) + public void dropSchema(KuduClientWrapper client, String schemaName) { if (DEFAULT_SCHEMA.equals(schemaName)) { throw new TrinoException(GENERIC_USER_ERROR, "Deleting default schema not allowed."); @@ -123,7 +123,7 @@ public void dropSchema(KuduClient client, String schemaName) } @Override - public List listSchemaNames(KuduClient client) + public List listSchemaNames(KuduClientWrapper client) { try { if (rawSchemasTable == null) { @@ -149,7 +149,7 @@ public List listSchemaNames(KuduClient client) } } - private KuduTable getSchemasTable(KuduClient client) + private KuduTable getSchemasTable(KuduClientWrapper client) throws KuduException { if (rawSchemasTable == null) { @@ -158,7 +158,7 @@ private KuduTable getSchemasTable(KuduClient client) return rawSchemasTable; } - private void createAndFillSchemasTable(KuduClient client) + private void createAndFillSchemasTable(KuduClientWrapper client) throws KuduException { List existingSchemaNames = listSchemaNamesFromTablets(client); @@ -181,7 +181,7 @@ private void createAndFillSchemasTable(KuduClient client) } } - private List listSchemaNamesFromTablets(KuduClient client) + private List listSchemaNamesFromTablets(KuduClientWrapper client) throws KuduException { List tables = client.getTablesList().getTablesList(); diff --git a/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestForwardingKuduClient.java b/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestForwardingKuduClient.java new file mode 100644 index 000000000000..0e833e3682d1 --- /dev/null +++ b/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestForwardingKuduClient.java @@ -0,0 +1,27 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.kudu; + +import org.testng.annotations.Test; + +import static io.trino.spi.testing.InterfaceTestUtils.assertAllMethodsOverridden; + +public class TestForwardingKuduClient +{ + @Test + public void testEverythingDelegated() + { + assertAllMethodsOverridden(KuduClientWrapper.class, ForwardingKuduClient.class); + } +} diff --git a/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestKuduAuthenticationConfig.java b/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestKuduAuthenticationConfig.java new file mode 100644 index 000000000000..857702489abc --- /dev/null +++ b/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestKuduAuthenticationConfig.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.kudu; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; + +import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +public class TestKuduAuthenticationConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(KuduAuthenticationConfig.class) + .setAuthenticationType(KuduAuthenticationConfig.KuduAuthenticationType.NONE)); + } + + @Test + public void testExplicitPropertyMappingsKerberos() + { + Map properties = new ImmutableMap.Builder() + .put("kudu.authentication.type", "KERBEROS") + .buildOrThrow(); + + KuduAuthenticationConfig expected = new KuduAuthenticationConfig() + .setAuthenticationType(KuduAuthenticationConfig.KuduAuthenticationType.KERBEROS); + + assertFullMapping(properties, expected); + } +} diff --git a/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestKuduKerberosConfig.java b/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestKuduKerberosConfig.java new file mode 100644 index 000000000000..1c518ab70039 --- /dev/null +++ b/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestKuduKerberosConfig.java @@ -0,0 +1,83 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.kudu; + +import com.google.common.collect.ImmutableMap; +import com.google.inject.ConfigurationException; +import io.airlift.configuration.ConfigurationFactory; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; + +import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static java.nio.file.Files.createTempFile; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; + +public class TestKuduKerberosConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(KuduKerberosConfig.class) + .setClientPrincipal(null) + .setClientKeytab(null) + .setConfig(null) + .setKuduPrincipalPrimary(null)); + } + + @Test + public void testExplicitPropertyMappings() + throws IOException + { + Path keytab = createTempFile(null, null); + Path config = createTempFile(null, null); + + Map properties = new ImmutableMap.Builder() + .put("kudu.authentication.client.principal", "principal") + .put("kudu.authentication.client.keytab", keytab.toString()) + .put("kudu.authentication.config", config.toString()) + .put("kudu.authentication.server.principal.primary", "principal-primary") + .buildOrThrow(); + + KuduKerberosConfig expected = new KuduKerberosConfig() + .setClientPrincipal("principal") + .setClientKeytab(keytab.toFile()) + .setConfig(config.toFile()) + .setKuduPrincipalPrimary("principal-primary"); + + assertFullMapping(properties, expected); + } + + @Test + public void testExplicitPropertyMappingsWithNonExistentPathsThrowsErrors() + { + Map properties = new ImmutableMap.Builder() + .put("kudu.authentication.client.principal", "principal") + .put("kudu.authentication.client.keytab", "/path/does/not/exist") + .put("kudu.authentication.config", "/path/does/not/exist") + .put("kudu.authentication.server.principal.primary", "principal-primary") + .buildOrThrow(); + + ConfigurationFactory configurationFactory = new ConfigurationFactory(properties); + assertThatThrownBy(() -> configurationFactory.build(KuduKerberosConfig.class)) + .isInstanceOf(ConfigurationException.class) + .hasMessageContainingAll( + "Invalid configuration property kudu.authentication.config: file does not exist", + "Invalid configuration property kudu.authentication.client.keytab: file does not exist"); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentModule.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentModule.java index 0fa323fb168b..989911c8740c 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentModule.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentModule.java @@ -24,6 +24,7 @@ import io.trino.tests.product.launcher.env.common.HydraIdentityProvider; import io.trino.tests.product.launcher.env.common.Kafka; import io.trino.tests.product.launcher.env.common.KafkaSsl; +import io.trino.tests.product.launcher.env.common.Kerberos; import io.trino.tests.product.launcher.env.common.Phoenix; import io.trino.tests.product.launcher.env.common.Standard; import io.trino.tests.product.launcher.env.common.StandardMultinode; @@ -71,6 +72,7 @@ public void configure(Binder binder) binder.bind(Standard.class).in(SINGLETON); binder.bind(StandardMultinode.class).in(SINGLETON); binder.bind(Phoenix.class).in(SINGLETON); + binder.bind(Kerberos.class).in(SINGLETON); MapBinder environments = newMapBinder(binder, String.class, EnvironmentProvider.class); findEnvironmentsByBasePackage(ENVIRONMENT_PACKAGE).forEach(clazz -> environments.addBinding(nameForEnvironmentClass(clazz)).to(clazz).in(SINGLETON)); diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Kerberos.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Kerberos.java new file mode 100644 index 000000000000..6ea6fea3052c --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Kerberos.java @@ -0,0 +1,65 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.env.common; + +import io.trino.tests.product.launcher.env.DockerContainer; +import io.trino.tests.product.launcher.env.Environment; +import io.trino.tests.product.launcher.env.EnvironmentConfig; +import io.trino.tests.product.launcher.testcontainers.PortBinder; +import org.testcontainers.containers.startupcheck.IsRunningStartupCheckStrategy; +import org.testcontainers.containers.wait.strategy.WaitAllStrategy; +import org.testcontainers.containers.wait.strategy.WaitStrategy; + +import javax.inject.Inject; + +import static io.trino.tests.product.launcher.docker.ContainerUtil.forSelectedPorts; +import static java.util.Objects.requireNonNull; +import static org.testcontainers.containers.wait.strategy.Wait.forLogMessage; + +public class Kerberos + implements EnvironmentExtender +{ + private static final int KERBEROS_PORT = 88; + private static final int KERBEROS_ADMIN_PORT = 89; + + public static final String KERBEROS = "kerberos"; + + public static final WaitStrategy DEFAULT_WAIT_STRATEGY = new WaitAllStrategy() + .withStrategy(forSelectedPorts(KERBEROS_PORT, KERBEROS_ADMIN_PORT)) + .withStrategy(forLogMessage(".*krb5kdc entered RUNNING state.*", 1)); + + private final PortBinder portBinder; + private final String imagesVersion; + + @Inject + public Kerberos(EnvironmentConfig environmentConfig, PortBinder portBinder) + { + this.portBinder = requireNonNull(portBinder, "portBinder is null"); + imagesVersion = requireNonNull(environmentConfig, "environmentConfig is null").getImagesVersion(); + } + + @Override + @SuppressWarnings("resource") + public void extendEnvironment(Environment.Builder builder) + { + DockerContainer container = new DockerContainer("ghcr.io/trinodb/testing/kerberos:" + imagesVersion, KERBEROS) + .withStartupCheckStrategy(new IsRunningStartupCheckStrategy()) + .waitingFor(DEFAULT_WAIT_STRATEGY); + + portBinder.exposePort(container, KERBEROS_PORT); + portBinder.exposePort(container, KERBEROS_ADMIN_PORT); + + builder.addContainer(container); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeKerberosKudu.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeKerberosKudu.java new file mode 100644 index 000000000000..4548f98fcc6b --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeKerberosKudu.java @@ -0,0 +1,152 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.env.environment; + +import io.trino.tests.product.launcher.docker.DockerFiles; +import io.trino.tests.product.launcher.env.DockerContainer; +import io.trino.tests.product.launcher.env.Environment; +import io.trino.tests.product.launcher.env.EnvironmentProvider; +import io.trino.tests.product.launcher.env.common.Kerberos; +import io.trino.tests.product.launcher.env.common.StandardMultinode; +import io.trino.tests.product.launcher.env.common.TestsEnvironment; +import io.trino.tests.product.launcher.testcontainers.PortBinder; + +import javax.inject.Inject; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import static io.trino.tests.product.launcher.docker.ContainerUtil.forSelectedPorts; +import static io.trino.tests.product.launcher.env.EnvironmentContainers.isPrestoContainer; +import static io.trino.tests.product.launcher.env.common.Kerberos.DEFAULT_WAIT_STRATEGY; +import static io.trino.tests.product.launcher.env.common.Kerberos.KERBEROS; +import static io.trino.tests.product.launcher.env.common.Standard.CONTAINER_PRESTO_ETC; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; +import static org.testcontainers.containers.BindMode.READ_ONLY; +import static org.testcontainers.containers.BindMode.READ_WRITE; +import static org.testcontainers.containers.wait.strategy.Wait.forLogMessage; +import static org.testcontainers.utility.MountableFile.forHostPath; + +@TestsEnvironment +public class EnvMultinodeKerberosKudu + extends EnvironmentProvider +{ + private static final String KUDU_IMAGE = "apache/kudu:1.15.0"; + private static final Integer KUDU_MASTER_PORT = 7051; + private static final Integer NUMBER_OF_REPLICA = 3; + private static final String KUDU_MASTER = "kudu-master"; + private static final String KUDU_TABLET_TEMPLATE = "kudu-tserver-%s"; + + private static Integer initialKuduTserverPort = 7060; + + private final PortBinder portBinder; + private final DockerFiles.ResourceProvider configDir; + + @Inject + public EnvMultinodeKerberosKudu(PortBinder portBinder, DockerFiles dockerFiles, StandardMultinode standardMultinode, Kerberos kerberos) + { + super(standardMultinode, kerberos); + this.portBinder = requireNonNull(portBinder, "portBinder is null"); + configDir = requireNonNull(dockerFiles, "dockerFiles is null").getDockerFilesHostDirectory("conf/environment/multinode-kerberos-kudu/"); + } + + @Override + public void extendEnvironment(Environment.Builder builder) + { + Path kerberosCredentialsDirectory = DockerFiles.createTemporaryDirectoryForDocker(); + + builder.configureContainer(KERBEROS, container -> container + .withFileSystemBind(kerberosCredentialsDirectory.toString(), "/kerberos", READ_WRITE) + .withCopyFileToContainer(forHostPath(configDir.getPath("kerberos_init.sh"), 0777), "/docker/kerberos-init.d/kerberos_init.sh") + .waitingForAll( + DEFAULT_WAIT_STRATEGY, + forLogMessage(".*Kerberos init script completed successfully.*", 1))); + + DockerContainer kuduMaster = createKuduMaster(kerberosCredentialsDirectory); + List kuduTablets = createKuduTablets(kerberosCredentialsDirectory, kuduMaster); + + builder.addContainer(kuduMaster); + builder.containerDependsOn(kuduMaster.getLogicalName(), KERBEROS); + + kuduTablets.forEach(tablet -> { + builder.addContainer(tablet); + builder.containerDependsOn(tablet.getLogicalName(), KERBEROS); + }); + + builder.configureContainers(container -> { + if (isPrestoContainer(container.getLogicalName())) { + configurePrestoContainer(container, kerberosCredentialsDirectory); + addKrb5(container); + } + }); + } + + private DockerContainer createKuduMaster(Path kerberosCredentialsDirectory) + { + DockerContainer container = new DockerContainer(KUDU_IMAGE, KUDU_MASTER) + .withCommand("master") + .withEnv("MASTER_ARGS", format("--fs_wal_dir=/var/lib/kudu/master --logtostderr --use_hybrid_clock=false --rpc_authentication=required --rpc_bind_addresses=%s:%s --rpc_authentication=required --principal=kuduservice/kudu-master@STARBURSTDATA.COM --keytab_file=/kerberos/kudu-master.keytab", KUDU_MASTER, KUDU_MASTER_PORT)) + .withFileSystemBind(kerberosCredentialsDirectory.toString(), "/kerberos", READ_ONLY) + .waitingFor(forSelectedPorts(KUDU_MASTER_PORT)); + + addKrb5(container); + + portBinder.exposePort(container, KUDU_MASTER_PORT); + + return container; + } + + private List createKuduTablets(Path kerberosCredentialsDirectory, DockerContainer kuduMaster) + { + List tabletContainers = new ArrayList<>(); + + for (int i = 0; i < NUMBER_OF_REPLICA; i++) { + String instanceName = format(KUDU_TABLET_TEMPLATE, i); + DockerContainer kuduTablet = new DockerContainer(KUDU_IMAGE, instanceName) + .withCommand("tserver") + .withEnv("KUDU_MASTERS", format("%s:%s", KUDU_MASTER, KUDU_MASTER_PORT)) + .withEnv("TSERVER_ARGS", format("--fs_wal_dir=/var/lib/kudu/tserver --logtostderr --use_hybrid_clock=false --rpc_authentication=required --rpc_bind_addresses=%s:%s --rpc_authentication=required --principal=kuduservice/kudu-tserver-%s@STARBURSTDATA.COM --keytab_file=/kerberos/kudu-tserver-%s.keytab", instanceName, initialKuduTserverPort, i, i)) + .withFileSystemBind(kerberosCredentialsDirectory.toString(), "/kerberos", READ_ONLY) + .waitingFor(forSelectedPorts(initialKuduTserverPort)) + .dependsOn(kuduMaster); + + addKrb5(kuduTablet); + + portBinder.exposePort(kuduTablet, initialKuduTserverPort); + tabletContainers.add(kuduTablet); + initialKuduTserverPort += 1; + } + + return tabletContainers; + } + + private void addKrb5(DockerContainer container) + { + container + .withCopyFileToContainer( + forHostPath(configDir.getPath("krb5.conf")), + "/etc/krb5.conf"); + } + + private void configurePrestoContainer(DockerContainer container, Path kerberosCredentialsDirectory) + { + container + .withCopyFileToContainer( + forHostPath(configDir.getPath("kudu.properties")), + CONTAINER_PRESTO_ETC + "/catalog/kudu.properties") + .withFileSystemBind(kerberosCredentialsDirectory.toString(), "/kerberos", READ_ONLY); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite7NonGeneric.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite7NonGeneric.java index e40a90de9438..e404dd22dbfc 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite7NonGeneric.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite7NonGeneric.java @@ -17,6 +17,7 @@ import io.trino.tests.product.launcher.env.EnvironmentConfig; import io.trino.tests.product.launcher.env.EnvironmentDefaults; import io.trino.tests.product.launcher.env.environment.EnvMultinodeClickhouse; +import io.trino.tests.product.launcher.env.environment.EnvMultinodeKerberosKudu; import io.trino.tests.product.launcher.env.environment.EnvSinglenodeHiveIcebergRedirections; import io.trino.tests.product.launcher.env.environment.EnvSinglenodeKerberosHdfsImpersonationCrossRealm; import io.trino.tests.product.launcher.env.environment.EnvSinglenodeMysql; @@ -51,6 +52,7 @@ public List getTestRuns(EnvironmentConfig config) testOnEnvironment(EnvSinglenodeSparkIceberg.class).withGroups("iceberg").withExcludedGroups("storage_formats").build(), testOnEnvironment(EnvSinglenodeHiveIcebergRedirections.class).withGroups("hive_iceberg_redirections").build(), testOnEnvironment(EnvSinglenodeKerberosHdfsImpersonationCrossRealm.class).withGroups("storage_formats", "cli", "hdfs_impersonation").build(), + testOnEnvironment(EnvMultinodeKerberosKudu.class).withGroups("kudu").build(), testOnEnvironment(EnvTwoMixedHives.class).withGroups("two_hives").build(), testOnEnvironment(EnvTwoKerberosHives.class).withGroups("two_hives").build()); } diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kerberos-kudu/kerberos_init.sh b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kerberos-kudu/kerberos_init.sh new file mode 100644 index 000000000000..7d94b69a3b81 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kerberos-kudu/kerberos_init.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Set the max ticket lifetime, this is to facilitate testing of renewing kerberos tickets after expiry +MAX_TICKET_LIFETIME="1min" + +function create_principal_with_forward_and_reverse_dns_entries() { + PRIMARY="$1" + INSTANCE="$2" + KEYTAB="$3" + REALM="STARBURSTDATA.COM" + + PRINCIPAL="$PRIMARY/$INSTANCE@$REALM" + /usr/sbin/kadmin.local -q "addprinc -randkey -maxlife $MAX_TICKET_LIFETIME $PRINCIPAL" && \ + /usr/sbin/kadmin.local -q "xst -norandkey -k $KEYTAB $PRINCIPAL" + + # Create a separate principal in the same keytab + # This is because the java kerberos client impl does not support rdns=false and the forward/reverse dns entries + # do not match when running the product tests in docker due to the container naming and docker network + # See here for details: https://web.mit.edu/kerberos/krb5-1.12/doc/admin/princ_dns.html#provisioning-keytabs + PRINCIPAL="$PRIMARY/ptl-$INSTANCE.ptl-network@$REALM" + /usr/sbin/kadmin.local -q "addprinc -randkey -maxlife $MAX_TICKET_LIFETIME $PRINCIPAL" && \ + /usr/sbin/kadmin.local -q "xst -norandkey -k $KEYTAB $PRINCIPAL" +} + +# Modify ticket granting ticket principal max lifetime +/usr/sbin/kadmin.local -q "modprinc -maxlife $MAX_TICKET_LIFETIME krbtgt/STARBURSTDATA.COM@STARBURSTDATA.COM" + +create_principal_with_forward_and_reverse_dns_entries kuduservice kudu-master /kerberos/kudu-master.keytab + +create_principal_with_forward_and_reverse_dns_entries kuduservice kudu-tserver-0 /kerberos/kudu-tserver-0.keytab +create_principal_with_forward_and_reverse_dns_entries kuduservice kudu-tserver-1 /kerberos/kudu-tserver-1.keytab +create_principal_with_forward_and_reverse_dns_entries kuduservice kudu-tserver-2 /kerberos/kudu-tserver-2.keytab + +# Create principal and keytab for the trino connector to use +create_principal -p test -k /kerberos/test.keytab +/usr/sbin/kadmin.local -q "modprinc -maxlife $MAX_TICKET_LIFETIME test@STARBURSTDATA.COM" + +# The kudu container runs as: uid=1000(kudu) gid=1000(kudu) +chown -R 1000:1000 /kerberos +chmod -R 770 /kerberos +echo "Kerberos init script completed successfully" diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kerberos-kudu/krb5.conf b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kerberos-kudu/krb5.conf new file mode 100644 index 000000000000..9e7a39354128 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kerberos-kudu/krb5.conf @@ -0,0 +1,25 @@ +[logging] + default = FILE:/var/log/krb5libs.log + kdc = FILE:/var/log/krb5kdc.log + admin_server = FILE:/var/log/kadmind.log + +[libdefaults] + default_realm = STARBURSTDATA.COM + dns_lookup_realm = false + dns_lookup_kdc = false + forwardable = true + # Forward and reverse dns entries for the kudu servers do not match in the product test docker setup + # Kerberos will attempt to set the realm to PTL-NETWORK (the name of the docker network used by the product tests) + # This config allows kudu to accept incoming authentications using any key in its keytab that matches the service name and realm name + # See here for more details: https://web.mit.edu/kerberos/krb5-1.12/doc/admin/princ_dns.html#overriding-application-behavior + ignore_acceptor_hostname = true + +[realms] + STARBURSTDATA.COM = { + kdc = kerberos:88 + admin_server = kerberos + } + OTHER.STARBURSTDATA.COM = { + kdc = kerberos:89 + admin_server = kerberos + } diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kerberos-kudu/kudu.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kerberos-kudu/kudu.properties new file mode 100644 index 000000000000..6d77db57ecf3 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kerberos-kudu/kudu.properties @@ -0,0 +1,7 @@ +connector.name=kudu +kudu.client.master-addresses=kudu-master:7051 +kudu.authentication.type=KERBEROS +kudu.authentication.client.principal=test@STARBURSTDATA.COM +kudu.authentication.client.keytab=/kerberos/test.keytab +kudu.authentication.config=/etc/krb5.conf +kudu.authentication.server.principal.primary=kuduservice diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestGroups.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestGroups.java index f5e605af1507..f137d35b36a2 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestGroups.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestGroups.java @@ -68,6 +68,7 @@ public final class TestGroups public static final String AVRO = "avro"; public static final String PHOENIX = "phoenix"; public static final String CLICKHOUSE = "clickhouse"; + public static final String KUDU = "kudu"; private TestGroups() {} } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/kudu/TestKuduConnectoKerberosSmokeTest.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/kudu/TestKuduConnectoKerberosSmokeTest.java new file mode 100644 index 000000000000..2551b54dbbd6 --- /dev/null +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/kudu/TestKuduConnectoKerberosSmokeTest.java @@ -0,0 +1,49 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.kudu; + +import io.trino.tempto.query.QueryResult; +import org.testng.annotations.Test; + +import java.util.UUID; + +import static io.trino.tempto.assertions.QueryAssert.Row.row; +import static io.trino.tempto.assertions.QueryAssert.assertThat; +import static io.trino.tests.product.TestGroups.KUDU; +import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; +import static io.trino.tests.product.utils.QueryExecutors.onTrino; +import static java.lang.String.format; + +public class TestKuduConnectoKerberosSmokeTest +{ + @Test(groups = {KUDU, PROFILE_SPECIFIC_TESTS}) + public void kerberosAuthTicketExpiryTest() throws InterruptedException + { + String kuduTable = "kudu.default.nation_" + UUID.randomUUID().toString().replace("-", ""); + String table = "tpch.tiny.nation"; + + assertThat(onTrino().executeQuery(format("SELECT count(*) from %s", table))).containsExactlyInOrder(row(25)); + QueryResult result = onTrino().executeQuery(format("CREATE TABLE %s AS SELECT * FROM %s", kuduTable, table)); + try { + assertThat(result).updatedRowsCountIsEqualTo(25); + assertThat(onTrino().executeQuery(format("SELECT count(*) FROM %s", kuduTable))).containsExactlyInOrder(row(25)); + // Kerberos tickets are configured to expire after 60 seconds, this should expire the ticket + Thread.sleep(70_000L); + assertThat(onTrino().executeQuery(format("SELECT count(*) FROM %s", kuduTable))).containsExactlyInOrder(row(25)); + } + finally { + onTrino().executeQuery(format("DROP TABLE %s", kuduTable)); + } + } +}