diff --git a/testing/trino-product-tests-launcher/pom.xml b/testing/trino-product-tests-launcher/pom.xml index e8bcdc43f50d..a16828d53113 100644 --- a/testing/trino-product-tests-launcher/pom.xml +++ b/testing/trino-product-tests-launcher/pom.xml @@ -146,6 +146,19 @@ testcontainers + + com.databricks + databricks-jdbc + 2.6.32 + runtime + + + * + * + + + + io.confluent kafka-protobuf-provider @@ -172,6 +185,20 @@ + + org.apache.hive + hive-jdbc + 3.1.3 + standalone + runtime + + + * + * + + + + io.trino trino-jdbc @@ -193,6 +220,24 @@ + + org.basepom.maven + duplicate-finder-maven-plugin + + + + com.databricks + databricks-jdbc + + + + org.apache.hive + hive-jdbc + + + + + org.apache.maven.plugins maven-shade-plugin @@ -254,6 +299,21 @@ jar ${project.build.directory} + + org.apache.hive + hive-jdbc + standalone + jar + ${project.build.directory} + hive-jdbc.jar + + + com.databricks + databricks-jdbc + jar + ${project.build.directory} + databricks-jdbc.jar + diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/AbstractSinglenodeDeltaLakeDatabricks.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/AbstractSinglenodeDeltaLakeDatabricks.java index ee040a083a75..ef7666f5cbb1 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/AbstractSinglenodeDeltaLakeDatabricks.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/AbstractSinglenodeDeltaLakeDatabricks.java @@ -19,6 +19,8 @@ import io.trino.tests.product.launcher.env.EnvironmentProvider; import io.trino.tests.product.launcher.env.common.Standard; +import java.io.File; + import static io.trino.tests.product.launcher.env.EnvironmentContainers.COORDINATOR; import static io.trino.tests.product.launcher.env.EnvironmentContainers.TESTS; import static io.trino.tests.product.launcher.env.EnvironmentContainers.configureTempto; @@ -33,6 +35,8 @@ public abstract class AbstractSinglenodeDeltaLakeDatabricks extends EnvironmentProvider { + private static final File DATABRICKS_JDBC_PROVIDER = new File("testing/trino-product-tests-launcher/target/databricks-jdbc.jar"); + private final DockerFiles dockerFiles; abstract String databricksTestJdbcUrl(); @@ -69,7 +73,10 @@ public void extendEnvironment(Environment.Builder builder) .withEnv("AWS_REGION", awsRegion) .withEnv("DATABRICKS_JDBC_URL", databricksTestJdbcUrl) .withEnv("DATABRICKS_LOGIN", databricksTestLogin) - .withEnv("DATABRICKS_TOKEN", databricksTestToken)); + .withEnv("DATABRICKS_TOKEN", databricksTestToken) + .withCopyFileToContainer( + forHostPath(DATABRICKS_JDBC_PROVIDER.getAbsolutePath()), + "/docker/jdbc/databricks-jdbc.jar")); configureTempto(builder, configDir); } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeOss.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeOss.java index 068770667ae3..c327ae6e512a 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeOss.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeOss.java @@ -25,8 +25,10 @@ import io.trino.tests.product.launcher.env.common.Standard; import io.trino.tests.product.launcher.env.common.TestsEnvironment; import io.trino.tests.product.launcher.testcontainers.PortBinder; +import org.testcontainers.containers.BindMode; import org.testcontainers.containers.startupcheck.IsRunningStartupCheckStrategy; +import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.Files; @@ -55,6 +57,8 @@ public class EnvSinglenodeDeltaLakeOss extends EnvironmentProvider { + private static final File HIVE_JDBC_PROVIDER = new File("testing/trino-product-tests-launcher/target/hive-jdbc.jar"); + private static final int SPARK_THRIFT_PORT = 10213; private static final String SPARK_CONTAINER_NAME = "spark"; @@ -92,7 +96,9 @@ public void extendEnvironment(Environment.Builder builder) CONTAINER_TRINO_ETC + "/catalog/delta.properties"); builder.configureContainer(TESTS, dockerContainer -> { - dockerContainer.withEnv("S3_BUCKET", S3_BUCKET_NAME); + dockerContainer.withEnv("S3_BUCKET", S3_BUCKET_NAME) + // Binding instead of copying for avoiding OutOfMemoryError https://github.com/testcontainers/testcontainers-java/issues/2863 + .withFileSystemBind(HIVE_JDBC_PROVIDER.getParent(), "/docker/jdbc", BindMode.READ_ONLY); }); builder.addContainer(createSparkContainer()) diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/tempto-configuration.yaml b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/tempto-configuration.yaml index 048e3c6e552e..6f8071225ac6 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/tempto-configuration.yaml +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/tempto-configuration.yaml @@ -3,6 +3,7 @@ databases: jdbc_user: root delta: jdbc_driver_class: com.databricks.client.jdbc.Driver + jdbc_jar: /docker/jdbc/databricks-jdbc.jar schema: default prepare_statement: - USE ${databases.delta.schema} diff --git a/testing/trino-product-tests/pom.xml b/testing/trino-product-tests/pom.xml index 534fa66cdfa0..a2a3836c9799 100644 --- a/testing/trino-product-tests/pom.xml +++ b/testing/trino-product-tests/pom.xml @@ -38,12 +38,6 @@ java-driver-core - - com.google.code.findbugs - jsr305 - true - - com.google.guava guava @@ -110,11 +104,6 @@ units - - io.airlift.resolver - resolver - - io.confluent kafka-protobuf-provider @@ -239,11 +228,6 @@ annotations - - org.sonatype.aether - aether-api - - org.testng testng diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/utils/DeltaQueryExecutors.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/utils/DeltaQueryExecutors.java deleted file mode 100644 index a91bb6dcaa91..000000000000 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/utils/DeltaQueryExecutors.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.tests.product.utils; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import dev.failsafe.Failsafe; -import dev.failsafe.RetryPolicy; -import io.airlift.log.Logger; -import io.airlift.resolver.ArtifactResolver; -import io.airlift.resolver.DefaultArtifact; -import io.trino.tempto.context.TestContext; -import io.trino.tempto.query.JdbcConnectionsPool; -import io.trino.tempto.query.JdbcConnectivityParamsState; -import io.trino.tempto.query.JdbcQueryExecutor; - -import javax.annotation.concurrent.GuardedBy; - -import java.time.Duration; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -import static com.google.common.collect.MoreCollectors.onlyElement; -import static io.airlift.resolver.ArtifactResolver.MAVEN_CENTRAL_URI; -import static io.airlift.resolver.ArtifactResolver.USER_LOCAL_REPO; -import static java.util.Map.entry; - -public final class DeltaQueryExecutors -{ - private static final Logger log = Logger.get(DeltaQueryExecutors.class); - - private static final Map> ARTIFACTS = ImmutableMap.>builder() - .put("org.apache.hive.jdbc.HiveDriver", entry("org.apache.hive:hive-jdbc:jar:standalone:3.1.3", "hive-jdbc-3.1.3-standalone.jar")) - .put("com.databricks.client.jdbc.Driver", entry("com.databricks:databricks-jdbc:2.6.32", "databricks-jdbc-2.6.32.jar")) - .buildOrThrow(); - - @GuardedBy("DRIVERS") - private static final Map DRIVERS = new HashMap<>(); - @GuardedBy("JDBC_EXECUTORS") - private static final Map JDBC_EXECUTORS = new HashMap<>(); - - private static final RetryPolicy loadDatabaseDriverRetryPolicy = RetryPolicy.builder() - .withMaxRetries(30) - .withDelay(Duration.ofSeconds(10)) - .onRetry(event -> log.warn(event.getLastException(), "Download failed on attempt %d, will retry.", event.getAttemptCount())) - .build(); - - private DeltaQueryExecutors() {} - - public static JdbcQueryExecutor createDeltaQueryExecutor(TestContext testContext) - { - JdbcConnectivityParamsState jdbcParamsState = testContext.getDependency(JdbcConnectivityParamsState.class, "delta"); - JdbcConnectionsPool jdbcConnectionsPool = testContext.getDependency(JdbcConnectionsPool.class); - synchronized (JDBC_EXECUTORS) { - return JDBC_EXECUTORS.computeIfAbsent(jdbcParamsState, param -> new JdbcQueryExecutor(withRuntimeJar(param), jdbcConnectionsPool, testContext)); - } - } - - private static JdbcConnectivityParamsState withRuntimeJar(JdbcConnectivityParamsState jdbcParamsState) - { - String jarFilePath; - synchronized (DRIVERS) { - jarFilePath = DRIVERS.computeIfAbsent(jdbcParamsState.driverClass, className -> Failsafe.with(loadDatabaseDriverRetryPolicy) - .get(() -> loadDatabaseDriverJar(className))); - } - - return JdbcConnectivityParamsState.builder() - .setName(jdbcParamsState.getName().orElseThrow()) - .setDriverClass(jdbcParamsState.driverClass) - .setUrl(jdbcParamsState.url) - .setUser(jdbcParamsState.user) - .setPassword(jdbcParamsState.password) - .setPooling(jdbcParamsState.pooling) - .setJar(Optional.of(jarFilePath)) - .setPrepareStatements(jdbcParamsState.prepareStatements) - .setKerberosPrincipal(jdbcParamsState.kerberosPrincipal) - .setKerberosKeytab(jdbcParamsState.kerberosKeytab) - .build(); - } - - private static String loadDatabaseDriverJar(String driverClassName) - { - // TODO Add support for maven coordinate in tempto - String coords = ARTIFACTS.get(driverClassName).getKey(); - String jar = ARTIFACTS.get(driverClassName).getValue(); - - ArtifactResolver resolver = new ArtifactResolver(USER_LOCAL_REPO, ImmutableList.of(MAVEN_CENTRAL_URI)); - return resolver.resolveArtifacts(new DefaultArtifact(coords)).stream() - .filter(artifact -> artifact.getFile().getName().equals(jar)) - .map(artifact -> artifact.getFile().getAbsolutePath()) - .collect(onlyElement()); - } -} diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/utils/QueryExecutors.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/utils/QueryExecutors.java index a89ddbf8e637..b9028155877b 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/utils/QueryExecutors.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/utils/QueryExecutors.java @@ -24,7 +24,6 @@ import java.time.temporal.ChronoUnit; import static io.trino.tempto.context.ThreadLocalTestContextHolder.testContext; -import static io.trino.tests.product.utils.DeltaQueryExecutors.createDeltaQueryExecutor; import static io.trino.tests.product.utils.HadoopTestUtils.ERROR_COMMITTING_WRITE_TO_HIVE_RETRY_POLICY; public final class QueryExecutors @@ -127,7 +126,7 @@ public static QueryExecutor onDelta() return new QueryExecutor() { - private final QueryExecutor delegate = createDeltaQueryExecutor(testContext()); + private final QueryExecutor delegate = testContext().getDependency(QueryExecutor.class, "delta"); @Override public QueryResult executeQuery(String sql, QueryParam... params) diff --git a/testing/trino-product-tests/src/main/resources/tempto-configuration.yaml b/testing/trino-product-tests/src/main/resources/tempto-configuration.yaml index 45989ef16dfa..a0ab89b98d76 100644 --- a/testing/trino-product-tests/src/main/resources/tempto-configuration.yaml +++ b/testing/trino-product-tests/src/main/resources/tempto-configuration.yaml @@ -171,6 +171,7 @@ databases: delta: jdbc_driver_class: org.apache.hive.jdbc.HiveDriver + jdbc_jar: /docker/jdbc/hive-jdbc.jar schema: default prepare_statement: - USE ${databases.delta.schema}