Add Delta Lake product tests

.github/workflows/ci.yml

@@ -669,6 +669,10 @@ jobs:
           - config: default
             suite: suite-all
             jdk: 11
+          # this suite is not meant to be run with different configs
+          - config: default
+            suite: suite-delta-lake
+            jdk: 11
     # PT Launcher's timeout defaults to 2h, add some margin
     timeout-minutes: 130
     needs: build-pt

plugin/trino-delta-lake/pom.xml

@@ -271,6 +271,12 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>io.trino</groupId>
+            <artifactId>trino-testing-resources</artifactId>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>io.trino</groupId>
             <artifactId>trino-testing-services</artifactId>
@@ -338,13 +344,6 @@
             <scope>test</scope>
         </dependency>
 
-        <dependency>
-            <groupId>io.minio</groupId>
-            <artifactId>minio</artifactId>
-            <version>7.1.4</version>
-            <scope>test</scope>
-        </dependency>
-
         <dependency>
             <groupId>org.assertj</groupId>
             <artifactId>assertj-core</artifactId>

plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/util/DockerizedMinioDataLake.java

@@ -13,6 +13,7 @@
  */
 package io.trino.plugin.deltalake.util;
 
+import io.trino.testing.minio.MinioClient;
 import net.jodah.failsafe.Failsafe;
 import net.jodah.failsafe.RetryPolicy;
 

pom.xml

@@ -66,8 +66,9 @@
         <dep.coral.version>2.0.55</dep.coral.version>
         <dep.confluent.version>5.5.2</dep.confluent.version>
         <dep.casandra.version>4.14.0</dep.casandra.version>
+        <dep.minio.version>7.1.4</dep.minio.version>
 
-        <dep.docker.images.version>56</dep.docker.images.version>
+        <dep.docker.images.version>57</dep.docker.images.version>
 
         <!--
           America/Bahia_Banderas has:
@@ -168,6 +169,7 @@
         <module>testing/trino-testing</module>
         <module>testing/trino-testing-containers</module>
         <module>testing/trino-testing-kafka</module>
+        <module>testing/trino-testing-resources</module>
         <module>testing/trino-testing-services</module>
         <module>testing/trino-tests</module>
     </modules>
@@ -536,6 +538,12 @@
                 <version>${project.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>io.trino</groupId>
+                <artifactId>trino-testing-resources</artifactId>
+                <version>${project.version}</version>
+            </dependency>
+
             <dependency>
                 <groupId>io.trino</groupId>
                 <artifactId>trino-testing-services</artifactId>
@@ -1002,6 +1010,12 @@
                 </exclusions>
             </dependency>
 
+            <dependency>
+                <groupId>com.databricks</groupId>
+                <artifactId>databricks-jdbc</artifactId>
+                <version>2.6.25-1</version>
+            </dependency>
+
             <dependency>
                 <groupId>com.datastax.oss</groupId>
                 <artifactId>java-driver-core</artifactId>
@@ -1351,6 +1365,12 @@
                 <version>${dep.jsonwebtoken.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>io.minio</groupId>
+                <artifactId>minio</artifactId>
+                <version>${dep.minio.version}</version>
+            </dependency>
+
             <!-- io.confluent:kafka-avro-serializer uses multiple versions of this transitive dependency -->
             <dependency>
                 <groupId>io.swagger</groupId>

testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentModule.java

@@ -26,6 +26,7 @@
 import io.trino.tests.product.launcher.env.common.KafkaSaslPlaintext;
 import io.trino.tests.product.launcher.env.common.KafkaSsl;
 import io.trino.tests.product.launcher.env.common.Kerberos;
+import io.trino.tests.product.launcher.env.common.Minio;
 import io.trino.tests.product.launcher.env.common.Phoenix;
 import io.trino.tests.product.launcher.env.common.Standard;
 import io.trino.tests.product.launcher.env.common.StandardMultinode;
@@ -75,6 +76,7 @@ public void configure(Binder binder)
         binder.bind(StandardMultinode.class).in(SINGLETON);
         binder.bind(Phoenix.class).in(SINGLETON);
         binder.bind(Kerberos.class).in(SINGLETON);
+        binder.bind(Minio.class).in(SINGLETON);
 
         MapBinder<String, EnvironmentProvider> environments = newMapBinder(binder, String.class, EnvironmentProvider.class);
         findEnvironmentsByBasePackage(ENVIRONMENT_PACKAGE).forEach(clazz -> environments.addBinding(nameForEnvironmentClass(clazz)).to(clazz).in(SINGLETON));

testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Minio.java

@@ -0,0 +1,86 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.tests.product.launcher.env.common;
+
+import com.google.common.collect.ImmutableMap;
+import io.trino.tests.product.launcher.docker.DockerFiles;
+import io.trino.tests.product.launcher.env.DockerContainer;
+import io.trino.tests.product.launcher.env.Environment;
+import io.trino.tests.product.launcher.testcontainers.PortBinder;
+import org.testcontainers.containers.startupcheck.IsRunningStartupCheckStrategy;
+
+import javax.inject.Inject;
+
+import java.time.Duration;
+
+import static io.trino.tests.product.launcher.docker.ContainerUtil.forSelectedPorts;
+import static io.trino.tests.product.launcher.env.EnvironmentContainers.HADOOP;
+import static io.trino.tests.product.launcher.env.common.Hadoop.CONTAINER_HADOOP_INIT_D;
+import static java.lang.String.format;
+import static java.util.Objects.requireNonNull;
+import static org.testcontainers.utility.MountableFile.forHostPath;
+
+public class Minio
+        implements EnvironmentExtender
+{
+    private final DockerFiles dockerFiles;
+
+    public static final String MINIO_CONTAINER_NAME = "minio";
+
+    private static final String MINIO_ACCESS_KEY = "minio-access-key";
+    private static final String MINIO_SECRET_KEY = "minio-secret-key";
+    private static final String MINIO_RELEASE = "RELEASE.2021-07-15T22-27-34Z";
+
+    private static final int MINIO_PORT = 9080; // minio uses 9000 by default, which conflicts with hadoop
+
+    private final PortBinder portBinder;
+
+    @Inject
+    public Minio(DockerFiles dockerFiles, PortBinder portBinder)
+    {
+        this.dockerFiles = requireNonNull(dockerFiles, "dockerFiles is null");
+        this.portBinder = requireNonNull(portBinder, "portBinder is null");
+    }
+
+    @Override
+    public void extendEnvironment(Environment.Builder builder)
+    {
+        builder.addContainer(createMinioContainer());
+
+        builder.configureContainers(container -> {
+            if (container.getLogicalName().equals(HADOOP)) {
+                container.withCopyFileToContainer(
+                        forHostPath(dockerFiles.getDockerFilesHostPath("common/minio/apply-minio-config.sh")),
+                        CONTAINER_HADOOP_INIT_D + "apply-minio-config.sh");
+            }
+        });
+    }
+
+    private DockerContainer createMinioContainer()
+    {
+        DockerContainer container = new DockerContainer("minio/minio:" + MINIO_RELEASE, MINIO_CONTAINER_NAME)
+                .withEnv(ImmutableMap.<String, String>builder()
+                        .put("MINIO_ACCESS_KEY", MINIO_ACCESS_KEY)
+                        .put("MINIO_SECRET_KEY", MINIO_SECRET_KEY)
+                        .buildOrThrow())
+                .withCommand("server", "--address", format("0.0.0.0:%d", MINIO_PORT), "/data")
+                .withStartupCheckStrategy(new IsRunningStartupCheckStrategy())
+                .waitingFor(forSelectedPorts(MINIO_PORT))
+                .withStartupTimeout(Duration.ofMinutes(1));
+
+        portBinder.exposePort(container, MINIO_PORT);
+
+        return container;
+    }
+}

testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/AbstractSinglenodeDeltaLakeDatabricks.java

@@ -0,0 +1,94 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.tests.product.launcher.env.environment;
+
+import io.trino.tests.product.launcher.docker.DockerFiles;
+import io.trino.tests.product.launcher.env.DockerContainer;
+import io.trino.tests.product.launcher.env.Environment;
+import io.trino.tests.product.launcher.env.EnvironmentProvider;
+import io.trino.tests.product.launcher.env.common.Standard;
+
+import static io.trino.tests.product.launcher.env.EnvironmentContainers.COORDINATOR;
+import static io.trino.tests.product.launcher.env.EnvironmentContainers.TESTS;
+import static io.trino.tests.product.launcher.env.EnvironmentContainers.configureTempto;
+import static io.trino.tests.product.launcher.env.common.Standard.CONTAINER_PRESTO_ETC;
+import static java.lang.String.format;
+import static java.util.Objects.requireNonNull;
+import static org.testcontainers.utility.MountableFile.forHostPath;
+
+/**
+ * Trino with Delta Lake connector and real S3 storage
+ */
+public abstract class AbstractSinglenodeDeltaLakeDatabricks
+        extends EnvironmentProvider
+{
+    private final DockerFiles dockerFiles;
+
+    abstract String databricksTestJdbcUrl();
+
+    public AbstractSinglenodeDeltaLakeDatabricks(Standard standard, DockerFiles dockerFiles)
+    {
+        super(standard);
+        this.dockerFiles = requireNonNull(dockerFiles, "dockerFiles is null");
+    }
+
+    @Override
+    public void extendEnvironment(Environment.Builder builder)
+    {
+        String databricksTestJdbcUrl = databricksTestJdbcUrl();
+        String databricksTestJdbcDriverClass = requireNonNull(System.getenv("DATABRICKS_TEST_JDBC_DRIVER_CLASS"), "Environment DATABRICKS_TEST_JDBC_DRIVER_CLASS was not set");
+        String databricksTestLogin = requireNonNull(System.getenv("DATABRICKS_TEST_LOGIN"), "Environment DATABRICKS_TEST_LOGIN was not set");
+        String databricksTestToken = requireNonNull(System.getenv("DATABRICKS_TEST_TOKEN"), "Environment DATABRICKS_TEST_TOKEN was not set");
+        String hiveMetastoreUri = requireNonNull(System.getenv("HIVE_METASTORE_URI"), "Environment HIVE_METASTORE_URI was not set");
+        String s3Bucket = requireNonNull(System.getenv("S3_BUCKET"), "Environment S3_BUCKET was not set");
+        DockerFiles.ResourceProvider configDir = dockerFiles.getDockerFilesHostDirectory("conf/environment/singlenode-delta-lake-databricks");
+
+        builder.configureContainer(COORDINATOR, dockerContainer -> exportAWSCredentials(dockerContainer)
+                .withEnv("HIVE_METASTORE_URI", hiveMetastoreUri)
+                .withEnv("DATABRICKS_TEST_JDBC_URL", databricksTestJdbcUrl)
+                .withEnv("DATABRICKS_TEST_LOGIN", databricksTestLogin)
+                .withEnv("DATABRICKS_TEST_TOKEN", databricksTestToken)
+                .withCopyFileToContainer(forHostPath(configDir.getPath("hive.properties")), CONTAINER_PRESTO_ETC + "/catalog/hive.properties")
+                .withCopyFileToContainer(forHostPath(configDir.getPath("delta.properties")), CONTAINER_PRESTO_ETC + "/catalog/delta.properties"));
+
+        builder.configureContainer(TESTS, container -> exportAWSCredentials(container)
+                .withEnv("S3_BUCKET", s3Bucket)
+                .withEnv("DATABRICKS_TEST_JDBC_DRIVER_CLASS", databricksTestJdbcDriverClass)
+                .withEnv("DATABRICKS_TEST_JDBC_URL", databricksTestJdbcUrl)
+                .withEnv("DATABRICKS_TEST_LOGIN", databricksTestLogin)
+                .withEnv("DATABRICKS_TEST_TOKEN", databricksTestToken)
+                .withEnv("HIVE_METASTORE_URI", hiveMetastoreUri));
+
+        configureTempto(builder, configDir);
+    }
+
+    private DockerContainer exportAWSCredentials(DockerContainer container)
+    {
+        container = exportAWSCredential(container, "AWS_ACCESS_KEY_ID", true);
+        container = exportAWSCredential(container, "AWS_SECRET_ACCESS_KEY", true);
+        return exportAWSCredential(container, "AWS_SESSION_TOKEN", false);
+    }
+
+    private DockerContainer exportAWSCredential(DockerContainer container, String credentialEnvVariable, boolean required)
+    {
+        String credentialValue = System.getenv(credentialEnvVariable);
+        if (credentialValue == null) {
+            if (required) {
+                throw new IllegalStateException(format("Environment variable %s not set", credentialEnvVariable));
+            }
+            return container;
+        }
+        return container.withEnv(credentialEnvVariable, credentialValue);
+    }
+}

testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeMinioDataLake.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.tests.product.launcher.env.environment;
+
+import io.trino.tests.product.launcher.docker.DockerFiles;
+import io.trino.tests.product.launcher.env.DockerContainer;
+import io.trino.tests.product.launcher.env.Environment;
+import io.trino.tests.product.launcher.env.EnvironmentProvider;
+import io.trino.tests.product.launcher.env.common.Hadoop;
+import io.trino.tests.product.launcher.env.common.Minio;
+import io.trino.tests.product.launcher.env.common.StandardMultinode;
+import io.trino.tests.product.launcher.env.common.TestsEnvironment;
+
+import javax.inject.Inject;
+
+import static io.trino.tests.product.launcher.env.EnvironmentContainers.COORDINATOR;
+import static io.trino.tests.product.launcher.env.EnvironmentContainers.WORKER;
+import static io.trino.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_HIVE_PROPERTIES;
+import static io.trino.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_ICEBERG_PROPERTIES;
+import static io.trino.tests.product.launcher.env.common.Standard.CONTAINER_PRESTO_ETC;
+import static java.util.Objects.requireNonNull;
+import static org.testcontainers.utility.MountableFile.forHostPath;
+
+/**
+ * Trino with S3-compatible Data Lake setup based on MinIO
+ */
+@TestsEnvironment
+public class EnvMultinodeMinioDataLake
+        extends EnvironmentProvider
+{
+    private final DockerFiles dockerFiles;
+
+    @Inject
+    public EnvMultinodeMinioDataLake(StandardMultinode standardMultinode, Hadoop hadoop, Minio minio, DockerFiles dockerFiles)
+    {
+        super(standardMultinode, hadoop, minio);
+        this.dockerFiles = requireNonNull(dockerFiles, "dockerFiles is null");
+    }
+
+    @Override
+    public void extendEnvironment(Environment.Builder builder)
+    {
+        builder.configureContainer(COORDINATOR, this::configureTrinoContainer);
+        builder.configureContainer(WORKER, this::configureTrinoContainer);
+    }
+
+    private void configureTrinoContainer(DockerContainer container)
+    {
+        container.withCopyFileToContainer(
+                forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-minio-data-lake/hive.properties")),
+                CONTAINER_PRESTO_HIVE_PROPERTIES);
+        container.withCopyFileToContainer(
+                forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-minio-data-lake/delta.properties")),
+                CONTAINER_PRESTO_ETC + "/catalog/delta.properties");
+        container.withCopyFileToContainer(
+                forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-minio-data-lake/iceberg.properties")),
+                CONTAINER_PRESTO_ICEBERG_PROPERTIES);
+        container.withCopyFileToContainer(
+                forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-minio-data-lake/memory.properties")),
+                CONTAINER_PRESTO_ETC + "/catalog/memory.properties");
+    }
+}