diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/AbstractSinglenodeDeltaLakeDatabricks.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/AbstractSinglenodeDeltaLakeDatabricks.java index 3c174d7fc148..ec37af15e687 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/AbstractSinglenodeDeltaLakeDatabricks.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/AbstractSinglenodeDeltaLakeDatabricks.java @@ -58,9 +58,12 @@ public void extendEnvironment(Environment.Builder builder) .withEnv("HIVE_METASTORE_URI", hiveMetastoreUri) .withEnv("DATABRICKS_TEST_JDBC_URL", databricksTestJdbcUrl) .withEnv("DATABRICKS_TEST_LOGIN", databricksTestLogin) - .withEnv("DATABRICKS_TEST_TOKEN", databricksTestToken) - .withCopyFileToContainer(forHostPath(configDir.getPath("hive.properties")), CONTAINER_PRESTO_ETC + "/catalog/hive.properties") - .withCopyFileToContainer(forHostPath(configDir.getPath("delta.properties")), CONTAINER_PRESTO_ETC + "/catalog/delta.properties")); + .withEnv("DATABRICKS_TEST_TOKEN", databricksTestToken)); + builder.addConnector("hive", forHostPath(configDir.getPath("hive.properties"))); + builder.addConnector( + "delta-lake", + forHostPath(configDir.getPath("delta.properties")), + CONTAINER_PRESTO_ETC + "/catalog/delta.properties"); builder.configureContainer(TESTS, container -> exportAWSCredentials(container) .withEnv("S3_BUCKET", s3Bucket) diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeMinioDataLake.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeMinioDataLake.java index 379a70ac9820..e4c1dbff0ef1 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeMinioDataLake.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeMinioDataLake.java @@ -14,7 +14,6 @@ package io.trino.tests.product.launcher.env.environment; import io.trino.tests.product.launcher.docker.DockerFiles; -import io.trino.tests.product.launcher.env.DockerContainer; import io.trino.tests.product.launcher.env.Environment; import io.trino.tests.product.launcher.env.EnvironmentProvider; import io.trino.tests.product.launcher.env.common.Hadoop; @@ -24,10 +23,6 @@ import javax.inject.Inject; -import static io.trino.tests.product.launcher.env.EnvironmentContainers.COORDINATOR; -import static io.trino.tests.product.launcher.env.EnvironmentContainers.WORKER; -import static io.trino.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_HIVE_PROPERTIES; -import static io.trino.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_ICEBERG_PROPERTIES; import static io.trino.tests.product.launcher.env.common.Standard.CONTAINER_PRESTO_ETC; import static java.util.Objects.requireNonNull; import static org.testcontainers.utility.MountableFile.forHostPath; @@ -39,35 +34,24 @@ public class EnvMultinodeMinioDataLake extends EnvironmentProvider { - private final DockerFiles dockerFiles; + private final DockerFiles.ResourceProvider configDir; @Inject public EnvMultinodeMinioDataLake(StandardMultinode standardMultinode, Hadoop hadoop, Minio minio, DockerFiles dockerFiles) { super(standardMultinode, hadoop, minio); - this.dockerFiles = requireNonNull(dockerFiles, "dockerFiles is null"); + this.configDir = requireNonNull(dockerFiles, "dockerFiles is null").getDockerFilesHostDirectory("conf/environment/singlenode-minio-data-lake"); } @Override public void extendEnvironment(Environment.Builder builder) { - builder.configureContainer(COORDINATOR, this::configureTrinoContainer); - builder.configureContainer(WORKER, this::configureTrinoContainer); - } - - private void configureTrinoContainer(DockerContainer container) - { - container.withCopyFileToContainer( - forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-minio-data-lake/hive.properties")), - CONTAINER_PRESTO_HIVE_PROPERTIES); - container.withCopyFileToContainer( - forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-minio-data-lake/delta.properties")), + builder.addConnector("hive", forHostPath(configDir.getPath("hive.properties"))); + builder.addConnector( + "delta-lake", + forHostPath(configDir.getPath("delta.properties")), CONTAINER_PRESTO_ETC + "/catalog/delta.properties"); - container.withCopyFileToContainer( - forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-minio-data-lake/iceberg.properties")), - CONTAINER_PRESTO_ICEBERG_PROPERTIES); - container.withCopyFileToContainer( - forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-minio-data-lake/memory.properties")), - CONTAINER_PRESTO_ETC + "/catalog/memory.properties"); + builder.addConnector("iceberg", forHostPath(configDir.getPath("iceberg.properties"))); + builder.addConnector("memory", forHostPath(configDir.getPath("memory.properties"))); } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeKerberizedHdfs.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeKerberizedHdfs.java index d7e419df544b..884095e08633 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeKerberizedHdfs.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeKerberizedHdfs.java @@ -23,7 +23,6 @@ import javax.inject.Inject; -import static io.trino.tests.product.launcher.env.EnvironmentContainers.COORDINATOR; import static io.trino.tests.product.launcher.env.common.Standard.CONTAINER_PRESTO_ETC; import static java.util.Objects.requireNonNull; import static org.testcontainers.utility.MountableFile.forHostPath; @@ -32,22 +31,21 @@ public class EnvSinglenodeDeltaLakeKerberizedHdfs extends EnvironmentProvider { - private final DockerFiles dockerFiles; + private final DockerFiles.ResourceProvider configDir; @Inject public EnvSinglenodeDeltaLakeKerberizedHdfs(Standard standard, Hadoop hadoop, HadoopKerberos hadoopKerberos, DockerFiles dockerFiles) { super(standard, hadoop, hadoopKerberos); - this.dockerFiles = requireNonNull(dockerFiles, "dockerFiles is null"); + this.configDir = requireNonNull(dockerFiles, "dockerFiles is null").getDockerFilesHostDirectory("conf/environment/singlenode-delta-lake-kerberized-hdfs"); } @Override public void extendEnvironment(Environment.Builder builder) { - builder.configureContainer(COORDINATOR, dockerContainer -> { - dockerContainer.withCopyFileToContainer( - forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-delta-lake-kerberized-hdfs/delta.properties")), - CONTAINER_PRESTO_ETC + "/catalog/delta.properties"); - }); + builder.addConnector( + "delta-lake", + forHostPath(configDir.getPath("delta.properties")), + CONTAINER_PRESTO_ETC + "/catalog/delta.properties"); } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeOss.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeOss.java index da348a30cd55..4938af17fe77 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeOss.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeOss.java @@ -37,7 +37,6 @@ import java.util.Set; import static io.trino.tests.product.launcher.docker.ContainerUtil.forSelectedPorts; -import static io.trino.tests.product.launcher.env.EnvironmentContainers.COORDINATOR; import static io.trino.tests.product.launcher.env.EnvironmentContainers.HADOOP; import static io.trino.tests.product.launcher.env.EnvironmentContainers.TESTS; import static io.trino.tests.product.launcher.env.EnvironmentContainers.configureTempto; @@ -94,9 +93,11 @@ public void extendEnvironment(Environment.Builder builder) container.setDockerImageName("ghcr.io/trinodb/testing/hdp3.1-hive:" + hadoopImagesVersion); }); - builder.configureContainer(COORDINATOR, container -> container - .withCopyFileToContainer(forHostPath(configDir.getPath("hive.properties")), CONTAINER_PRESTO_ETC + "/catalog/hive.properties") - .withCopyFileToContainer(forHostPath(configDir.getPath("delta.properties")), CONTAINER_PRESTO_ETC + "/catalog/delta.properties")); + builder.addConnector("hive", forHostPath(configDir.getPath("hive.properties"))); + builder.addConnector( + "delta-lake", + forHostPath(configDir.getPath("delta.properties")), + CONTAINER_PRESTO_ETC + "/catalog/delta.properties"); builder.configureContainer(TESTS, dockerContainer -> { dockerContainer.withEnv("S3_BUCKET", s3Bucket) diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLake.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLake.java index 67755679904e..f18a3e272d67 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLake.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLake.java @@ -33,26 +33,26 @@ public List getTestRuns(EnvironmentConfig config) { return ImmutableList.of( testOnEnvironment(EnvMultinodeMinioDataLake.class) - .withGroups("delta-lake-minio") + .withGroups("configured_features", "delta-lake-minio") .build(), testOnEnvironment(EnvSinglenodeDeltaLakeKerberizedHdfs.class) - .withGroups("delta-lake-hdfs") + .withGroups("configured_features", "delta-lake-hdfs") .build(), //TODO enable the product tests against Databricks testing environment // testOnEnvironment(EnvSinglenodeDeltaLakeDatabricks.class) -// .withGroups("delta-lake-databricks") +// .withGroups("configured_features", "delta-lake-databricks") // .withExcludedGroups("delta-lake-exclude-73") // .build(), // // testOnEnvironment(EnvSinglenodeDeltaLakeDatabricks91.class) -// .withGroups("delta-lake-databricks") +// .withGroups("configured_features", "delta-lake-databricks") // .build(), testOnEnvironment(EnvSinglenodeDeltaLakeOss.class) // TODO: make the list of tests run here as close to those run on SinglenodeDeltaLakeDatabricks // e.g. replace `delta-lake-oss` group with `delta-lake-databricks` + any exclusions, of needed - .withGroups("delta-lake-oss") + .withGroups("configured_features", "delta-lake-oss") .build()); } }