diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestHiveOnDataLake.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestHiveOnDataLake.java index 5207baadf658..c1c40082ea25 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestHiveOnDataLake.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestHiveOnDataLake.java @@ -32,6 +32,7 @@ import io.trino.plugin.hive.s3.S3HiveQueryRunner; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.QueryRunner; +import org.testcontainers.containers.Network; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -46,30 +47,34 @@ import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.joining; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.testcontainers.containers.Network.newNetwork; public abstract class BaseTestHiveOnDataLake extends AbstractTestQueryFramework { - private static final String HIVE_TEST_SCHEMA = "hive_insert_overwrite"; + protected final String bucketName; + protected final String schemaName; + protected final Network network; - private String bucketName; - private HiveMinioDataLake dockerizedS3DataLake; - private HiveMetastore metastoreClient; + protected HiveMinioDataLake dockerizedS3DataLake; + protected HiveMetastore metastoreClient; private final String hiveHadoopImage; public BaseTestHiveOnDataLake(String hiveHadoopImage) { this.hiveHadoopImage = requireNonNull(hiveHadoopImage, "hiveHadoopImage is null"); + this.bucketName = "test-hive-data-lake-" + randomTableSuffix(); + this.schemaName = "hive_data_lake_" + randomTableSuffix(); + this.network = closeAfterClass(newNetwork()); } @Override protected QueryRunner createQueryRunner() throws Exception { - this.bucketName = "test-hive-insert-overwrite-" + randomTableSuffix(); this.dockerizedS3DataLake = closeAfterClass( - new HiveMinioDataLake(bucketName, ImmutableMap.of(), hiveHadoopImage)); + new HiveMinioDataLake(bucketName, ImmutableMap.of(), hiveHadoopImage, Optional.of(network))); this.dockerizedS3DataLake.start(); this.metastoreClient = new BridgingHiveMetastore(new ThriftHiveMetastore( new TestingMetastoreLocator( @@ -87,16 +92,18 @@ protected QueryRunner createQueryRunner() new NoHdfsAuthentication()), false), HiveIdentity.none()); - return S3HiveQueryRunner.create( - dockerizedS3DataLake, - ImmutableMap.builder() - // This is required when using MinIO which requires path style access - .put("hive.insert-existing-partitions-behavior", "OVERWRITE") - .put("hive.non-managed-table-writes-enabled", "true") - // Below are required to enable caching on metastore - .put("hive.metastore-cache-ttl", "1d") - .put("hive.metastore-refresh-interval", "1d") - .buildOrThrow()); + return S3HiveQueryRunner.create(dockerizedS3DataLake, getAdditionalHivePropertiesBuilder().buildOrThrow()); + } + + protected ImmutableMap.Builder getAdditionalHivePropertiesBuilder() + { + return ImmutableMap.builder() + // This is required when using MinIO which requires path style access + .put("hive.insert-existing-partitions-behavior", "OVERWRITE") + .put("hive.non-managed-table-writes-enabled", "true") + // Below are required to enable caching on metastore + .put("hive.metastore-cache-ttl", "1d") + .put("hive.metastore-refresh-interval", "1d"); } @BeforeClass @@ -104,7 +111,7 @@ public void setUp() { computeActual(format( "CREATE SCHEMA hive.%1$s WITH (location='s3a://%2$s/%1$s')", - HIVE_TEST_SCHEMA, + schemaName, bucketName)); } @@ -190,7 +197,7 @@ public void testInsertOverwritePartitionedAndBucketedExternalTable() "partitioned_by=ARRAY['regionkey']", "bucketed_by = ARRAY['nationkey']", "bucket_count = 3", - format("external_location = 's3a://%s/%s/%s/'", this.bucketName, HIVE_TEST_SCHEMA, testTable))); + format("external_location = 's3a://%s/%s/%s/'", this.bucketName, schemaName, testTable))); copyTpchNationToTable(testTable); assertOverwritePartition(externalTableName); } @@ -231,7 +238,7 @@ public void testFlushPartitionCache() // Refresh cache for schema_name => 'dummy_schema', table_name => 'dummy_table', partition_column => getQueryRunner().execute(format( "CALL system.flush_metadata_cache(schema_name => '%s', table_name => '%s', partition_column => ARRAY['%s'], partition_value => ARRAY['%s'])", - HIVE_TEST_SCHEMA, + schemaName, tableName, partitionColumn, partitionValue1)); @@ -247,7 +254,7 @@ public void testFlushPartitionCache() private void renamePartitionResourcesOutsideTrino(String tableName, String partitionColumn, String regionKey) { String partitionName = format("%s=%s", partitionColumn, regionKey); - String partitionS3KeyPrefix = format("%s/%s/%s", HIVE_TEST_SCHEMA, tableName, partitionName); + String partitionS3KeyPrefix = format("%s/%s/%s", schemaName, tableName, partitionName); String renamedPartitionSuffix = "CP"; // Copy whole partition to new location @@ -264,13 +271,13 @@ private void renamePartitionResourcesOutsideTrino(String tableName, String parti }); // Delete old partition and update metadata to point to location of new copy - Table hiveTable = metastoreClient.getTable(HIVE_TEST_SCHEMA, tableName).get(); + Table hiveTable = metastoreClient.getTable(schemaName, tableName).get(); Partition hivePartition = metastoreClient.getPartition(hiveTable, List.of(regionKey)).get(); Map partitionStatistics = metastoreClient.getPartitionStatistics(hiveTable, List.of(hivePartition)); - metastoreClient.dropPartition(HIVE_TEST_SCHEMA, tableName, List.of(regionKey), true); - metastoreClient.addPartitions(HIVE_TEST_SCHEMA, tableName, List.of( + metastoreClient.dropPartition(schemaName, tableName, List.of(regionKey), true); + metastoreClient.addPartitions(schemaName, tableName, List.of( new PartitionWithStatistics( Partition.builder(hivePartition) .withStorage(builder -> builder.setLocation( @@ -333,7 +340,7 @@ protected String getTestTableName() protected String getTestTableName(String tableName) { - return format("hive.%s.%s", HIVE_TEST_SCHEMA, tableName); + return format("hive.%s.%s", schemaName, tableName); } protected String getCreateTableStatement(String tableName, String... propertiesEntries) diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive2OnDataLakeOverProxy.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive2OnDataLakeOverProxy.java new file mode 100644 index 000000000000..1a900c89363e --- /dev/null +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive2OnDataLakeOverProxy.java @@ -0,0 +1,59 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive; + +import com.google.common.collect.ImmutableMap; +import io.trino.plugin.hive.containers.HiveHadoop; +import io.trino.testing.containers.Httpd; +import io.trino.testing.containers.Minio; + +import static io.trino.testing.containers.Httpd.builderForHttpProxy; + +public class TestHive2OnDataLakeOverProxy + extends BaseTestHiveOnDataLake +{ + public TestHive2OnDataLakeOverProxy() + { + super(HiveHadoop.DEFAULT_IMAGE); + } + + @Override + protected ImmutableMap.Builder getAdditionalHivePropertiesBuilder() + { + Httpd httpd = closeAfterClass(builderForHttpProxy(8888) + .addModuleConfiguration("" + + "\n" + + " ErrorLog /var/log/http.log\n" + + " \n" + + " ProxyRequests Off\n" + + " ProxyVia Block\n" + + " ProxyPreserveHost On\n" + + " \n" + + " \n" + + " Require all granted\n" + + " \n" + + " \n" + + " ProxyPass / http://" + Minio.DEFAULT_HOST_NAME + ":" + Minio.MINIO_API_PORT + "/\n" + + " ProxyPassReverse / http://" + Minio.DEFAULT_HOST_NAME + ":" + Minio.MINIO_API_PORT + "/\n" + + "") + .withNetwork(network) + .build()); + httpd.start(); + return super.getAdditionalHivePropertiesBuilder() + .put("hive.s3.ssl.enabled", "false") + .put("hive.s3.proxy.protocol", "http") + .put("hive.s3.proxy.host", "localhost") + .put("hive.s3.proxy.port", "" + httpd.getListenPort()); + } +} diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/HiveMinioDataLake.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/HiveMinioDataLake.java index 28502f371d7f..3326fee83a4b 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/HiveMinioDataLake.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/HiveMinioDataLake.java @@ -24,6 +24,7 @@ import org.testcontainers.containers.Network; import java.util.Map; +import java.util.Optional; import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; @@ -42,6 +43,7 @@ public class HiveMinioDataLake private final AutoCloseableCloser closer = AutoCloseableCloser.create(); private State state = State.INITIAL; + private Optional network; private AmazonS3 s3Client; public HiveMinioDataLake(String bucketName, Map hiveHadoopFilesToMount) @@ -51,25 +53,33 @@ public HiveMinioDataLake(String bucketName, Map hiveHadoopFilesT public HiveMinioDataLake(String bucketName, Map hiveHadoopFilesToMount, String hiveHadoopImage) { + this(bucketName, hiveHadoopFilesToMount, hiveHadoopImage, Optional.of(newNetwork())); + } + + public HiveMinioDataLake(String bucketName, Map hiveHadoopFilesToMount, String hiveHadoopImage, Optional network) + { + Minio.Builder minioBuilder = Minio.builder() + .withEnvVars(ImmutableMap.builder() + .put("MINIO_ACCESS_KEY", ACCESS_KEY) + .put("MINIO_SECRET_KEY", SECRET_KEY) + .buildOrThrow()); + HiveHadoop.Builder hiveHadoopBuilder = HiveHadoop.builder() + .withFilesToMount(ImmutableMap.builder() + .put("hive_minio_datalake/hive-core-site.xml", "/etc/hadoop/conf/core-site.xml") + .putAll(hiveHadoopFilesToMount) + .buildOrThrow()) + .withImage(hiveHadoopImage); + + this.network = network; + this.network.ifPresent(value -> { + closer.register(value); + minioBuilder.withNetwork(value); + hiveHadoopBuilder.withNetwork(value); + }); + this.bucketName = requireNonNull(bucketName, "bucketName is null"); - Network network = closer.register(newNetwork()); - this.minio = closer.register( - Minio.builder() - .withNetwork(network) - .withEnvVars(ImmutableMap.builder() - .put("MINIO_ACCESS_KEY", ACCESS_KEY) - .put("MINIO_SECRET_KEY", SECRET_KEY) - .buildOrThrow()) - .build()); - this.hiveHadoop = closer.register( - HiveHadoop.builder() - .withFilesToMount(ImmutableMap.builder() - .put("hive_minio_datalake/hive-core-site.xml", "/etc/hadoop/conf/core-site.xml") - .putAll(hiveHadoopFilesToMount) - .buildOrThrow()) - .withImage(hiveHadoopImage) - .withNetwork(network) - .build()); + this.minio = closer.register(minioBuilder.build()); + this.hiveHadoop = closer.register(hiveHadoopBuilder.build()); } public void start() @@ -92,6 +102,11 @@ public void start() state = State.STARTED; } + public Optional getNetwork() + { + return network; + } + public AmazonS3 getS3Client() { checkState(state == State.STARTED, "Can't provide client when MinIO state is: %s", state); diff --git a/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/BaseTestContainer.java b/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/BaseTestContainer.java index 96a9aaec6d22..1c210181da49 100644 --- a/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/BaseTestContainer.java +++ b/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/BaseTestContainer.java @@ -92,16 +92,20 @@ protected void withRunCommand(List runCommand) protected void copyFileToContainer(String resourcePath, String dockerPath) { - container.withCopyFileToContainer( - forHostPath( - forClasspathResource(resourcePath) - // Container fails to mount jar:file:/! resources - // This assures that JAR resources are being copied out to tmp locations - // and mounted from there. - .getResolvedPath()), + copyHostFileToContainer( + forClasspathResource(resourcePath) + // Container fails to mount jar:file:/! resources + // This assures that JAR resources are being copied out to tmp locations + // and mounted from there. + .getResolvedPath(), dockerPath); } + protected void copyHostFileToContainer(String hostPath, String dockerPath) + { + container.withCopyFileToContainer(forHostPath(hostPath), dockerPath); + } + protected HostAndPort getMappedHostAndPortForExposedPort(int exposedPort) { return fromParts(container.getHost(), container.getMappedPort(exposedPort)); diff --git a/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/Httpd.java b/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/Httpd.java new file mode 100644 index 000000000000..0fd1e4876e0c --- /dev/null +++ b/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/Httpd.java @@ -0,0 +1,174 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.testing.containers; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import io.airlift.log.Logger; +import org.testcontainers.containers.Network; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.requireNonNull; + +public class Httpd + extends BaseTestContainer +{ + private static final Logger log = Logger.get(Httpd.class); + + public static final String DEFAULT_IMAGE = "httpd:2.4.51"; + public static final int DEFAULT_LISTEN_PORT = 8888; + + public static Builder builder() + { + return new Builder(); + } + + public static Builder builderForHttpProxy(int proxyPort) + { + return new Builder() + .withHostName("localhost") + .withListenPort(proxyPort) + .addLoadModule("mpm_event_module", "modules/mod_mpm_event.so") + .addLoadModule("authn_file_module", "modules/mod_authn_file.so") + .addLoadModule("authn_core_module", "modules/mod_authn_core.so") + .addLoadModule("authz_host_module", "modules/mod_authz_host.so") + .addLoadModule("authz_groupfile_module", "modules/mod_authz_groupfile.so") + .addLoadModule("authz_user_module", "modules/mod_authz_user.so") + .addLoadModule("authz_core_module", "modules/mod_authz_core.so") + .addLoadModule("access_compat_module", "modules/mod_access_compat.so") + .addLoadModule("auth_basic_module", "modules/mod_auth_basic.so") + .addLoadModule("reqtimeout_module", "modules/mod_reqtimeout.so") + .addLoadModule("filter_module", "modules/mod_filter.so") + .addLoadModule("mime_module", "modules/mod_mime.so") + .addLoadModule("log_config_module", "modules/mod_log_config.so") + .addLoadModule("env_module", "modules/mod_env.so") + .addLoadModule("headers_module", "modules/mod_headers.so") + .addLoadModule("setenvif_module", "modules/mod_setenvif.so") + .addLoadModule("version_module", "modules/mod_version.so") + .addLoadModule("proxy_module", "modules/mod_proxy.so") + .addLoadModule("proxy_connect_module", "modules/mod_proxy_connect.so") + .addLoadModule("proxy_http_module", "modules/mod_proxy_http.so") + .addLoadModule("unixd_module", "modules/mod_unixd.so") + .addLoadModule("status_module", "modules/mod_status.so") + .addLoadModule("autoindex_module", "modules/mod_autoindex.so") + .addLoadModule("dir_module", "modules/mod_dir.so") + .addLoadModule("alias_module", "modules/mod_alias.so"); + } + + private final int listenPort; + private final Map loadModules; + private final List modulesConfiguration; + + private Httpd( + String image, + String hostName, + int listenPort, + Map loadModules, + List modulesConfiguration, + Map envVars, + Optional network, + int retryLimit) + { + super( + image, + hostName, + ImmutableSet.of(listenPort), + ImmutableMap.of(), + envVars, + network, + retryLimit); + this.listenPort = listenPort; + this.loadModules = requireNonNull(loadModules, "loadModules is null"); + this.modulesConfiguration = requireNonNull(modulesConfiguration, "modulesConfiguration is null"); + preareAndCopyHttpdConfiguration(hostName); + } + + private void preareAndCopyHttpdConfiguration(String hostName) + { + try { + Path httpConf = Files.createTempFile("httpd", ".conf"); + StringBuilder httpConfBody = new StringBuilder(); + httpConfBody.append("ServerRoot \"/usr/local/apache2\"\n\n"); + httpConfBody.append("ServerName " + hostName + "\n\n"); + httpConfBody.append("Listen " + listenPort + "\n\n"); + loadModules.entrySet().stream() + .map(entry -> "LoadModule " + entry.getKey() + " " + entry.getValue() + "\n") + .forEach(httpConfBody::append); + httpConfBody.append("\n"); + modulesConfiguration.stream().map(conf -> conf + "\n\n").forEach(httpConfBody::append); + Files.write(httpConf, httpConfBody.toString().getBytes(UTF_8)); + copyHostFileToContainer(httpConf.toString(), "/usr/local/apache2/conf/httpd.conf"); + } + catch (IOException e) { + throw new RuntimeException("Failed to write httpd.conf for Httpd test container", e); + } + } + + @Override + public void start() + { + super.start(); + log.info("Httpd container started. Listening on: " + listenPort); + } + + public int getListenPort() + { + return getMappedHostAndPortForExposedPort(listenPort).getPort(); + } + + public static class Builder + extends BaseTestContainer.Builder + { + private int listenPort = DEFAULT_LISTEN_PORT; + private ImmutableMap.Builder loadModules = ImmutableMap.builder(); + private ImmutableList.Builder modulesConfiguration = ImmutableList.builder(); + + private Builder() + { + this.image = DEFAULT_IMAGE; + } + + public Builder withListenPort(int listenPort) + { + this.listenPort = listenPort; + return self; + } + + public Builder addLoadModule(String moduleName, String moduleLib) + { + this.loadModules.put(moduleName, moduleLib); + return self; + } + + public Builder addModuleConfiguration(String moduleConfiguration) + { + this.modulesConfiguration.add(moduleConfiguration); + return self; + } + + @Override + public Httpd build() + { + return new Httpd(image, hostName, listenPort, loadModules.buildOrThrow(), modulesConfiguration.build(), envVars, network, startupRetryLimit); + } + } +}