Skip to content

Use memory file system in some Hive tests #19664

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dain merged 7 commits into from
Nov 12, 2023
Merged

Use memory file system in some Hive tests #19664

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
cbd05a8
Clarify that TestHiveAcidUtils only tests the behavior of code in Hive
dain Nov 7, 2023
1dc4d54
Convert TestBackgroundHiveSplitLoader to memory file system
dain Nov 7, 2023
b9d7aff
Convert TestOrcPageSourceFactory to memory file system
dain Nov 7, 2023
5af5887
Convert TestNodeLocalDynamicSplitPruning to memory file system
dain Nov 7, 2023
f9b573b
Cleanup TestNodeLocalDynamicSplitPruning
dain Nov 7, 2023
876284b
Convert TestHivePageSink to memory file system
dain Nov 7, 2023
5b9d306
Cleanup TestHivePageSink
dain Nov 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions plugin/trino-hive/src/test/java/io/trino/plugin/hive/HiveTestUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,11 @@ public static HiveSessionProperties getHiveSessionProperties(HiveConfig hiveConf

public static Set<HivePageSourceFactory> getDefaultHivePageSourceFactories(HdfsEnvironment hdfsEnvironment, HiveConfig hiveConfig)
{
TrinoFileSystemFactory fileSystemFactory = new HdfsFileSystemFactory(hdfsEnvironment, HDFS_FILE_SYSTEM_STATS);
return getDefaultHivePageSourceFactories(new HdfsFileSystemFactory(hdfsEnvironment, HDFS_FILE_SYSTEM_STATS), hiveConfig);
}

public static Set<HivePageSourceFactory> getDefaultHivePageSourceFactories(TrinoFileSystemFactory fileSystemFactory, HiveConfig hiveConfig)
{
FileFormatDataSourceStats stats = new FileFormatDataSourceStats();
return ImmutableSet.<HivePageSourceFactory>builder()
.add(new CsvPageSourceFactory(fileSystemFactory, hiveConfig))
Expand All @@ -189,7 +193,11 @@ public static Set<HivePageSourceFactory> getDefaultHivePageSourceFactories(HdfsE

public static Set<HiveFileWriterFactory> getDefaultHiveFileWriterFactories(HiveConfig hiveConfig, HdfsEnvironment hdfsEnvironment)
{
TrinoFileSystemFactory fileSystemFactory = new HdfsFileSystemFactory(hdfsEnvironment, HDFS_FILE_SYSTEM_STATS);
return getDefaultHiveFileWriterFactories(hiveConfig, new HdfsFileSystemFactory(hdfsEnvironment, HDFS_FILE_SYSTEM_STATS));
}

public static Set<HiveFileWriterFactory> getDefaultHiveFileWriterFactories(HiveConfig hiveConfig, TrinoFileSystemFactory fileSystemFactory)
{
NodeVersion nodeVersion = new NodeVersion("test_version");
return ImmutableSet.<HiveFileWriterFactory>builder()
.add(new CsvFileWriterFactory(fileSystemFactory, TESTING_TYPE_MANAGER))
Expand Down
745 changes: 316 additions & 429 deletions plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestBackgroundHiveSplitLoader.java
View file
Open in desktop

Large diffs are not rendered by default.

167 changes: 78 additions & 89 deletions plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHivePageSink.java
View file
Open in desktop

Large diffs are not rendered by default.

63 changes: 33 additions & 30 deletions plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestNodeLocalDynamicSplitPruning.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import io.airlift.testing.TempFile;
import io.trino.filesystem.Location;
import io.trino.filesystem.TrinoFileSystemFactory;
import io.trino.filesystem.memory.MemoryFileSystemFactory;
import io.trino.metadata.TableHandle;
import io.trino.plugin.hive.metastore.Column;
import io.trino.plugin.hive.orc.OrcReaderConfig;
Expand All @@ -30,10 +32,10 @@
import io.trino.spi.connector.EmptyPageSource;
import io.trino.spi.predicate.Domain;
import io.trino.spi.predicate.TupleDomain;
import io.trino.spi.security.ConnectorIdentity;
import io.trino.testing.TestingConnectorSession;
import org.junit.jupiter.api.Test;

import java.io.File;
import java.io.IOException;
import java.util.Map;
import java.util.Optional;
Expand All @@ -44,7 +46,6 @@

import static io.trino.plugin.hive.HiveColumnHandle.ColumnType.PARTITION_KEY;
import static io.trino.plugin.hive.HiveColumnHandle.ColumnType.REGULAR;
import static io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT;
import static io.trino.plugin.hive.HiveTestUtils.getDefaultHivePageSourceFactories;
import static io.trino.plugin.hive.HiveType.HIVE_INT;
import static io.trino.plugin.hive.util.HiveBucketing.BucketingVersion.BUCKETING_V1;
Expand All @@ -56,75 +57,77 @@
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_INPUT_FORMAT;
import static org.testng.Assert.assertEquals;

public class TestNodeLocalDynamicSplitPruning
class TestNodeLocalDynamicSplitPruning
{
private static final String SCHEMA_NAME = "test";
private static final String TABLE_NAME = "test";
private static final Column BUCKET_COLUMN = new Column("l_orderkey", HIVE_INT, Optional.empty());
private static final Column PARTITION_COLUMN = new Column("l_partkey", HIVE_INT, Optional.empty());
private static final Column BUCKET_COLUMN = new Column("l_orderkey", HIVE_INT, Optional.empty(), ImmutableMap.of());
private static final Column PARTITION_COLUMN = new Column("l_partkey", HIVE_INT, Optional.empty(), ImmutableMap.of());
private static final HiveColumnHandle BUCKET_HIVE_COLUMN_HANDLE = new HiveColumnHandle(
BUCKET_COLUMN.getName(),
0,
BUCKET_COLUMN.getType(),
TESTING_TYPE_MANAGER.getType(BUCKET_COLUMN.getType().getTypeSignature()),
INTEGER,
Optional.empty(),
REGULAR,
Optional.empty());
private static final HiveColumnHandle PARTITION_HIVE_COLUMN_HANDLE = new HiveColumnHandle(
PARTITION_COLUMN.getName(),
0,
PARTITION_COLUMN.getType(),
TESTING_TYPE_MANAGER.getType(PARTITION_COLUMN.getType().getTypeSignature()),
INTEGER,
Optional.empty(),
PARTITION_KEY,
Optional.empty());

@Test
public void testDynamicBucketPruning()
void testDynamicBucketPruning()
throws IOException
{
HiveConfig config = new HiveConfig();
HiveTransactionHandle transaction = new HiveTransactionHandle(false);
try (TempFile tempFile = new TempFile()) {
try (ConnectorPageSource emptyPageSource = createTestingPageSource(transaction, config, tempFile.file(), getDynamicFilter(getTupleDomainForBucketSplitPruning()))) {
assertEquals(emptyPageSource.getClass(), EmptyPageSource.class);
}
try (ConnectorPageSource emptyPageSource = createTestingPageSource(transaction, config, getDynamicFilter(getTupleDomainForBucketSplitPruning()))) {
assertEquals(emptyPageSource.getClass(), EmptyPageSource.class);
}

try (ConnectorPageSource nonEmptyPageSource = createTestingPageSource(transaction, config, tempFile.file(), getDynamicFilter(getNonSelectiveBucketTupleDomain()))) {
assertEquals(nonEmptyPageSource.getClass(), HivePageSource.class);
}
try (ConnectorPageSource nonEmptyPageSource = createTestingPageSource(transaction, config, getDynamicFilter(getNonSelectiveBucketTupleDomain()))) {
assertEquals(nonEmptyPageSource.getClass(), HivePageSource.class);
}
}

@Test
public void testDynamicPartitionPruning()
void testDynamicPartitionPruning()
throws IOException
{
HiveConfig config = new HiveConfig();
HiveTransactionHandle transaction = new HiveTransactionHandle(false);
try (TempFile tempFile = new TempFile()) {
try (ConnectorPageSource emptyPageSource = createTestingPageSource(transaction, config, tempFile.file(), getDynamicFilter(getTupleDomainForPartitionSplitPruning()))) {
assertEquals(emptyPageSource.getClass(), EmptyPageSource.class);
}

try (ConnectorPageSource nonEmptyPageSource = createTestingPageSource(transaction, config, tempFile.file(), getDynamicFilter(getNonSelectivePartitionTupleDomain()))) {
assertEquals(nonEmptyPageSource.getClass(), HivePageSource.class);
}
try (ConnectorPageSource emptyPageSource = createTestingPageSource(transaction, config, getDynamicFilter(getTupleDomainForPartitionSplitPruning()))) {
assertEquals(emptyPageSource.getClass(), EmptyPageSource.class);
}

try (ConnectorPageSource nonEmptyPageSource = createTestingPageSource(transaction, config, getDynamicFilter(getNonSelectivePartitionTupleDomain()))) {
assertEquals(nonEmptyPageSource.getClass(), HivePageSource.class);
}
}

private static ConnectorPageSource createTestingPageSource(HiveTransactionHandle transaction, HiveConfig hiveConfig, File outputFile, DynamicFilter dynamicFilter)
private static ConnectorPageSource createTestingPageSource(HiveTransactionHandle transaction, HiveConfig hiveConfig, DynamicFilter dynamicFilter)
throws IOException
{
Location location = Location.of("memory:///file");
TrinoFileSystemFactory fileSystemFactory = new MemoryFileSystemFactory();
fileSystemFactory.create(ConnectorIdentity.ofUser("test")).newOutputFile(location).create().close();

Properties splitProperties = new Properties();
splitProperties.setProperty(FILE_INPUT_FORMAT, hiveConfig.getHiveStorageFormat().getInputFormat());
splitProperties.setProperty(SERIALIZATION_LIB, hiveConfig.getHiveStorageFormat().getSerde());
HiveSplit split = new HiveSplit(
"",
"file:///" + outputFile.getAbsolutePath(),
location.toString(),
0,
0,
0,
0,
outputFile.length(),
outputFile.length(),
outputFile.lastModified(),
splitProperties,
ImmutableList.of(new HivePartitionKey(PARTITION_COLUMN.getName(), "42")),
ImmutableList.of(),
Expand Down Expand Up @@ -156,7 +159,7 @@ private static ConnectorPageSource createTestingPageSource(HiveTransactionHandle
HivePageSourceProvider provider = new HivePageSourceProvider(
TESTING_TYPE_MANAGER,
hiveConfig,
getDefaultHivePageSourceFactories(HDFS_ENVIRONMENT, hiveConfig));
getDefaultHivePageSourceFactories(fileSystemFactory, hiveConfig));

return provider.createPageSource(
transaction,
Expand Down
11 changes: 9 additions & 2 deletions ...rino-hive/src/test/java/io/trino/plugin/hive/metastore/file/TestingFileHiveMetastore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
*/
package io.trino.plugin.hive.metastore.file;

import io.trino.filesystem.Location;
import io.trino.filesystem.TrinoFileSystemFactory;
import io.trino.plugin.hive.NodeVersion;
import io.trino.plugin.hive.metastore.HiveMetastoreConfig;

Expand All @@ -25,13 +27,18 @@ public final class TestingFileHiveMetastore
private TestingFileHiveMetastore() {}

public static FileHiveMetastore createTestingFileHiveMetastore(File catalogDirectory)
{
return createTestingFileHiveMetastore(HDFS_FILE_SYSTEM_FACTORY, Location.of(catalogDirectory.toURI().toString()));
}

public static FileHiveMetastore createTestingFileHiveMetastore(TrinoFileSystemFactory fileSystemFactory, Location catalogDirectory)
{
return new FileHiveMetastore(
new NodeVersion("testversion"),
HDFS_FILE_SYSTEM_FACTORY,
fileSystemFactory,
new HiveMetastoreConfig().isHideDeltaLakeTables(),
new FileHiveMetastoreConfig()
.setCatalogDirectory(catalogDirectory.toURI().toString())
.setCatalogDirectory(catalogDirectory.toString())
.setMetastoreUser("test"));
}
}
Loading