diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveColumnProperties.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveColumnProperties.java index 82eafa1c04c5..11f3da8cb590 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveColumnProperties.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveColumnProperties.java @@ -14,7 +14,7 @@ package io.trino.plugin.hive; import com.google.common.collect.ImmutableList; -import io.trino.plugin.hive.aws.athena.projection.ProjectionType; +import io.trino.plugin.hive.projection.ProjectionType; import io.trino.spi.session.PropertyMetadata; import io.trino.spi.type.ArrayType; @@ -22,13 +22,13 @@ import java.util.List; import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_DIGITS; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_FORMAT; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_INTERVAL; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_INTERVAL_UNIT; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_RANGE; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_TYPE; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_VALUES; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.COLUMN_PROJECTION_DIGITS; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.COLUMN_PROJECTION_FORMAT; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.COLUMN_PROJECTION_INTERVAL; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.COLUMN_PROJECTION_INTERVAL_UNIT; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.COLUMN_PROJECTION_RANGE; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.COLUMN_PROJECTION_TYPE; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.COLUMN_PROJECTION_VALUES; import static io.trino.spi.session.PropertyMetadata.enumProperty; import static io.trino.spi.session.PropertyMetadata.integerProperty; import static io.trino.spi.session.PropertyMetadata.stringProperty; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java index 112f41e4f63f..02f7652ad295 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java @@ -39,7 +39,6 @@ import io.trino.plugin.hive.LocationService.WriteInfo; import io.trino.plugin.hive.acid.AcidOperation; import io.trino.plugin.hive.acid.AcidTransaction; -import io.trino.plugin.hive.aws.athena.PartitionProjectionService; import io.trino.plugin.hive.fs.DirectoryLister; import io.trino.plugin.hive.metastore.Column; import io.trino.plugin.hive.metastore.Database; @@ -270,6 +269,9 @@ import static io.trino.plugin.hive.metastore.StorageFormat.VIEW_STORAGE_FORMAT; import static io.trino.plugin.hive.metastore.StorageFormat.fromHiveStorageFormat; import static io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.STATS_PROPERTIES; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.arePartitionProjectionPropertiesSet; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.getPartitionProjectionHiveTableProperties; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.getPartitionProjectionTrinoTableProperties; import static io.trino.plugin.hive.type.Category.PRIMITIVE; import static io.trino.plugin.hive.util.AcidTables.deltaSubdir; import static io.trino.plugin.hive.util.AcidTables.isFullAcidTable; @@ -301,6 +303,7 @@ import static io.trino.plugin.hive.util.Statistics.reduce; import static io.trino.plugin.hive.util.SystemTables.getSourceTableNameFromSystemTable; import static io.trino.spi.StandardErrorCode.INVALID_ANALYZE_PROPERTY; +import static io.trino.spi.StandardErrorCode.INVALID_COLUMN_PROPERTY; import static io.trino.spi.StandardErrorCode.INVALID_SCHEMA_PROPERTY; import static io.trino.spi.StandardErrorCode.INVALID_TABLE_PROPERTY; import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; @@ -383,7 +386,7 @@ public class HiveMetadata private final HiveMaterializedViewMetadata hiveMaterializedViewMetadata; private final AccessControlMetadata accessControlMetadata; private final DirectoryLister directoryLister; - private final PartitionProjectionService partitionProjectionService; + private final boolean partitionProjectionEnabled; private final boolean allowTableRename; private final long maxPartitionDropsPerQuery; private final HiveTimestampPrecision hiveViewsTimestampPrecision; @@ -411,7 +414,7 @@ public HiveMetadata( HiveMaterializedViewMetadata hiveMaterializedViewMetadata, AccessControlMetadata accessControlMetadata, DirectoryLister directoryLister, - PartitionProjectionService partitionProjectionService, + boolean partitionProjectionEnabled, boolean allowTableRename, long maxPartitionDropsPerQuery, HiveTimestampPrecision hiveViewsTimestampPrecision) @@ -438,7 +441,7 @@ public HiveMetadata( this.hiveMaterializedViewMetadata = requireNonNull(hiveMaterializedViewMetadata, "hiveMaterializedViewMetadata is null"); this.accessControlMetadata = requireNonNull(accessControlMetadata, "accessControlMetadata is null"); this.directoryLister = requireNonNull(directoryLister, "directoryLister is null"); - this.partitionProjectionService = requireNonNull(partitionProjectionService, "partitionProjectionService is null"); + this.partitionProjectionEnabled = partitionProjectionEnabled; this.allowTableRename = allowTableRename; this.maxPartitionDropsPerQuery = maxPartitionDropsPerQuery; this.hiveViewsTimestampPrecision = requireNonNull(hiveViewsTimestampPrecision, "hiveViewsTimestampPrecision is null"); @@ -741,7 +744,7 @@ else if (isTrinoView || isTrinoMaterializedView) { } // Partition Projection specific properties - properties.putAll(partitionProjectionService.getPartitionProjectionTrinoTableProperties(table)); + properties.putAll(getPartitionProjectionTrinoTableProperties(table)); return new ConnectorTableMetadata(tableName, columns, properties.buildOrThrow(), comment); } @@ -1219,7 +1222,14 @@ else if (avroSchemaLiteral != null) { tableMetadata.getComment().ifPresent(value -> tableProperties.put(TABLE_COMMENT, value)); // Partition Projection specific properties - tableProperties.putAll(partitionProjectionService.getPartitionProjectionHiveTableProperties(tableMetadata)); + if (partitionProjectionEnabled) { + tableProperties.putAll(getPartitionProjectionHiveTableProperties(tableMetadata)); + } + else if (arePartitionProjectionPropertiesSet(tableMetadata)) { + throw new TrinoException( + INVALID_COLUMN_PROPERTY, + "Partition projection is disabled. Enable it in configuration by setting " + HiveConfig.CONFIGURATION_HIVE_PARTITION_PROJECTION_ENABLED + "=true"); + } Map baseProperties = tableProperties.buildOrThrow(); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadataFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadataFactory.java index 9e04878a434a..92b66ea251f3 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadataFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadataFactory.java @@ -20,7 +20,6 @@ import io.airlift.units.Duration; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.plugin.base.CatalogName; -import io.trino.plugin.hive.aws.athena.PartitionProjectionService; import io.trino.plugin.hive.fs.DirectoryLister; import io.trino.plugin.hive.fs.TransactionScopeCachingDirectoryListerFactory; import io.trino.plugin.hive.metastore.HiveMetastoreConfig; @@ -77,7 +76,7 @@ public class HiveMetadataFactory private final ScheduledExecutorService heartbeatService; private final DirectoryLister directoryLister; private final TransactionScopeCachingDirectoryListerFactory transactionScopeCachingDirectoryListerFactory; - private final PartitionProjectionService partitionProjectionService; + private final boolean partitionProjectionEnabled; private final boolean allowTableRename; private final HiveTimestampPrecision hiveViewsTimestampPrecision; @@ -103,7 +102,6 @@ public HiveMetadataFactory( AccessControlMetadataFactory accessControlMetadataFactory, DirectoryLister directoryLister, TransactionScopeCachingDirectoryListerFactory transactionScopeCachingDirectoryListerFactory, - PartitionProjectionService partitionProjectionService, @AllowHiveTableRename boolean allowTableRename) { this( @@ -139,7 +137,7 @@ public HiveMetadataFactory( accessControlMetadataFactory, directoryLister, transactionScopeCachingDirectoryListerFactory, - partitionProjectionService, + hiveConfig.isPartitionProjectionEnabled(), allowTableRename, hiveConfig.getTimestampPrecision()); } @@ -177,7 +175,7 @@ public HiveMetadataFactory( AccessControlMetadataFactory accessControlMetadataFactory, DirectoryLister directoryLister, TransactionScopeCachingDirectoryListerFactory transactionScopeCachingDirectoryListerFactory, - PartitionProjectionService partitionProjectionService, + boolean partitionProjectionEnabled, boolean allowTableRename, HiveTimestampPrecision hiveViewsTimestampPrecision) { @@ -220,7 +218,7 @@ public HiveMetadataFactory( this.heartbeatService = requireNonNull(heartbeatService, "heartbeatService is null"); this.directoryLister = requireNonNull(directoryLister, "directoryLister is null"); this.transactionScopeCachingDirectoryListerFactory = requireNonNull(transactionScopeCachingDirectoryListerFactory, "transactionScopeCachingDirectoryListerFactory is null"); - this.partitionProjectionService = requireNonNull(partitionProjectionService, "partitionProjectionService is null"); + this.partitionProjectionEnabled = partitionProjectionEnabled; this.allowTableRename = allowTableRename; this.hiveViewsTimestampPrecision = requireNonNull(hiveViewsTimestampPrecision, "hiveViewsTimestampPrecision is null"); } @@ -229,7 +227,7 @@ public HiveMetadataFactory( public TransactionalMetadata create(ConnectorIdentity identity, boolean autoCommit) { CachingHiveMetastore cachingHiveMetastore = createPerTransactionCache(metastoreFactory.createMetastore(Optional.of(identity)), perTransactionCacheMaximumSize); - HiveMetastoreClosure hiveMetastoreClosure = new HiveMetastoreClosure(cachingHiveMetastore); + HiveMetastoreClosure hiveMetastoreClosure = new HiveMetastoreClosure(cachingHiveMetastore, typeManager, partitionProjectionEnabled); DirectoryLister directoryLister = transactionScopeCachingDirectoryListerFactory.get(this.directoryLister); SemiTransactionalHiveMetastore metastore = new SemiTransactionalHiveMetastore( @@ -268,7 +266,7 @@ public TransactionalMetadata create(ConnectorIdentity identity, boolean autoComm hiveMaterializedViewMetadataFactory.create(hiveMetastoreClosure), accessControlMetadataFactory.create(metastore), directoryLister, - partitionProjectionService, + partitionProjectionEnabled, allowTableRename, maxPartitionDropsPerQuery, hiveViewsTimestampPrecision); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetastoreClosure.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetastoreClosure.java index 21e0141b8f6a..190e81bd28fe 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetastoreClosure.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetastoreClosure.java @@ -28,6 +28,8 @@ import io.trino.plugin.hive.metastore.PartitionWithStatistics; import io.trino.plugin.hive.metastore.PrincipalPrivileges; import io.trino.plugin.hive.metastore.Table; +import io.trino.plugin.hive.projection.PartitionProjection; +import io.trino.spi.TrinoException; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.connector.TableNotFoundException; import io.trino.spi.function.LanguageFunction; @@ -35,6 +37,7 @@ import io.trino.spi.predicate.TupleDomain; import io.trino.spi.security.RoleGrant; import io.trino.spi.type.Type; +import io.trino.spi.type.TypeManager; import java.util.Collection; import java.util.List; @@ -47,20 +50,26 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.Maps.immutableEntry; +import static io.trino.plugin.hive.HiveErrorCode.HIVE_TABLE_DROPPED_DURING_QUERY; import static io.trino.plugin.hive.HivePartitionManager.extractPartitionValues; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.getPartitionProjectionFromTable; import static java.util.Objects.requireNonNull; public class HiveMetastoreClosure { private final HiveMetastore delegate; + private final TypeManager typeManager; + private final boolean partitionProjectionEnabled; /** * Do not use this directly. Instead, the closure should be fetched from the current SemiTransactionalHiveMetastore, * which can be fetched from the current HiveMetadata. */ - public HiveMetastoreClosure(HiveMetastore delegate) + public HiveMetastoreClosure(HiveMetastore delegate, TypeManager typeManager, boolean partitionProjectionEnabled) { this.delegate = requireNonNull(delegate, "delegate is null"); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + this.partitionProjectionEnabled = partitionProjectionEnabled; } public Optional getDatabase(String databaseName) @@ -247,12 +256,21 @@ public Optional> getPartitionNamesByFilter( List columnNames, TupleDomain partitionKeysFilter) { + if (partitionProjectionEnabled) { + Table table = getTable(databaseName, tableName) + .orElseThrow(() -> new TrinoException(HIVE_TABLE_DROPPED_DURING_QUERY, "Table does not exists: " + tableName)); + + Optional projection = getPartitionProjectionFromTable(table, typeManager); + if (projection.isPresent()) { + return projection.get().getProjectedPartitionNamesByFilter(columnNames, partitionKeysFilter); + } + } return delegate.getPartitionNamesByFilter(databaseName, tableName, columnNames, partitionKeysFilter); } private List getExistingPartitionsByNames(Table table, List partitionNames) { - Map partitions = delegate.getPartitionsByNames(table, partitionNames).entrySet().stream() + Map partitions = getPartitionsByNames(table, partitionNames).entrySet().stream() .map(entry -> immutableEntry(entry.getKey(), entry.getValue().orElseThrow(() -> new PartitionNotFoundException(table.getSchemaTableName(), extractPartitionValues(entry.getKey()))))) .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -265,11 +283,22 @@ private List getExistingPartitionsByNames(Table table, List p public Map> getPartitionsByNames(String databaseName, String tableName, List partitionNames) { return delegate.getTable(databaseName, tableName) - .map(table -> delegate.getPartitionsByNames(table, partitionNames)) + .map(table -> getPartitionsByNames(table, partitionNames)) .orElseGet(() -> partitionNames.stream() .collect(toImmutableMap(name -> name, name -> Optional.empty()))); } + private Map> getPartitionsByNames(Table table, List partitionNames) + { + if (partitionProjectionEnabled) { + Optional projection = getPartitionProjectionFromTable(table, typeManager); + if (projection.isPresent()) { + return projection.get().getProjectedPartitionsByNames(table, partitionNames); + } + } + return delegate.getPartitionsByNames(table, partitionNames); + } + public void addPartitions(String databaseName, String tableName, List partitions) { delegate.addPartitions(databaseName, tableName, partitions); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSinkProvider.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSinkProvider.java index 1b4219dce3c5..115f277e236d 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSinkProvider.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSinkProvider.java @@ -172,7 +172,7 @@ private HivePageSink createPageSink(HiveWritableTableHandle handle, boolean isCr handle.getLocationHandle(), locationService, session.getQueryId(), - new HivePageSinkMetadataProvider(handle.getPageSinkMetadata(), new HiveMetastoreClosure(cachingHiveMetastore)), + new HivePageSinkMetadataProvider(handle.getPageSinkMetadata(), new HiveMetastoreClosure(cachingHiveMetastore, typeManager, false)), typeManager, pageSorter, writerSortBufferSize, diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveTableProperties.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveTableProperties.java index 51a70f9bc7e6..321a57bb977a 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveTableProperties.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveTableProperties.java @@ -30,9 +30,9 @@ import java.util.Optional; import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.PARTITION_PROJECTION_ENABLED; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.PARTITION_PROJECTION_IGNORE; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.PARTITION_PROJECTION_LOCATION_TEMPLATE; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.PARTITION_PROJECTION_ENABLED; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.PARTITION_PROJECTION_IGNORE; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.PARTITION_PROJECTION_LOCATION_TEMPLATE; import static io.trino.plugin.hive.util.HiveBucketing.BucketingVersion.BUCKETING_V1; import static io.trino.plugin.hive.util.HiveBucketing.BucketingVersion.BUCKETING_V2; import static io.trino.spi.StandardErrorCode.INVALID_TABLE_PROPERTY; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/InternalHiveConnectorFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/InternalHiveConnectorFactory.java index b740d5f42fb0..995606d56797 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/InternalHiveConnectorFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/InternalHiveConnectorFactory.java @@ -39,7 +39,6 @@ import io.trino.plugin.base.jmx.ConnectorObjectNameGeneratorModule; import io.trino.plugin.base.jmx.MBeanServerModule; import io.trino.plugin.base.session.SessionPropertiesProvider; -import io.trino.plugin.hive.aws.athena.PartitionProjectionModule; import io.trino.plugin.hive.fs.CachingDirectoryListerModule; import io.trino.plugin.hive.fs.DirectoryLister; import io.trino.plugin.hive.metastore.HiveMetastore; @@ -106,7 +105,6 @@ public static Connector createConnector( new JsonModule(), new TypeDeserializerModule(context.getTypeManager()), new HiveModule(), - new PartitionProjectionModule(), new CachingDirectoryListerModule(directoryLister), new HiveMetastoreModule(metastore), new HiveSecurityModule(), diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/PartitionProjectionMetastoreDecorator.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/PartitionProjectionMetastoreDecorator.java deleted file mode 100644 index c221e87e5ad9..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/PartitionProjectionMetastoreDecorator.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.trino.plugin.hive.aws.athena; - -import com.google.inject.Inject; -import io.trino.plugin.hive.metastore.ForwardingHiveMetastore; -import io.trino.plugin.hive.metastore.HiveMetastore; -import io.trino.plugin.hive.metastore.HiveMetastoreDecorator; -import io.trino.plugin.hive.metastore.Partition; -import io.trino.plugin.hive.metastore.Table; -import io.trino.spi.TrinoException; -import io.trino.spi.predicate.TupleDomain; - -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import static io.trino.plugin.hive.HiveErrorCode.HIVE_TABLE_DROPPED_DURING_QUERY; -import static java.util.Objects.requireNonNull; - -public class PartitionProjectionMetastoreDecorator - implements HiveMetastoreDecorator -{ - private final PartitionProjectionService partitionProjectionService; - - @Inject - public PartitionProjectionMetastoreDecorator(PartitionProjectionService partitionProjectionService) - { - this.partitionProjectionService = requireNonNull(partitionProjectionService, "partitionProjectionService is null"); - } - - @Override - public int getPriority() - { - return PRIORITY_PARTITION_PROJECTION; - } - - @Override - public HiveMetastore decorate(HiveMetastore hiveMetastore) - { - return new PartitionProjectionMetastore(hiveMetastore, partitionProjectionService); - } - - private static class PartitionProjectionMetastore - extends ForwardingHiveMetastore - { - private final PartitionProjectionService partitionProjectionService; - - public PartitionProjectionMetastore(HiveMetastore hiveMetastore, PartitionProjectionService partitionProjectionService) - { - super(hiveMetastore); - this.partitionProjectionService = requireNonNull(partitionProjectionService, "partitionProjectionService is null"); - } - - @Override - public Optional> getPartitionNamesByFilter(String databaseName, String tableName, List columnNames, TupleDomain partitionKeysFilter) - { - Table table = super.getTable(databaseName, tableName) - .orElseThrow(() -> new TrinoException(HIVE_TABLE_DROPPED_DURING_QUERY, "Table does not exists: " + tableName)); - - Optional projection = getPartitionProjection(table); - if (projection.isPresent()) { - return projection.get().getProjectedPartitionNamesByFilter(columnNames, partitionKeysFilter); - } - - return super.getPartitionNamesByFilter(databaseName, tableName, columnNames, partitionKeysFilter); - } - - @Override - public Map> getPartitionsByNames(Table table, List partitionNames) - { - Optional projection = getPartitionProjection(table); - if (projection.isPresent()) { - return projection.get().getProjectedPartitionsByNames(table, partitionNames); - } - return super.getPartitionsByNames(table, partitionNames); - } - - private Optional getPartitionProjection(Table table) - { - return partitionProjectionService.getPartitionProjectionFromTable(table) - .filter(PartitionProjection::isEnabled); - } - } -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/PartitionProjectionModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/PartitionProjectionModule.java deleted file mode 100644 index e1c3ae1ff185..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/PartitionProjectionModule.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.aws.athena; - -import com.google.inject.Binder; -import com.google.inject.Scopes; -import com.google.inject.multibindings.MapBinder; -import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.trino.plugin.hive.aws.athena.projection.DateProjectionFactory; -import io.trino.plugin.hive.aws.athena.projection.EnumProjectionFactory; -import io.trino.plugin.hive.aws.athena.projection.InjectedProjectionFactory; -import io.trino.plugin.hive.aws.athena.projection.IntegerProjectionFactory; -import io.trino.plugin.hive.aws.athena.projection.ProjectionFactory; -import io.trino.plugin.hive.aws.athena.projection.ProjectionType; -import io.trino.plugin.hive.metastore.HiveMetastoreDecorator; - -import static com.google.inject.multibindings.MapBinder.newMapBinder; -import static com.google.inject.multibindings.Multibinder.newSetBinder; - -public class PartitionProjectionModule - extends AbstractConfigurationAwareModule -{ - @Override - public void setup(Binder binder) - { - MapBinder projectionFactoriesBinder = - newMapBinder(binder, ProjectionType.class, ProjectionFactory.class); - projectionFactoriesBinder.addBinding(ProjectionType.ENUM).to(EnumProjectionFactory.class).in(Scopes.SINGLETON); - projectionFactoriesBinder.addBinding(ProjectionType.INTEGER).to(IntegerProjectionFactory.class).in(Scopes.SINGLETON); - projectionFactoriesBinder.addBinding(ProjectionType.DATE).to(DateProjectionFactory.class).in(Scopes.SINGLETON); - projectionFactoriesBinder.addBinding(ProjectionType.INJECTED).to(InjectedProjectionFactory.class).in(Scopes.SINGLETON); - - binder.bind(PartitionProjectionService.class).in(Scopes.SINGLETON); - - newSetBinder(binder, HiveMetastoreDecorator.class) - .addBinding() - .to(PartitionProjectionMetastoreDecorator.class) - .in(Scopes.SINGLETON); - } -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/PartitionProjectionProperties.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/PartitionProjectionProperties.java deleted file mode 100644 index 38733e0f1f74..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/PartitionProjectionProperties.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.aws.athena; - -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; - -import static io.trino.plugin.hive.aws.athena.projection.Projection.invalidProjectionException; -import static java.lang.String.format; - -public final class PartitionProjectionProperties -{ - /** - * General properties suffixes - */ - static final String TABLE_PROJECTION_ENABLED_SUFFIX = "enabled"; - /** - * Forces Trino to not use Athena table projection for a given table. - * Kill switch to be used as a workaround if compatibility issues are found. - */ - static final String TABLE_PROJECTION_IGNORE_SUFFIX = "ignore"; - static final String COLUMN_PROJECTION_TYPE_SUFFIX = "type"; - static final String COLUMN_PROJECTION_VALUES_SUFFIX = "values"; - static final String COLUMN_PROJECTION_RANGE_SUFFIX = "range"; - static final String COLUMN_PROJECTION_INTERVAL_SUFFIX = "interval"; - static final String COLUMN_PROJECTION_DIGITS_SUFFIX = "digits"; - static final String COLUMN_PROJECTION_FORMAT_SUFFIX = "format"; - - /** - * Metastore table properties - */ - private static final String METASTORE_PROPERTY_SEPARATOR = "."; - - private static final String METASTORE_PROPERTY_PREFIX = "projection" + METASTORE_PROPERTY_SEPARATOR; - - static final String METASTORE_PROPERTY_PROJECTION_INTERVAL_UNIT_SUFFIX = "interval" + METASTORE_PROPERTY_SEPARATOR + "unit"; - - static final String METASTORE_PROPERTY_PROJECTION_ENABLED = METASTORE_PROPERTY_PREFIX + TABLE_PROJECTION_ENABLED_SUFFIX; - static final String METASTORE_PROPERTY_PROJECTION_LOCATION_TEMPLATE = "storage" + METASTORE_PROPERTY_SEPARATOR + "location" + METASTORE_PROPERTY_SEPARATOR + "template"; - /** - * See {@link #TABLE_PROJECTION_IGNORE_SUFFIX } to understand duplication with enable property - **/ - static final String METASTORE_PROPERTY_PROJECTION_IGNORE = "trino" + METASTORE_PROPERTY_SEPARATOR + "partition_projection" + METASTORE_PROPERTY_SEPARATOR + TABLE_PROJECTION_IGNORE_SUFFIX; - - /** - * Trino table properties - */ - private static final String PROPERTY_KEY_SEPARATOR = "_"; - - static final String PROPERTY_KEY_PREFIX = "partition" + PROPERTY_KEY_SEPARATOR + "projection" + PROPERTY_KEY_SEPARATOR; - - private static final String PROPERTY_KEY_SUFFIX_COLUMN_PROJECTION_INTERVAL_UNIT = "interval" + PROPERTY_KEY_SEPARATOR + "unit"; - - public static final String PARTITION_PROJECTION_ENABLED = PROPERTY_KEY_PREFIX + TABLE_PROJECTION_ENABLED_SUFFIX; - public static final String PARTITION_PROJECTION_LOCATION_TEMPLATE = PROPERTY_KEY_PREFIX + "location" + PROPERTY_KEY_SEPARATOR + "template"; - /** - * See {@link #TABLE_PROJECTION_IGNORE_SUFFIX } to understand duplication with enable property - **/ - public static final String PARTITION_PROJECTION_IGNORE = PROPERTY_KEY_PREFIX + TABLE_PROJECTION_IGNORE_SUFFIX; - - public static final String COLUMN_PROJECTION_TYPE = PROPERTY_KEY_PREFIX + COLUMN_PROJECTION_TYPE_SUFFIX; - public static final String COLUMN_PROJECTION_VALUES = PROPERTY_KEY_PREFIX + COLUMN_PROJECTION_VALUES_SUFFIX; - public static final String COLUMN_PROJECTION_RANGE = PROPERTY_KEY_PREFIX + COLUMN_PROJECTION_RANGE_SUFFIX; - public static final String COLUMN_PROJECTION_INTERVAL = PROPERTY_KEY_PREFIX + COLUMN_PROJECTION_INTERVAL_SUFFIX; - public static final String COLUMN_PROJECTION_INTERVAL_UNIT = PROPERTY_KEY_PREFIX + PROPERTY_KEY_SUFFIX_COLUMN_PROJECTION_INTERVAL_UNIT; - public static final String COLUMN_PROJECTION_DIGITS = PROPERTY_KEY_PREFIX + COLUMN_PROJECTION_DIGITS_SUFFIX; - public static final String COLUMN_PROJECTION_FORMAT = PROPERTY_KEY_PREFIX + COLUMN_PROJECTION_FORMAT_SUFFIX; - - static String getMetastoreProjectionPropertyKey(String columnName, String propertyKeySuffix) - { - return METASTORE_PROPERTY_PREFIX + columnName + METASTORE_PROPERTY_SEPARATOR + propertyKeySuffix; - } - - public static T getProjectionPropertyRequiredValue( - String columnName, - Map columnProjectionProperties, - String propertyKey, - Function decoder) - { - return getProjectionPropertyValue(columnProjectionProperties, propertyKey, decoder) - .orElseThrow(() -> invalidProjectionException(columnName, format("Missing required property: '%s'", propertyKey))); - } - - public static Optional getProjectionPropertyValue( - Map columnProjectionProperties, - String propertyKey, - Function decoder) - { - return Optional.ofNullable( - columnProjectionProperties.get(propertyKey)) - .map(value -> decoder.apply(value)); - } - - private PartitionProjectionProperties() - { - } -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/PartitionProjectionService.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/PartitionProjectionService.java deleted file mode 100644 index d9175050adec..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/PartitionProjectionService.java +++ /dev/null @@ -1,381 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.aws.athena; - -import com.google.common.base.Joiner; -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.inject.Inject; -import io.trino.plugin.hive.HiveConfig; -import io.trino.plugin.hive.aws.athena.projection.Projection; -import io.trino.plugin.hive.aws.athena.projection.ProjectionFactory; -import io.trino.plugin.hive.aws.athena.projection.ProjectionType; -import io.trino.plugin.hive.metastore.Column; -import io.trino.plugin.hive.metastore.Table; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeManager; - -import java.time.temporal.ChronoUnit; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.google.common.collect.ImmutableMap.toImmutableMap; -import static io.trino.plugin.hive.HiveTableProperties.getPartitionedBy; -import static io.trino.plugin.hive.HiveTimestampPrecision.DEFAULT_PRECISION; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_DIGITS; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_DIGITS_SUFFIX; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_FORMAT; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_FORMAT_SUFFIX; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_INTERVAL; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_INTERVAL_SUFFIX; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_INTERVAL_UNIT; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_RANGE; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_RANGE_SUFFIX; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_TYPE; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_TYPE_SUFFIX; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_VALUES; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_VALUES_SUFFIX; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.METASTORE_PROPERTY_PROJECTION_ENABLED; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.METASTORE_PROPERTY_PROJECTION_IGNORE; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.METASTORE_PROPERTY_PROJECTION_INTERVAL_UNIT_SUFFIX; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.METASTORE_PROPERTY_PROJECTION_LOCATION_TEMPLATE; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.PARTITION_PROJECTION_ENABLED; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.PARTITION_PROJECTION_IGNORE; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.PARTITION_PROJECTION_LOCATION_TEMPLATE; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.PROPERTY_KEY_PREFIX; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.getMetastoreProjectionPropertyKey; -import static io.trino.plugin.hive.aws.athena.projection.Projection.unsupportedProjectionColumnTypeException; -import static io.trino.spi.StandardErrorCode.INVALID_COLUMN_PROPERTY; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; -import static java.util.function.Function.identity; - -public final class PartitionProjectionService -{ - private final boolean partitionProjectionEnabled; - private final Map projectionFactories; - private final TypeManager typeManager; - - @Inject - public PartitionProjectionService( - HiveConfig hiveConfig, - Map projectionFactories, - TypeManager typeManager) - { - this.partitionProjectionEnabled = hiveConfig.isPartitionProjectionEnabled(); - this.typeManager = requireNonNull(typeManager, "typeManager is null"); - this.projectionFactories = ImmutableMap.copyOf(requireNonNull(projectionFactories, "projectionFactories is null")); - } - - public Map getPartitionProjectionTrinoTableProperties(Table table) - { - Map metastoreTableProperties = table.getParameters(); - ImmutableMap.Builder trinoTablePropertiesBuilder = ImmutableMap.builder(); - rewriteProperty(metastoreTableProperties, trinoTablePropertiesBuilder, METASTORE_PROPERTY_PROJECTION_IGNORE, PARTITION_PROJECTION_IGNORE, Boolean::valueOf); - rewriteProperty(metastoreTableProperties, trinoTablePropertiesBuilder, METASTORE_PROPERTY_PROJECTION_ENABLED, PARTITION_PROJECTION_ENABLED, Boolean::valueOf); - rewriteProperty(metastoreTableProperties, trinoTablePropertiesBuilder, METASTORE_PROPERTY_PROJECTION_LOCATION_TEMPLATE, PARTITION_PROJECTION_LOCATION_TEMPLATE, String::valueOf); - return trinoTablePropertiesBuilder.buildOrThrow(); - } - - public static Map getPartitionProjectionTrinoColumnProperties(Table table, String columnName) - { - Map metastoreTableProperties = table.getParameters(); - return rewriteColumnProjectionProperties(metastoreTableProperties, columnName); - } - - public Map getPartitionProjectionHiveTableProperties(ConnectorTableMetadata tableMetadata) - { - // If partition projection is globally disabled we don't allow defining its properties - if (!partitionProjectionEnabled && isAnyPartitionProjectionPropertyUsed(tableMetadata)) { - throw columnProjectionException("Partition projection is disabled. Enable it in configuration by setting " - + HiveConfig.CONFIGURATION_HIVE_PARTITION_PROJECTION_ENABLED + "=true"); - } - - ImmutableMap.Builder metastoreTablePropertiesBuilder = ImmutableMap.builder(); - // Handle Table Properties - Map trinoTableProperties = tableMetadata.getProperties(); - rewriteProperty( - trinoTableProperties, - metastoreTablePropertiesBuilder, - PARTITION_PROJECTION_IGNORE, - METASTORE_PROPERTY_PROJECTION_IGNORE, - value -> value.toString().toLowerCase(Locale.ENGLISH)); - rewriteProperty( - trinoTableProperties, - metastoreTablePropertiesBuilder, - PARTITION_PROJECTION_ENABLED, - METASTORE_PROPERTY_PROJECTION_ENABLED, - value -> value.toString().toLowerCase(Locale.ENGLISH)); - rewriteProperty( - trinoTableProperties, - metastoreTablePropertiesBuilder, - PARTITION_PROJECTION_LOCATION_TEMPLATE, - METASTORE_PROPERTY_PROJECTION_LOCATION_TEMPLATE, - Object::toString); - - // Handle Column Properties - tableMetadata.getColumns().stream() - .filter(columnMetadata -> !columnMetadata.getProperties().isEmpty()) - .forEach(columnMetadata -> { - Map columnProperties = columnMetadata.getProperties(); - String columnName = columnMetadata.getName(); - rewriteProperty( - columnProperties, - metastoreTablePropertiesBuilder, - COLUMN_PROJECTION_TYPE, - getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_TYPE_SUFFIX), - value -> ((ProjectionType) value).name().toLowerCase(Locale.ENGLISH)); - rewriteProperty( - columnProperties, - metastoreTablePropertiesBuilder, - COLUMN_PROJECTION_VALUES, - getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_VALUES_SUFFIX), - value -> Joiner.on(",").join((List) value)); - rewriteProperty( - columnProperties, - metastoreTablePropertiesBuilder, - COLUMN_PROJECTION_RANGE, - getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_RANGE_SUFFIX), - value -> Joiner.on(",").join((List) value)); - rewriteProperty( - columnProperties, - metastoreTablePropertiesBuilder, - COLUMN_PROJECTION_INTERVAL, - getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_INTERVAL_SUFFIX), - value -> ((Integer) value).toString()); - rewriteProperty( - columnProperties, - metastoreTablePropertiesBuilder, - COLUMN_PROJECTION_INTERVAL_UNIT, - getMetastoreProjectionPropertyKey(columnName, METASTORE_PROPERTY_PROJECTION_INTERVAL_UNIT_SUFFIX), - value -> ((ChronoUnit) value).name().toLowerCase(Locale.ENGLISH)); - rewriteProperty( - columnProperties, - metastoreTablePropertiesBuilder, - COLUMN_PROJECTION_DIGITS, - getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_DIGITS_SUFFIX), - value -> ((Integer) value).toString()); - rewriteProperty( - columnProperties, - metastoreTablePropertiesBuilder, - COLUMN_PROJECTION_FORMAT, - getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_FORMAT_SUFFIX), - String.class::cast); - }); - - // We initialize partition projection to validate properties. - Map metastoreTableProperties = metastoreTablePropertiesBuilder.buildOrThrow(); - List partitionColumnNames = getPartitionedBy(tableMetadata.getProperties()); - createPartitionProjection( - tableMetadata.getColumns() - .stream() - .map(ColumnMetadata::getName) - .collect(toImmutableList()), - tableMetadata.getColumns().stream() - .filter(columnMetadata -> partitionColumnNames.contains(columnMetadata.getName())) - .collect(toImmutableMap(ColumnMetadata::getName, ColumnMetadata::getType)), - metastoreTableProperties); - - return metastoreTableProperties; - } - - private boolean isAnyPartitionProjectionPropertyUsed(ConnectorTableMetadata tableMetadata) - { - if (tableMetadata.getProperties().keySet().stream() - .anyMatch(propertyKey -> propertyKey.startsWith(PROPERTY_KEY_PREFIX))) { - return true; - } - return tableMetadata.getColumns().stream() - .map(columnMetadata -> columnMetadata.getProperties().keySet()) - .flatMap(Set::stream) - .anyMatch(propertyKey -> propertyKey.startsWith(PROPERTY_KEY_PREFIX)); - } - - public Optional getPartitionProjectionFromTable(Table table) - { - if (!partitionProjectionEnabled) { - return Optional.empty(); - } - - Map tableProperties = table.getParameters(); - if (Optional.ofNullable(tableProperties.get(METASTORE_PROPERTY_PROJECTION_IGNORE)) - .map(Boolean::valueOf) - .orElse(false)) { - return Optional.empty(); - } - - return Optional.of( - createPartitionProjection( - table.getDataColumns() - .stream() - .map(Column::getName) - .collect(toImmutableList()), - table.getPartitionColumns() - .stream().collect(toImmutableMap( - Column::getName, - column -> column.getType().getType( - typeManager, - DEFAULT_PRECISION))), - tableProperties)); - } - - private PartitionProjection createPartitionProjection(List dataColumns, Map partitionColumns, Map tableProperties) - { - Optional projectionEnabledProperty = Optional.ofNullable(tableProperties.get(METASTORE_PROPERTY_PROJECTION_ENABLED)).map(Boolean::valueOf); - if (projectionEnabledProperty.orElse(false) && partitionColumns.size() < 1) { - throw columnProjectionException("Partition projection can't be enabled when no partition columns are defined."); - } - - Map columnProjections = ImmutableSet.builder() - .addAll(partitionColumns.keySet()) - .addAll(dataColumns) - .build() - .stream() - .collect(toImmutableMap( - identity(), - columnName -> rewriteColumnProjectionProperties(tableProperties, columnName))) - .entrySet() - .stream() - .filter(entry -> !entry.getValue().isEmpty()) - .collect(toImmutableMap( - Map.Entry::getKey, - entry -> { - String columnName = entry.getKey(); - if (partitionColumns.containsKey(columnName)) { - return parseColumnProjection(columnName, partitionColumns.get(columnName), entry.getValue()); - } - throw columnProjectionException("Partition projection can't be defined for non partition column: '" + columnName + "'"); - })); - - Optional storageLocationTemplate = Optional.ofNullable(tableProperties.get(METASTORE_PROPERTY_PROJECTION_LOCATION_TEMPLATE)); - if (projectionEnabledProperty.isPresent()) { - for (String columnName : partitionColumns.keySet()) { - if (!columnProjections.containsKey(columnName)) { - throw columnProjectionException("Partition projection definition for column: '" + columnName + "' missing"); - } - if (storageLocationTemplate.isPresent()) { - String locationTemplate = storageLocationTemplate.get(); - if (!locationTemplate.contains("${" + columnName + "}")) { - throw columnProjectionException(format("Partition projection location template: %s is missing partition column: '%s' placeholder", locationTemplate, columnName)); - } - } - } - } - else if (!columnProjections.isEmpty()) { - throw columnProjectionException(format( - "Columns %s projections are disallowed when partition projection property '%s' is missing", - columnProjections.keySet().stream().collect(Collectors.joining("', '", "['", "']")), - PARTITION_PROJECTION_ENABLED)); - } - - return new PartitionProjection(projectionEnabledProperty.orElse(false), storageLocationTemplate, columnProjections); - } - - private static Map rewriteColumnProjectionProperties(Map metastoreTableProperties, String columnName) - { - ImmutableMap.Builder trinoTablePropertiesBuilder = ImmutableMap.builder(); - rewriteProperty( - metastoreTableProperties, - trinoTablePropertiesBuilder, - getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_TYPE_SUFFIX), - COLUMN_PROJECTION_TYPE, - value -> ProjectionType.valueOf(value.toUpperCase(Locale.ENGLISH))); - rewriteProperty( - metastoreTableProperties, - trinoTablePropertiesBuilder, - getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_VALUES_SUFFIX), - COLUMN_PROJECTION_VALUES, - PartitionProjectionService::splitCommaSeparatedString); - rewriteProperty( - metastoreTableProperties, - trinoTablePropertiesBuilder, - getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_RANGE_SUFFIX), - COLUMN_PROJECTION_RANGE, - PartitionProjectionService::splitCommaSeparatedString); - rewriteProperty( - metastoreTableProperties, - trinoTablePropertiesBuilder, - getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_INTERVAL_SUFFIX), - COLUMN_PROJECTION_INTERVAL, - Integer::valueOf); - rewriteProperty( - metastoreTableProperties, - trinoTablePropertiesBuilder, - getMetastoreProjectionPropertyKey(columnName, METASTORE_PROPERTY_PROJECTION_INTERVAL_UNIT_SUFFIX), - COLUMN_PROJECTION_INTERVAL_UNIT, - value -> ChronoUnit.valueOf(value.toUpperCase(Locale.ENGLISH))); - rewriteProperty( - metastoreTableProperties, - trinoTablePropertiesBuilder, - getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_DIGITS_SUFFIX), - COLUMN_PROJECTION_DIGITS, - Integer::valueOf); - rewriteProperty( - metastoreTableProperties, - trinoTablePropertiesBuilder, - getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_FORMAT_SUFFIX), - COLUMN_PROJECTION_FORMAT, - value -> value); - return trinoTablePropertiesBuilder.buildOrThrow(); - } - - private Projection parseColumnProjection(String columnName, Type columnType, Map columnProperties) - { - ProjectionType projectionType = (ProjectionType) columnProperties.get(COLUMN_PROJECTION_TYPE); - if (Objects.isNull(projectionType)) { - throw columnProjectionException("Projection type property missing for column: '" + columnName + "'"); - } - ProjectionFactory projectionFactory = Optional.ofNullable(projectionFactories.get(projectionType)) - .orElseThrow(() -> columnProjectionException(format("Partition projection type %s for column: '%s' not supported", projectionType, columnName))); - if (!projectionFactory.isSupportedColumnType(columnType)) { - throw unsupportedProjectionColumnTypeException(columnName, columnType); - } - return projectionFactory.create(columnName, columnType, columnProperties); - } - - private static void rewriteProperty( - Map sourceProperties, - ImmutableMap.Builder targetPropertiesBuilder, - String sourcePropertyKey, - String targetPropertyKey, - Function valueMapper) - { - Optional.ofNullable(sourceProperties.get(sourcePropertyKey)) - .ifPresent(value -> targetPropertiesBuilder.put(targetPropertyKey, valueMapper.apply(value))); - } - - private TrinoException columnProjectionException(String message) - { - return new TrinoException(INVALID_COLUMN_PROPERTY, message); - } - - private static List splitCommaSeparatedString(String value) - { - return Splitter.on(',') - .trimResults() - .omitEmptyStrings() - .splitToList(value); - } -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/DateProjection.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/DateProjection.java deleted file mode 100644 index 20d9408f3aa3..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/DateProjection.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.aws.athena.projection; - -import com.google.common.collect.ImmutableList; -import io.trino.spi.predicate.Domain; -import io.trino.spi.type.DateType; -import io.trino.spi.type.TimestampType; -import io.trino.spi.type.Type; -import io.trino.spi.type.VarcharType; - -import java.text.DateFormat; -import java.text.ParseException; -import java.time.Instant; -import java.time.temporal.ChronoField; -import java.time.temporal.ChronoUnit; -import java.util.Date; -import java.util.List; -import java.util.Optional; -import java.util.function.Supplier; - -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.plugin.hive.aws.athena.projection.DateProjectionFactory.UTC_TIME_ZONE_ID; -import static io.trino.spi.predicate.Domain.singleValue; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -public class DateProjection - extends Projection -{ - private final DateFormat dateFormat; - private final Supplier leftBound; - private final Supplier rightBound; - private final int interval; - private final ChronoUnit intervalUnit; - - public DateProjection(String columnName, DateFormat dateFormat, Supplier leftBound, Supplier rightBound, int interval, ChronoUnit intervalUnit) - { - super(columnName); - this.dateFormat = requireNonNull(dateFormat, "dateFormatPattern is null"); - this.leftBound = requireNonNull(leftBound, "leftBound is null"); - this.rightBound = requireNonNull(rightBound, "rightBound is null"); - this.interval = interval; - this.intervalUnit = requireNonNull(intervalUnit, "intervalUnit is null"); - } - - @Override - public List getProjectedValues(Optional partitionValueFilter) - { - ImmutableList.Builder builder = ImmutableList.builder(); - - Instant leftBound = adjustBoundToDateFormat(this.leftBound.get()); - Instant rightBound = adjustBoundToDateFormat(this.rightBound.get()); - - Instant currentValue = leftBound; - while (!currentValue.isAfter(rightBound)) { - String currentValueFormatted = formatValue(currentValue); - if (isValueInDomain(partitionValueFilter, currentValue, currentValueFormatted)) { - builder.add(currentValueFormatted); - } - currentValue = currentValue.atZone(UTC_TIME_ZONE_ID) - .plus(interval, intervalUnit) - .toInstant(); - } - - return builder.build(); - } - - private Instant adjustBoundToDateFormat(Instant value) - { - String formatted = formatValue(value.with(ChronoField.MILLI_OF_SECOND, 0)); - try { - return dateFormat.parse(formatted).toInstant(); - } - catch (ParseException e) { - throw invalidProjectionException(formatted, e.getMessage()); - } - } - - private String formatValue(Instant current) - { - return dateFormat.format(new Date(current.toEpochMilli())); - } - - private boolean isValueInDomain(Optional valueDomain, Instant value, String formattedValue) - { - if (valueDomain.isEmpty() || valueDomain.get().isAll()) { - return true; - } - Domain domain = valueDomain.get(); - Type type = domain.getType(); - if (type instanceof VarcharType) { - return domain.contains(singleValue(type, utf8Slice(formattedValue))); - } - if (type instanceof DateType) { - return domain.contains(singleValue(type, MILLISECONDS.toDays(value.toEpochMilli()))); - } - if (type instanceof TimestampType && ((TimestampType) type).isShort()) { - return domain.contains(singleValue(type, MILLISECONDS.toMicros(value.toEpochMilli()))); - } - throw unsupportedProjectionColumnTypeException(type); - } -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/EnumProjectionFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/EnumProjectionFactory.java deleted file mode 100644 index 13e897f78204..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/EnumProjectionFactory.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.aws.athena.projection; - -import io.trino.spi.type.Type; -import io.trino.spi.type.VarcharType; - -import java.util.List; -import java.util.Map; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_VALUES; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.getProjectionPropertyRequiredValue; - -public class EnumProjectionFactory - implements ProjectionFactory -{ - @Override - public boolean isSupportedColumnType(Type columnType) - { - return columnType instanceof VarcharType; - } - - @Override - public Projection create(String columnName, Type columnType, Map columnProperties) - { - return new EnumProjection( - columnName, - getProjectionPropertyRequiredValue( - columnName, - columnProperties, - COLUMN_PROJECTION_VALUES, - value -> ((List) value).stream() - .map(String::valueOf) - .collect(toImmutableList()))); - } -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/InjectedProjectionFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/InjectedProjectionFactory.java deleted file mode 100644 index b90d0acc6ddb..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/InjectedProjectionFactory.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.aws.athena.projection; - -import io.trino.spi.type.Type; - -import java.util.Map; - -import static io.trino.plugin.hive.metastore.MetastoreUtil.canConvertSqlTypeToStringForParts; - -public class InjectedProjectionFactory - implements ProjectionFactory -{ - @Override - public boolean isSupportedColumnType(Type columnType) - { - return canConvertSqlTypeToStringForParts(columnType, true); - } - - @Override - public Projection create(String columnName, Type columnType, Map columnProperties) - { - return new InjectedProjection(columnName); - } -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/IntegerProjectionFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/IntegerProjectionFactory.java deleted file mode 100644 index 7117221c6fea..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/IntegerProjectionFactory.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.aws.athena.projection; - -import io.trino.spi.type.BigintType; -import io.trino.spi.type.IntegerType; -import io.trino.spi.type.Type; -import io.trino.spi.type.VarcharType; - -import java.util.List; -import java.util.Map; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_DIGITS; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_INTERVAL; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_RANGE; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.getProjectionPropertyRequiredValue; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.getProjectionPropertyValue; -import static io.trino.plugin.hive.aws.athena.projection.Projection.invalidProjectionException; -import static java.lang.String.format; - -public class IntegerProjectionFactory - implements ProjectionFactory -{ - @Override - public boolean isSupportedColumnType(Type columnType) - { - return columnType instanceof VarcharType - || columnType instanceof IntegerType - || columnType instanceof BigintType; - } - - @Override - public Projection create(String columnName, Type columnType, Map columnProperties) - { - List range = getProjectionPropertyRequiredValue( - columnName, - columnProperties, - COLUMN_PROJECTION_RANGE, - value -> ((List) value).stream() - .map(element -> Integer.valueOf((String) element)) - .collect(toImmutableList())); - if (range.size() != 2) { - invalidProjectionException( - columnName, - format("Property: '%s' needs to be list of 2 integers", COLUMN_PROJECTION_RANGE)); - } - return new IntegerProjection( - columnName, - range.get(0), - range.get(1), - getProjectionPropertyValue(columnProperties, COLUMN_PROJECTION_INTERVAL, Integer.class::cast).orElse(1), - getProjectionPropertyValue(columnProperties, COLUMN_PROJECTION_DIGITS, Integer.class::cast)); - } -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/Projection.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/Projection.java deleted file mode 100644 index 40066baf357f..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/Projection.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.aws.athena.projection; - -import io.trino.spi.TrinoException; -import io.trino.spi.predicate.Domain; -import io.trino.spi.type.Type; - -import java.util.List; -import java.util.Optional; - -import static io.trino.spi.StandardErrorCode.INVALID_COLUMN_PROPERTY; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; - -public abstract class Projection -{ - private final String columnName; - - public Projection(String columnName) - { - this.columnName = requireNonNull(columnName, "columnName is null"); - } - - public String getColumnName() - { - return columnName; - } - - public abstract List getProjectedValues(Optional partitionValueFilter); - - protected TrinoException unsupportedProjectionColumnTypeException(Type columnType) - { - return unsupportedProjectionColumnTypeException(columnName, columnType); - } - - public static TrinoException unsupportedProjectionColumnTypeException(String columnName, Type columnType) - { - return invalidProjectionException(columnName, "Unsupported column type: " + columnType.getDisplayName()); - } - - public static TrinoException invalidProjectionException(String columnName, String message) - { - throw new TrinoException(INVALID_COLUMN_PROPERTY, invalidProjectionMessage(columnName, message)); - } - - public static String invalidProjectionMessage(String columnName, String message) - { - return format("Column projection for column '%s' failed. %s", columnName, message); - } -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreDecorator.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreDecorator.java index 50d0dd005e32..db7cd2305a9e 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreDecorator.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreDecorator.java @@ -16,7 +16,6 @@ public interface HiveMetastoreDecorator { - int PRIORITY_PARTITION_PROJECTION = 50; int PRIORITY_TRACING = 100; int getPriority(); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/DateProjectionFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/DateProjection.java similarity index 56% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/DateProjectionFactory.java rename to plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/DateProjection.java index 8f6914a2f6e8..18cc546b236d 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/DateProjectionFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/DateProjection.java @@ -11,20 +11,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.aws.athena.projection; +package io.trino.plugin.hive.projection; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import io.trino.spi.TrinoException; +import io.trino.spi.predicate.Domain; import io.trino.spi.type.DateType; import io.trino.spi.type.TimestampType; import io.trino.spi.type.Type; import io.trino.spi.type.VarcharType; +import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.time.Instant; import java.time.ZoneId; +import java.time.temporal.ChronoField; import java.time.temporal.ChronoUnit; +import java.util.Date; import java.util.List; import java.util.Locale; import java.util.Map; @@ -35,13 +40,14 @@ import java.util.regex.Pattern; import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_FORMAT; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_INTERVAL; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_INTERVAL_UNIT; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.COLUMN_PROJECTION_RANGE; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.getProjectionPropertyRequiredValue; -import static io.trino.plugin.hive.aws.athena.PartitionProjectionProperties.getProjectionPropertyValue; -import static io.trino.plugin.hive.aws.athena.projection.Projection.invalidProjectionException; +import static io.airlift.slice.Slices.utf8Slice; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.COLUMN_PROJECTION_FORMAT; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.COLUMN_PROJECTION_INTERVAL; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.COLUMN_PROJECTION_INTERVAL_UNIT; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.COLUMN_PROJECTION_RANGE; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.getProjectionPropertyRequiredValue; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.getProjectionPropertyValue; +import static io.trino.spi.predicate.Domain.singleValue; import static java.lang.String.format; import static java.time.temporal.ChronoUnit.DAYS; import static java.time.temporal.ChronoUnit.HOURS; @@ -49,31 +55,38 @@ import static java.time.temporal.ChronoUnit.MONTHS; import static java.time.temporal.ChronoUnit.SECONDS; import static java.util.Objects.nonNull; +import static java.util.Objects.requireNonNull; import static java.util.TimeZone.getTimeZone; +import static java.util.concurrent.TimeUnit.MILLISECONDS; -public class DateProjectionFactory - implements ProjectionFactory +final class DateProjection + implements Projection { - public static final ZoneId UTC_TIME_ZONE_ID = ZoneId.of("UTC"); - + private static final ZoneId UTC_TIME_ZONE_ID = ZoneId.of("UTC"); // Limited to only DAYS, HOURS, MINUTES, SECONDS as we are not fully sure how everything above day // is implemented in Athena. So we limit it to a subset of interval units which are explicitly clear how to calculate. - // Rest will be implemented if this will be required as it would require making compatibility tests + // The rest will be implemented if this is required as it would require making compatibility tests // for results received from Athena and verifying if we receive identical with Trino. private static final Set DATE_PROJECTION_INTERVAL_UNITS = ImmutableSet.of(DAYS, HOURS, MINUTES, SECONDS); private static final Pattern DATE_RANGE_BOUND_EXPRESSION_PATTERN = Pattern.compile("^\\s*NOW\\s*(([+-])\\s*([0-9]+)\\s*(DAY|HOUR|MINUTE|SECOND)S?\\s*)?$"); - @Override - public boolean isSupportedColumnType(Type columnType) - { - return columnType instanceof VarcharType - || columnType instanceof DateType - || (columnType instanceof TimestampType && ((TimestampType) columnType).isShort()); - } + private final String columnName; + private final DateFormat dateFormat; + private final Supplier leftBound; + private final Supplier rightBound; + private final int interval; + private final ChronoUnit intervalUnit; - @Override - public Projection create(String columnName, Type columnType, Map columnProperties) + public DateProjection(String columnName, Type columnType, Map columnProperties) { + if (!(columnType instanceof VarcharType) && + !(columnType instanceof DateType) && + !(columnType instanceof TimestampType timestampType && timestampType.isShort())) { + throw new InvalidProjectionException(columnName, columnType); + } + + this.columnName = requireNonNull(columnName, "columnName is null"); + String dateFormatPattern = getProjectionPropertyRequiredValue( columnName, columnProperties, @@ -88,24 +101,26 @@ public Projection create(String columnName, Type columnType, Map .map(String.class::cast) .collect(toImmutableList())); if (range.size() != 2) { - throw invalidRangeProperty(columnName, dateFormatPattern); + throw invalidRangeProperty(columnName, dateFormatPattern, Optional.empty()); } SimpleDateFormat dateFormat = new SimpleDateFormat(dateFormatPattern); dateFormat.setLenient(false); dateFormat.setTimeZone(getTimeZone(UTC_TIME_ZONE_ID)); - Supplier leftBound = parseDateRangerBound(columnName, range.get(0), dateFormat); - Supplier rangeBound = parseDateRangerBound(columnName, range.get(1), dateFormat); - if (!leftBound.get().isBefore(rangeBound.get())) { - throw invalidRangeProperty(columnName, dateFormatPattern); + this.dateFormat = requireNonNull(dateFormat, "dateFormatPattern is null"); + + leftBound = parseDateRangerBound(columnName, range.get(0), dateFormat); + rightBound = parseDateRangerBound(columnName, range.get(1), dateFormat); + if (!leftBound.get().isBefore(rightBound.get())) { + throw invalidRangeProperty(columnName, dateFormatPattern, Optional.empty()); } - int interval = getProjectionPropertyValue(columnProperties, COLUMN_PROJECTION_INTERVAL, Integer.class::cast).orElse(1); - ChronoUnit intervalUnit = getProjectionPropertyValue(columnProperties, COLUMN_PROJECTION_INTERVAL_UNIT, ChronoUnit.class::cast) + interval = getProjectionPropertyValue(columnProperties, COLUMN_PROJECTION_INTERVAL, Integer.class::cast).orElse(1); + intervalUnit = getProjectionPropertyValue(columnProperties, COLUMN_PROJECTION_INTERVAL_UNIT, ChronoUnit.class::cast) .orElseGet(() -> resolveDefaultChronoUnit(columnName, dateFormatPattern)); if (!DATE_PROJECTION_INTERVAL_UNITS.contains(intervalUnit)) { - throw invalidProjectionException( + throw new InvalidProjectionException( columnName, format( "Property: '%s' value '%s' is invalid. Available options: %s", @@ -113,17 +128,72 @@ public Projection create(String columnName, Type columnType, Map intervalUnit, DATE_PROJECTION_INTERVAL_UNITS)); } + } - return new DateProjection(columnName, dateFormat, leftBound, rangeBound, interval, intervalUnit); + @Override + public List getProjectedValues(Optional partitionValueFilter) + { + ImmutableList.Builder builder = ImmutableList.builder(); + + Instant leftBound = adjustBoundToDateFormat(this.leftBound.get()); + Instant rightBound = adjustBoundToDateFormat(this.rightBound.get()); + + Instant currentValue = leftBound; + while (!currentValue.isAfter(rightBound)) { + String currentValueFormatted = formatValue(currentValue); + if (isValueInDomain(partitionValueFilter, currentValue, currentValueFormatted)) { + builder.add(currentValueFormatted); + } + currentValue = currentValue.atZone(UTC_TIME_ZONE_ID) + .plus(interval, intervalUnit) + .toInstant(); + } + + return builder.build(); } - private ChronoUnit resolveDefaultChronoUnit(String columnName, String dateFormatPattern) + private Instant adjustBoundToDateFormat(Instant value) + { + String formatted = formatValue(value.with(ChronoField.MILLI_OF_SECOND, 0)); + try { + return dateFormat.parse(formatted).toInstant(); + } + catch (ParseException e) { + throw new InvalidProjectionException(formatted, e.getMessage()); + } + } + + private String formatValue(Instant current) + { + return dateFormat.format(new Date(current.toEpochMilli())); + } + + private boolean isValueInDomain(Optional valueDomain, Instant value, String formattedValue) + { + if (valueDomain.isEmpty() || valueDomain.get().isAll()) { + return true; + } + Domain domain = valueDomain.get(); + Type type = domain.getType(); + if (type instanceof VarcharType) { + return domain.contains(singleValue(type, utf8Slice(formattedValue))); + } + if (type instanceof DateType) { + return domain.contains(singleValue(type, MILLISECONDS.toDays(value.toEpochMilli()))); + } + if (type instanceof TimestampType && ((TimestampType) type).isShort()) { + return domain.contains(singleValue(type, MILLISECONDS.toMicros(value.toEpochMilli()))); + } + throw new InvalidProjectionException(columnName, type); + } + + private static ChronoUnit resolveDefaultChronoUnit(String columnName, String dateFormatPattern) { String datePatternWithoutText = dateFormatPattern.replaceAll("'.*?'", ""); if (datePatternWithoutText.contains("S") || datePatternWithoutText.contains("s") || datePatternWithoutText.contains("m") || datePatternWithoutText.contains("H")) { // When the provided dates are at single-day or single-month precision. - throw invalidProjectionException( + throw new InvalidProjectionException( columnName, format( "Property: '%s' needs to be set when provided '%s' is less that single-day precision. Interval defaults to 1 day or 1 month, respectively. Otherwise, interval is required", @@ -136,7 +206,7 @@ private ChronoUnit resolveDefaultChronoUnit(String columnName, String dateFormat return MONTHS; } - private Supplier parseDateRangerBound(String columnName, String value, SimpleDateFormat dateFormat) + private static Supplier parseDateRangerBound(String columnName, String value, SimpleDateFormat dateFormat) { Matcher matcher = DATE_RANGE_BOUND_EXPRESSION_PATTERN.matcher(value); if (matcher.matches()) { @@ -167,37 +237,21 @@ private Supplier parseDateRangerBound(String columnName, String value, return () -> dateBound; } - private TrinoException invalidRangeProperty(String columnName, String dateFormatPattern) - { - return invalidRangeProperty(columnName, dateFormatPattern, Optional.empty()); - } - - private TrinoException invalidRangeProperty(String columnName, String dateFormatPattern, Optional errorDetail) + private static TrinoException invalidRangeProperty(String columnName, String dateFormatPattern, Optional errorDetail) { - return invalidProjectionException( + throw new InvalidProjectionException( columnName, format( "Property: '%s' needs to be a list of 2 valid dates formatted as '%s' or '%s' that are sequential%s", COLUMN_PROJECTION_RANGE, dateFormatPattern, DATE_RANGE_BOUND_EXPRESSION_PATTERN.pattern(), - errorDetail.map(error -> ". " + error).orElse(""))); + errorDetail.map(error -> ": " + error).orElse(""))); } - private static class DateExpressionBound + private record DateExpressionBound(int multiplier, ChronoUnit unit, boolean increment) implements Supplier { - private final int multiplier; - private final ChronoUnit unit; - private final boolean increment; - - public DateExpressionBound(int multiplier, ChronoUnit unit, boolean increment) - { - this.multiplier = multiplier; - this.unit = unit; - this.increment = increment; - } - @Override public Instant get() { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/EnumProjection.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/EnumProjection.java similarity index 58% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/EnumProjection.java rename to plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/EnumProjection.java index 50eb7d1a63fd..45af272920a8 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/EnumProjection.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/EnumProjection.java @@ -11,30 +11,44 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.aws.athena.projection; +package io.trino.plugin.hive.projection; -import com.google.common.collect.ImmutableList; import io.trino.spi.predicate.Domain; import io.trino.spi.type.Type; import io.trino.spi.type.VarcharType; import java.util.List; +import java.util.Map; import java.util.Optional; +import static com.google.common.collect.ImmutableList.toImmutableList; import static io.airlift.slice.Slices.utf8Slice; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.COLUMN_PROJECTION_VALUES; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.getProjectionPropertyRequiredValue; import static io.trino.spi.predicate.Domain.singleValue; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toList; -public class EnumProjection - extends Projection +final class EnumProjection + implements Projection { + private final String columnName; private final List values; - public EnumProjection(String columnName, List values) + public EnumProjection(String columnName, Type columnType, Map columnProperties) { - super(columnName); - this.values = ImmutableList.copyOf(requireNonNull(values, "values is null")); + if (!(columnType instanceof VarcharType)) { + throw new InvalidProjectionException(columnName, columnType); + } + + this.columnName = requireNonNull(columnName, "columnName is null"); + this.values = getProjectionPropertyRequiredValue( + columnName, + columnProperties, + COLUMN_PROJECTION_VALUES, + value -> ((List) value).stream() + .map(String::valueOf) + .collect(toImmutableList())); } @Override @@ -54,6 +68,6 @@ private boolean isValueInDomain(Domain valueDomain, String value) if (type instanceof VarcharType) { return valueDomain.contains(singleValue(type, utf8Slice(value))); } - throw unsupportedProjectionColumnTypeException(type); + throw new InvalidProjectionException(columnName, type); } } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/InjectedProjection.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/InjectedProjection.java similarity index 59% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/InjectedProjection.java rename to plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/InjectedProjection.java index 3c0d3c9ae423..acead53bf047 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/InjectedProjection.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/InjectedProjection.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.aws.athena.projection; +package io.trino.plugin.hive.projection; import com.google.common.collect.ImmutableList; import io.trino.spi.predicate.Domain; @@ -22,26 +22,32 @@ import static io.trino.plugin.hive.metastore.MetastoreUtil.canConvertSqlTypeToStringForParts; import static io.trino.plugin.hive.metastore.MetastoreUtil.sqlScalarToString; +import static java.util.Objects.requireNonNull; -public class InjectedProjection - extends Projection +final class InjectedProjection + implements Projection { - public InjectedProjection(String columnName) + private final String columnName; + + public InjectedProjection(String columnName, Type columnType) { - super(columnName); + if (!canConvertSqlTypeToStringForParts(columnType, true)) { + throw new InvalidProjectionException(columnName, columnType); + } + this.columnName = requireNonNull(columnName, "columnName is null"); } @Override public List getProjectedValues(Optional partitionValueFilter) { Domain domain = partitionValueFilter - .orElseThrow(() -> invalidProjectionException(getColumnName(), "Injected projection requires single predicate for it's column in where clause")); + .orElseThrow(() -> new InvalidProjectionException(columnName, "Injected projection requires single predicate for it's column in where clause")); Type type = domain.getType(); if (!domain.isNullableSingleValue() || !canConvertSqlTypeToStringForParts(type, true)) { - throw invalidProjectionException(getColumnName(), "Injected projection requires single predicate for it's column in where clause. Currently provided can't be converted to single partition."); + throw new InvalidProjectionException(columnName, "Injected projection requires single predicate for it's column in where clause. Currently provided can't be converted to single partition."); } return Optional.ofNullable(sqlScalarToString(type, domain.getNullableSingleValue(), null)) .map(ImmutableList::of) - .orElseThrow(() -> unsupportedProjectionColumnTypeException(type)); + .orElseThrow(() -> new InvalidProjectionException(columnName, type)); } } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/IntegerProjection.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/IntegerProjection.java similarity index 51% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/IntegerProjection.java rename to plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/IntegerProjection.java index 645b94ca6908..2b07e78bbf15 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/IntegerProjection.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/IntegerProjection.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.aws.athena.projection; +package io.trino.plugin.hive.projection; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; @@ -22,27 +22,52 @@ import io.trino.spi.type.VarcharType; import java.util.List; +import java.util.Map; import java.util.Optional; +import static com.google.common.collect.ImmutableList.toImmutableList; import static io.airlift.slice.Slices.utf8Slice; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.COLUMN_PROJECTION_DIGITS; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.COLUMN_PROJECTION_INTERVAL; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.COLUMN_PROJECTION_RANGE; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.getProjectionPropertyRequiredValue; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.getProjectionPropertyValue; import static io.trino.spi.predicate.Domain.singleValue; +import static java.lang.String.format; import static java.util.Objects.requireNonNull; -public class IntegerProjection - extends Projection +final class IntegerProjection + implements Projection { + private final String columnName; private final int leftBound; private final int rightBound; private final int interval; private final Optional digits; - public IntegerProjection(String columnName, int leftBound, int rightBound, int interval, Optional digits) + public IntegerProjection(String columnName, Type columnType, Map columnProperties) { - super(columnName); - this.leftBound = leftBound; - this.rightBound = rightBound; - this.interval = interval; - this.digits = requireNonNull(digits, "digits is null"); + if (!(columnType instanceof VarcharType) && !(columnType instanceof IntegerType) && !(columnType instanceof BigintType)) { + throw new InvalidProjectionException(columnName, columnType); + } + + this.columnName = requireNonNull(columnName, "columnName is null"); + + List range = getProjectionPropertyRequiredValue( + columnName, + columnProperties, + COLUMN_PROJECTION_RANGE, + value -> ((List) value).stream() + .map(element -> Integer.valueOf((String) element)) + .collect(toImmutableList())); + if (range.size() != 2) { + throw new InvalidProjectionException(columnName, format("Property: '%s' needs to be list of 2 integers", COLUMN_PROJECTION_RANGE)); + } + this.leftBound = range.get(0); + this.rightBound = range.get(1); + + this.interval = getProjectionPropertyValue(columnProperties, COLUMN_PROJECTION_INTERVAL, Integer.class::cast).orElse(1); + this.digits = getProjectionPropertyValue(columnProperties, COLUMN_PROJECTION_DIGITS, Integer.class::cast); } @Override @@ -53,7 +78,7 @@ public List getProjectedValues(Optional partitionValueFilter) while (current <= rightBound) { int currentValue = current; String currentValueFormatted = digits - .map(digits -> String.format("%0" + digits + "d", currentValue)) + .map(digits -> format("%0" + digits + "d", currentValue)) .orElseGet(() -> Integer.toString(currentValue)); if (isValueInDomain(partitionValueFilter, current, currentValueFormatted)) { builder.add(currentValueFormatted); @@ -74,8 +99,8 @@ private boolean isValueInDomain(Optional valueDomain, int value, String return domain.contains(singleValue(type, utf8Slice(formattedValue))); } if (type instanceof IntegerType || type instanceof BigintType) { - return domain.contains(singleValue(type, Long.valueOf(value))); + return domain.contains(singleValue(type, (long) value)); } - throw unsupportedProjectionColumnTypeException(type); + throw new InvalidProjectionException(columnName, type); } } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/InvalidProjectionException.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/InvalidProjectionException.java new file mode 100644 index 000000000000..938152015700 --- /dev/null +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/InvalidProjectionException.java @@ -0,0 +1,44 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive.projection; + +import io.trino.spi.TrinoException; +import io.trino.spi.type.Type; + +import static io.trino.spi.StandardErrorCode.INVALID_COLUMN_PROPERTY; +import static java.lang.String.format; + +public class InvalidProjectionException + extends TrinoException +{ + public InvalidProjectionException(String columnName, Type columnType) + { + this(columnName, "Unsupported column type: " + columnType.getDisplayName()); + } + + public InvalidProjectionException(String columnName, String message) + { + this(invalidProjectionMessage(columnName, message)); + } + + public InvalidProjectionException(String message) + { + super(INVALID_COLUMN_PROPERTY, message); + } + + public static String invalidProjectionMessage(String columnName, String message) + { + return format("Column projection for column '%s' failed. %s", columnName, message); + } +} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/PartitionProjection.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/PartitionProjection.java similarity index 89% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/PartitionProjection.java rename to plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/PartitionProjection.java index e20d8ed7f244..f3214e4ddf01 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/PartitionProjection.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/PartitionProjection.java @@ -11,11 +11,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.aws.athena; +package io.trino.plugin.hive.projection; import com.google.common.base.VerifyException; import com.google.common.collect.ImmutableMap; -import io.trino.plugin.hive.aws.athena.projection.Projection; +import io.trino.plugin.hive.metastore.Column; import io.trino.plugin.hive.metastore.Partition; import io.trino.plugin.hive.metastore.Table; import io.trino.spi.predicate.Domain; @@ -33,7 +33,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.collect.Sets.cartesianProduct; -import static io.trino.plugin.hive.aws.athena.projection.Projection.invalidProjectionMessage; +import static io.trino.plugin.hive.projection.InvalidProjectionException.invalidProjectionMessage; import static io.trino.plugin.hive.util.HiveUtil.escapePathName; import static io.trino.plugin.hive.util.HiveUtil.toPartitionValues; import static java.lang.String.format; @@ -43,23 +43,15 @@ public final class PartitionProjection { private static final Pattern PROJECTION_LOCATION_TEMPLATE_PLACEHOLDER_PATTERN = Pattern.compile("(\\$\\{[^}]+\\})"); - private final boolean enabled; - private final Optional storageLocationTemplate; private final Map columnProjections; - public PartitionProjection(boolean enabled, Optional storageLocationTemplate, Map columnProjections) + public PartitionProjection(Optional storageLocationTemplate, Map columnProjections) { - this.enabled = enabled; this.storageLocationTemplate = requireNonNull(storageLocationTemplate, "storageLocationTemplate is null"); this.columnProjections = ImmutableMap.copyOf(requireNonNull(columnProjections, "columnProjections is null")); } - public boolean isEnabled() - { - return enabled; - } - public Optional> getProjectedPartitionNamesByFilter(List columnNames, TupleDomain partitionKeysFilter) { if (partitionKeysFilter.isNone()) { @@ -108,7 +100,7 @@ private Partition buildPartitionObject(Table table, String partitionName) .map(template -> expandStorageLocationTemplate( template, table.getPartitionColumns().stream() - .map(column -> column.getName()).collect(Collectors.toList()), + .map(Column::getName).collect(Collectors.toList()), partitionValues)) .orElseGet(() -> format("%s/%s/", table.getStorage().getLocation(), partitionName))) .setBucketProperty(table.getStorage().getBucketProperty()) @@ -116,7 +108,7 @@ private Partition buildPartitionObject(Table table, String partitionName) .build(); } - private String expandStorageLocationTemplate(String template, List partitionColumns, List partitionValues) + private static String expandStorageLocationTemplate(String template, List partitionColumns, List partitionValues) { Matcher matcher = PROJECTION_LOCATION_TEMPLATE_PLACEHOLDER_PATTERN.matcher(template); StringBuilder location = new StringBuilder(); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/PartitionProjectionProperties.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/PartitionProjectionProperties.java new file mode 100644 index 000000000000..1cd046ea8647 --- /dev/null +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/PartitionProjectionProperties.java @@ -0,0 +1,336 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive.projection; + +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import io.trino.plugin.hive.metastore.Column; +import io.trino.plugin.hive.metastore.Table; +import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.connector.ConnectorTableMetadata; +import io.trino.spi.type.Type; +import io.trino.spi.type.TypeManager; + +import java.time.temporal.ChronoUnit; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static io.trino.plugin.hive.HiveTableProperties.getPartitionedBy; +import static io.trino.plugin.hive.HiveTimestampPrecision.DEFAULT_PRECISION; +import static java.lang.Boolean.parseBoolean; +import static java.lang.String.format; +import static java.util.Locale.ROOT; + +public final class PartitionProjectionProperties +{ + private static final String COLUMN_PROJECTION_TYPE_SUFFIX = "type"; + private static final String COLUMN_PROJECTION_VALUES_SUFFIX = "values"; + private static final String COLUMN_PROJECTION_RANGE_SUFFIX = "range"; + private static final String COLUMN_PROJECTION_INTERVAL_SUFFIX = "interval"; + private static final String COLUMN_PROJECTION_DIGITS_SUFFIX = "digits"; + private static final String COLUMN_PROJECTION_FORMAT_SUFFIX = "format"; + private static final String METASTORE_PROPERTY_PROJECTION_INTERVAL_UNIT_SUFFIX = "interval.unit"; + private static final String METASTORE_PROPERTY_PROJECTION_ENABLED = "projection.enabled"; + private static final String METASTORE_PROPERTY_PROJECTION_LOCATION_TEMPLATE = "storage.location.template"; + private static final String METASTORE_PROPERTY_PROJECTION_IGNORE = "trino.partition_projection.ignore"; + private static final String PROPERTY_KEY_PREFIX = "partition_projection_"; + + public static final String COLUMN_PROJECTION_FORMAT = PROPERTY_KEY_PREFIX + COLUMN_PROJECTION_FORMAT_SUFFIX; + public static final String COLUMN_PROJECTION_DIGITS = PROPERTY_KEY_PREFIX + COLUMN_PROJECTION_DIGITS_SUFFIX; + public static final String COLUMN_PROJECTION_INTERVAL_UNIT = PROPERTY_KEY_PREFIX + "interval_unit"; + public static final String COLUMN_PROJECTION_INTERVAL = PROPERTY_KEY_PREFIX + COLUMN_PROJECTION_INTERVAL_SUFFIX; + public static final String COLUMN_PROJECTION_RANGE = PROPERTY_KEY_PREFIX + COLUMN_PROJECTION_RANGE_SUFFIX; + public static final String COLUMN_PROJECTION_VALUES = PROPERTY_KEY_PREFIX + COLUMN_PROJECTION_VALUES_SUFFIX; + public static final String COLUMN_PROJECTION_TYPE = PROPERTY_KEY_PREFIX + COLUMN_PROJECTION_TYPE_SUFFIX; + public static final String PARTITION_PROJECTION_IGNORE = PROPERTY_KEY_PREFIX + "ignore"; + public static final String PARTITION_PROJECTION_LOCATION_TEMPLATE = PROPERTY_KEY_PREFIX + "location_template"; + public static final String PARTITION_PROJECTION_ENABLED = PROPERTY_KEY_PREFIX + "enabled"; + + private PartitionProjectionProperties() {} + + public static Map getPartitionProjectionTrinoTableProperties(Table table) + { + Map metastoreTableProperties = table.getParameters(); + ImmutableMap.Builder trinoTablePropertiesBuilder = ImmutableMap.builder(); + + String ignore = metastoreTableProperties.get(METASTORE_PROPERTY_PROJECTION_IGNORE); + if (ignore != null) { + trinoTablePropertiesBuilder.put(PARTITION_PROJECTION_IGNORE, Boolean.valueOf(ignore)); + } + + String enabled = metastoreTableProperties.get(METASTORE_PROPERTY_PROJECTION_ENABLED); + if (enabled != null) { + trinoTablePropertiesBuilder.put(PARTITION_PROJECTION_ENABLED, Boolean.valueOf(enabled)); + } + + String locationTemplate = metastoreTableProperties.get(METASTORE_PROPERTY_PROJECTION_LOCATION_TEMPLATE); + if (locationTemplate != null) { + trinoTablePropertiesBuilder.put(PARTITION_PROJECTION_LOCATION_TEMPLATE, locationTemplate); + } + + return trinoTablePropertiesBuilder.buildOrThrow(); + } + + public static Map getPartitionProjectionTrinoColumnProperties(Table table, String columnName) + { + Map metastoreTableProperties = table.getParameters(); + return rewriteColumnProjectionProperties(metastoreTableProperties, columnName); + } + + public static Map getPartitionProjectionHiveTableProperties(ConnectorTableMetadata tableMetadata) + { + ImmutableMap.Builder metastoreTablePropertiesBuilder = ImmutableMap.builder(); + // Handle Table Properties + Map trinoTableProperties = tableMetadata.getProperties(); + + Object ignore = trinoTableProperties.get(PARTITION_PROJECTION_IGNORE); + if (ignore != null) { + metastoreTablePropertiesBuilder.put(METASTORE_PROPERTY_PROJECTION_IGNORE, ignore.toString().toLowerCase(ROOT)); + } + + Object enabled = trinoTableProperties.get(PARTITION_PROJECTION_ENABLED); + if (enabled != null) { + metastoreTablePropertiesBuilder.put(METASTORE_PROPERTY_PROJECTION_ENABLED, enabled.toString().toLowerCase(ROOT)); + } + + Object locationTemplate = trinoTableProperties.get(PARTITION_PROJECTION_LOCATION_TEMPLATE); + if (locationTemplate != null) { + metastoreTablePropertiesBuilder.put(METASTORE_PROPERTY_PROJECTION_LOCATION_TEMPLATE, locationTemplate.toString()); + } + + // Handle Column Properties + tableMetadata.getColumns().stream() + .filter(columnMetadata -> !columnMetadata.getProperties().isEmpty()) + .forEach(columnMetadata -> { + Map columnProperties = columnMetadata.getProperties(); + String columnName = columnMetadata.getName(); + + if (columnProperties.get(COLUMN_PROJECTION_TYPE) instanceof ProjectionType projectionType) { + metastoreTablePropertiesBuilder.put(getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_TYPE_SUFFIX), projectionType.name().toLowerCase(ROOT)); + } + + if (columnProperties.get(COLUMN_PROJECTION_VALUES) instanceof List values) { + metastoreTablePropertiesBuilder.put(getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_VALUES_SUFFIX), Joiner.on(",").join(values)); + } + + if (columnProperties.get(COLUMN_PROJECTION_RANGE) instanceof List range) { + metastoreTablePropertiesBuilder.put(getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_RANGE_SUFFIX), Joiner.on(",").join(range)); + } + + if (columnProperties.get(COLUMN_PROJECTION_INTERVAL) instanceof Integer interval) { + metastoreTablePropertiesBuilder.put(getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_INTERVAL_SUFFIX), interval.toString()); + } + + if (columnProperties.get(COLUMN_PROJECTION_INTERVAL_UNIT) instanceof ChronoUnit intervalUnit) { + metastoreTablePropertiesBuilder.put(getMetastoreProjectionPropertyKey(columnName, METASTORE_PROPERTY_PROJECTION_INTERVAL_UNIT_SUFFIX), intervalUnit.name().toLowerCase(ROOT)); + } + + if (columnProperties.get(COLUMN_PROJECTION_DIGITS) instanceof Integer digits) { + metastoreTablePropertiesBuilder.put(getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_DIGITS_SUFFIX), digits.toString()); + } + + if (columnProperties.get(COLUMN_PROJECTION_FORMAT) instanceof String format) { + metastoreTablePropertiesBuilder.put(getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_FORMAT_SUFFIX), format); + } + }); + + // We initialize partition projection to validate properties. + Map metastoreTableProperties = metastoreTablePropertiesBuilder.buildOrThrow(); + Set partitionColumnNames = ImmutableSet.copyOf(getPartitionedBy(tableMetadata.getProperties())); + createPartitionProjection( + tableMetadata.getColumns() + .stream() + .map(ColumnMetadata::getName) + .filter(name -> !partitionColumnNames.contains(name)) + .collect(toImmutableList()), + tableMetadata.getColumns().stream() + .filter(columnMetadata -> partitionColumnNames.contains(columnMetadata.getName())) + .collect(toImmutableMap(ColumnMetadata::getName, ColumnMetadata::getType)), + metastoreTableProperties); + + return metastoreTableProperties; + } + + public static boolean arePartitionProjectionPropertiesSet(ConnectorTableMetadata tableMetadata) + { + if (tableMetadata.getProperties().keySet().stream() + .anyMatch(propertyKey -> propertyKey.startsWith(PROPERTY_KEY_PREFIX))) { + return true; + } + return tableMetadata.getColumns().stream() + .map(columnMetadata -> columnMetadata.getProperties().keySet()) + .flatMap(Set::stream) + .anyMatch(propertyKey -> propertyKey.startsWith(PROPERTY_KEY_PREFIX)); + } + + public static Optional getPartitionProjectionFromTable(Table table, TypeManager typeManager) + { + Map tableProperties = table.getParameters(); + if (parseBoolean(tableProperties.get(METASTORE_PROPERTY_PROJECTION_IGNORE)) || + !parseBoolean(tableProperties.get(METASTORE_PROPERTY_PROJECTION_ENABLED))) { + return Optional.empty(); + } + + Set partitionColumnNames = table.getPartitionColumns().stream().map(Column::getName).collect(Collectors.toSet()); + return createPartitionProjection( + table.getDataColumns().stream() + .map(Column::getName) + .filter(partitionColumnNames::contains) + .collect(toImmutableList()), + table.getPartitionColumns().stream() + .collect(toImmutableMap(Column::getName, column -> column.getType().getType(typeManager, DEFAULT_PRECISION))), + tableProperties); + } + + private static Optional createPartitionProjection(List dataColumns, Map partitionColumns, Map tableProperties) + { + // This method is used during table creation to validate the properties. The validation is performed even if the projection is disabled. + boolean enabled = parseBoolean(tableProperties.get(METASTORE_PROPERTY_PROJECTION_ENABLED)); + + if (!tableProperties.containsKey(METASTORE_PROPERTY_PROJECTION_ENABLED) && + partitionColumns.keySet().stream().anyMatch(columnName -> !rewriteColumnProjectionProperties(tableProperties, columnName).isEmpty())) { + throw new InvalidProjectionException("Columns partition projection properties cannot be set when '%s' is not set".formatted(PARTITION_PROJECTION_ENABLED)); + } + + if (enabled && partitionColumns.isEmpty()) { + throw new InvalidProjectionException("Partition projection cannot be enabled on a table that is not partitioned"); + } + + for (String columnName : dataColumns) { + if (!rewriteColumnProjectionProperties(tableProperties, columnName).isEmpty()) { + throw new InvalidProjectionException("Partition projection cannot be defined for non-partition column: '" + columnName + "'"); + } + } + + Map columnProjections = new HashMap<>(); + partitionColumns.forEach((columnName, type) -> { + Map columnProperties = rewriteColumnProjectionProperties(tableProperties, columnName); + if (enabled) { + columnProjections.put(columnName, parseColumnProjection(columnName, type, columnProperties)); + } + }); + + Optional storageLocationTemplate = Optional.ofNullable(tableProperties.get(METASTORE_PROPERTY_PROJECTION_LOCATION_TEMPLATE)); + storageLocationTemplate.ifPresent(locationTemplate -> { + for (String columnName : partitionColumns.keySet()) { + if (!locationTemplate.contains("${" + columnName + "}")) { + throw new InvalidProjectionException(format("Partition projection location template: %s is missing partition column: '%s' placeholder", locationTemplate, columnName)); + } + } + }); + + if (!enabled) { + return Optional.empty(); + } + return Optional.of(new PartitionProjection(storageLocationTemplate, columnProjections)); + } + + private static Map rewriteColumnProjectionProperties(Map metastoreTableProperties, String columnName) + { + ImmutableMap.Builder trinoTablePropertiesBuilder = ImmutableMap.builder(); + + String type = metastoreTableProperties.get(getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_TYPE_SUFFIX)); + if (type != null) { + trinoTablePropertiesBuilder.put(COLUMN_PROJECTION_TYPE, ProjectionType.valueOf(type.toUpperCase(ROOT))); + } + + String values = metastoreTableProperties.get(getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_VALUES_SUFFIX)); + if (values != null) { + trinoTablePropertiesBuilder.put(COLUMN_PROJECTION_VALUES, splitCommaSeparatedString(values)); + } + + String range = metastoreTableProperties.get(getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_RANGE_SUFFIX)); + if (range != null) { + trinoTablePropertiesBuilder.put(COLUMN_PROJECTION_RANGE, splitCommaSeparatedString(range)); + } + + String interval = metastoreTableProperties.get(getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_INTERVAL_SUFFIX)); + if (interval != null) { + trinoTablePropertiesBuilder.put(COLUMN_PROJECTION_INTERVAL, Integer.valueOf(interval)); + } + + String intervalUnit = metastoreTableProperties.get(getMetastoreProjectionPropertyKey(columnName, METASTORE_PROPERTY_PROJECTION_INTERVAL_UNIT_SUFFIX)); + if (intervalUnit != null) { + trinoTablePropertiesBuilder.put(COLUMN_PROJECTION_INTERVAL_UNIT, ChronoUnit.valueOf(intervalUnit.toUpperCase(ROOT))); + } + + String digits = metastoreTableProperties.get(getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_DIGITS_SUFFIX)); + if (digits != null) { + trinoTablePropertiesBuilder.put(COLUMN_PROJECTION_DIGITS, Integer.valueOf(digits)); + } + + String format = metastoreTableProperties.get(getMetastoreProjectionPropertyKey(columnName, COLUMN_PROJECTION_FORMAT_SUFFIX)); + if (format != null) { + trinoTablePropertiesBuilder.put(COLUMN_PROJECTION_FORMAT, format); + } + + return trinoTablePropertiesBuilder.buildOrThrow(); + } + + private static Projection parseColumnProjection(String columnName, Type columnType, Map columnProperties) + { + ProjectionType projectionType = (ProjectionType) columnProperties.get(COLUMN_PROJECTION_TYPE); + if (projectionType == null) { + throw new InvalidProjectionException(columnName, "Projection type property missing"); + } + return switch (projectionType) { + case ENUM -> new EnumProjection(columnName, columnType, columnProperties); + case INTEGER -> new IntegerProjection(columnName, columnType, columnProperties); + case DATE -> new DateProjection(columnName, columnType, columnProperties); + case INJECTED -> new InjectedProjection(columnName, columnType); + }; + } + + private static List splitCommaSeparatedString(String value) + { + return Splitter.on(',') + .trimResults() + .omitEmptyStrings() + .splitToList(value); + } + + private static String getMetastoreProjectionPropertyKey(String columnName, String propertyKeySuffix) + { + return "projection" + "." + columnName + "." + propertyKeySuffix; + } + + static T getProjectionPropertyRequiredValue( + String columnName, + Map columnProjectionProperties, + String propertyKey, + Function decoder) + { + return getProjectionPropertyValue(columnProjectionProperties, propertyKey, decoder) + .orElseThrow(() -> new InvalidProjectionException(columnName, format("Missing required property: '%s'", propertyKey))); + } + + static Optional getProjectionPropertyValue( + Map columnProjectionProperties, + String propertyKey, + Function decoder) + { + return Optional.ofNullable( + columnProjectionProperties.get(propertyKey)) + .map(decoder); + } +} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/ProjectionFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/Projection.java similarity index 64% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/ProjectionFactory.java rename to plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/Projection.java index 88d640f4bc89..0677769dfaba 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/ProjectionFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/Projection.java @@ -11,15 +11,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.aws.athena.projection; +package io.trino.plugin.hive.projection; -import io.trino.spi.type.Type; +import io.trino.spi.predicate.Domain; -import java.util.Map; +import java.util.List; +import java.util.Optional; -public interface ProjectionFactory +sealed interface Projection + permits DateProjection, EnumProjection, InjectedProjection, IntegerProjection { - boolean isSupportedColumnType(Type columnType); - - Projection create(String columnName, Type columnType, Map columnProperties); + List getProjectedValues(Optional partitionValueFilter); } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/ProjectionType.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/ProjectionType.java similarity index 85% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/ProjectionType.java rename to plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/ProjectionType.java index f5eca8363cc0..7521053f87d2 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/athena/projection/ProjectionType.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/ProjectionType.java @@ -11,12 +11,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.aws.athena.projection; +package io.trino.plugin.hive.projection; public enum ProjectionType { - ENUM, - INTEGER, - DATE, - INJECTED; + ENUM, INTEGER, DATE, INJECTED } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/HiveUtil.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/HiveUtil.java index 8d370af19d55..061b08e14e03 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/HiveUtil.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/HiveUtil.java @@ -28,7 +28,6 @@ import io.trino.plugin.hive.HivePartitionKey; import io.trino.plugin.hive.HiveTimestampPrecision; import io.trino.plugin.hive.HiveType; -import io.trino.plugin.hive.aws.athena.PartitionProjectionService; import io.trino.plugin.hive.metastore.Column; import io.trino.plugin.hive.metastore.SortingColumn; import io.trino.plugin.hive.metastore.Table; @@ -98,6 +97,7 @@ import static io.trino.plugin.hive.HiveType.toHiveTypes; import static io.trino.plugin.hive.metastore.SortingColumn.Order.ASCENDING; import static io.trino.plugin.hive.metastore.SortingColumn.Order.DESCENDING; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.getPartitionProjectionTrinoColumnProperties; import static io.trino.plugin.hive.util.HiveBucketing.isSupportedBucketing; import static io.trino.plugin.hive.util.HiveClassNames.HUDI_INPUT_FORMAT; import static io.trino.plugin.hive.util.HiveClassNames.HUDI_PARQUET_INPUT_FORMAT; @@ -864,7 +864,7 @@ public static Function columnMetadataGetter(Ta .setComment(handle.isHidden() ? Optional.empty() : columnComment.get(handle.getName())) .setExtraInfo(Optional.ofNullable(columnExtraInfo(handle.isPartitionKey()))) .setHidden(handle.isHidden()) - .setProperties(PartitionProjectionService.getPartitionProjectionTrinoColumnProperties(table, handle.getName())) + .setProperties(getPartitionProjectionTrinoColumnProperties(table, handle.getName())) .build(); } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java index 14767cbde0d6..122de9327503 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java @@ -35,7 +35,6 @@ import io.trino.plugin.base.CatalogName; import io.trino.plugin.base.metrics.LongCount; import io.trino.plugin.hive.LocationService.WriteInfo; -import io.trino.plugin.hive.aws.athena.PartitionProjectionService; import io.trino.plugin.hive.fs.DirectoryLister; import io.trino.plugin.hive.fs.RemoteIterator; import io.trino.plugin.hive.fs.TransactionScopeCachingDirectoryListerFactory; @@ -126,7 +125,6 @@ import io.trino.spi.type.SqlTimestamp; import io.trino.spi.type.SqlTimestampWithTimeZone; import io.trino.spi.type.SqlVarbinary; -import io.trino.spi.type.TestingTypeManager; import io.trino.spi.type.Type; import io.trino.spi.type.TypeId; import io.trino.spi.type.TypeOperators; @@ -895,7 +893,7 @@ public Optional getMaterializedView(Connect SqlStandardAccessControlMetadata::new, countingDirectoryLister, new TransactionScopeCachingDirectoryListerFactory(hiveConfig), - new PartitionProjectionService(hiveConfig, ImmutableMap.of(), new TestingTypeManager()), + false, true, HiveTimestampPrecision.DEFAULT_PRECISION); transactionManager = new HiveTransactionManager(metadataFactory); @@ -3320,7 +3318,7 @@ public void testUpdateTableColumnStatisticsEmptyOptionalFields() protected void testUpdateTableStatistics(SchemaTableName tableName, PartitionStatistics initialStatistics, PartitionStatistics... statistics) { - HiveMetastoreClosure metastoreClient = new HiveMetastoreClosure(getMetastoreClient()); + HiveMetastoreClosure metastoreClient = new HiveMetastoreClosure(getMetastoreClient(), TESTING_TYPE_MANAGER, false); assertThat(metastoreClient.getTableStatistics(tableName.getSchemaName(), tableName.getTableName(), Optional.empty())) .isEqualTo(initialStatistics); @@ -3406,7 +3404,7 @@ public void testDataColumnProperties() throws Exception { SchemaTableName tableName = temporaryTable("test_column_properties"); - HiveMetastoreClosure metastoreClient = new HiveMetastoreClosure(getMetastoreClient()); + HiveMetastoreClosure metastoreClient = new HiveMetastoreClosure(getMetastoreClient(), TESTING_TYPE_MANAGER, false); try { doCreateEmptyTable(tableName, ORC, List.of(new ColumnMetadata("id", BIGINT), new ColumnMetadata("part_key", createVarcharType(256)))); @@ -3449,7 +3447,7 @@ public void testPartitionColumnProperties() throws Exception { SchemaTableName tableName = temporaryTable("test_column_properties"); - HiveMetastoreClosure metastoreClient = new HiveMetastoreClosure(getMetastoreClient()); + HiveMetastoreClosure metastoreClient = new HiveMetastoreClosure(getMetastoreClient(), TESTING_TYPE_MANAGER, false); try { doCreateEmptyTable(tableName, ORC, List.of(new ColumnMetadata("id", BIGINT), new ColumnMetadata("part_key", createVarcharType(256)))); @@ -3615,7 +3613,7 @@ protected void createDummyPartitionedTable(SchemaTableName tableName, List new TableNotFoundException(tableName)); @@ -3645,7 +3643,7 @@ protected void testUpdatePartitionStatistics( String firstPartitionName = "ds=2016-01-01"; String secondPartitionName = "ds=2016-01-02"; - HiveMetastoreClosure metastoreClient = new HiveMetastoreClosure(getMetastoreClient()); + HiveMetastoreClosure metastoreClient = new HiveMetastoreClosure(getMetastoreClient(), TESTING_TYPE_MANAGER, false); assertThat(metastoreClient.getPartitionStatistics(tableName.getSchemaName(), tableName.getTableName(), ImmutableSet.of(firstPartitionName, secondPartitionName))) .isEqualTo(ImmutableMap.of(firstPartitionName, initialStatistics, secondPartitionName, initialStatistics)); @@ -3702,7 +3700,7 @@ protected void testStorePartitionWithStatistics( try { doCreateEmptyTable(tableName, ORC, columns); - HiveMetastoreClosure metastoreClient = new HiveMetastoreClosure(getMetastoreClient()); + HiveMetastoreClosure metastoreClient = new HiveMetastoreClosure(getMetastoreClient(), TESTING_TYPE_MANAGER, false); Table table = metastoreClient.getTable(tableName.getSchemaName(), tableName.getTableName()) .orElseThrow(() -> new TableNotFoundException(tableName)); @@ -3803,7 +3801,7 @@ protected void testPartitionStatisticsSampling(List columns, Par try { createDummyPartitionedTable(tableName, columns); - HiveMetastoreClosure metastoreClient = new HiveMetastoreClosure(getMetastoreClient()); + HiveMetastoreClosure metastoreClient = new HiveMetastoreClosure(getMetastoreClient(), TESTING_TYPE_MANAGER, false); metastoreClient.updatePartitionStatistics(tableName.getSchemaName(), tableName.getTableName(), "ds=2016-01-01", actualStatistics -> statistics); metastoreClient.updatePartitionStatistics(tableName.getSchemaName(), tableName.getTableName(), "ds=2016-01-02", actualStatistics -> statistics); diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHiveFileSystem.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHiveFileSystem.java index 86d89f66fb67..2ead1071c9fe 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHiveFileSystem.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHiveFileSystem.java @@ -34,7 +34,6 @@ import io.trino.operator.GroupByHashPageIndexerFactory; import io.trino.plugin.base.CatalogName; import io.trino.plugin.hive.AbstractTestHive.Transaction; -import io.trino.plugin.hive.aws.athena.PartitionProjectionService; import io.trino.plugin.hive.fs.FileSystemDirectoryLister; import io.trino.plugin.hive.fs.HiveFileIterator; import io.trino.plugin.hive.fs.TransactionScopeCachingDirectoryListerFactory; @@ -70,7 +69,6 @@ import io.trino.spi.connector.TableNotFoundException; import io.trino.spi.predicate.TupleDomain; import io.trino.spi.security.ConnectorIdentity; -import io.trino.spi.type.TestingTypeManager; import io.trino.spi.type.TypeOperators; import io.trino.sql.gen.JoinCompiler; import io.trino.testing.MaterializedResult; @@ -242,7 +240,6 @@ protected void setup(String host, int port, String databaseName, HdfsConfigurati SqlStandardAccessControlMetadata::new, new FileSystemDirectoryLister(), new TransactionScopeCachingDirectoryListerFactory(config), - new PartitionProjectionService(config, ImmutableMap.of(), new TestingTypeManager()), true); transactionManager = new HiveTransactionManager(metadataFactory); splitManager = new HiveSplitManager( diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive3OnDataLake.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive3OnDataLake.java index 36b2a793b453..9b911653e92c 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive3OnDataLake.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive3OnDataLake.java @@ -1331,7 +1331,7 @@ public void testPartitionProjectionInvalidTableProperties() ") WITH ( " + " partition_projection_enabled=true " + ")")) - .hasMessage("Partition projection can't be enabled when no partition columns are defined."); + .hasMessage("Partition projection cannot be enabled on a table that is not partitioned"); assertThatThrownBy(() -> getQueryRunner().execute( "CREATE TABLE " + getFullyQualifiedTestTableName("nation_" + randomNameSuffix()) + " ( " + @@ -1344,7 +1344,7 @@ public void testPartitionProjectionInvalidTableProperties() " partitioned_by=ARRAY['short_name1'], " + " partition_projection_enabled=true " + ")")) - .hasMessage("Partition projection can't be defined for non partition column: 'name'"); + .hasMessage("Partition projection cannot be defined for non-partition column: 'name'"); assertThatThrownBy(() -> getQueryRunner().execute( "CREATE TABLE " + getFullyQualifiedTestTableName("nation_" + randomNameSuffix()) + " ( " + @@ -1358,7 +1358,7 @@ public void testPartitionProjectionInvalidTableProperties() " partitioned_by=ARRAY['short_name1', 'short_name2'], " + " partition_projection_enabled=true " + ")")) - .hasMessage("Partition projection definition for column: 'short_name2' missing"); + .hasMessage("Column projection for column 'short_name2' failed. Projection type property missing"); assertThatThrownBy(() -> getQueryRunner().execute( "CREATE TABLE " + getFullyQualifiedTestTableName("nation_" + randomNameSuffix()) + " ( " + @@ -1421,7 +1421,7 @@ public void testPartitionProjectionInvalidTableProperties() " partition_projection_enabled=true " + ")")) .hasMessage("Column projection for column 'short_name1' failed. Property: 'partition_projection_range' needs to be a list of 2 valid dates formatted as 'yyyy-MM-dd HH' " + - "or '^\\s*NOW\\s*(([+-])\\s*([0-9]+)\\s*(DAY|HOUR|MINUTE|SECOND)S?\\s*)?$' that are sequential. Unparseable date: \"2001-01-01\""); + "or '^\\s*NOW\\s*(([+-])\\s*([0-9]+)\\s*(DAY|HOUR|MINUTE|SECOND)S?\\s*)?$' that are sequential: Unparseable date: \"2001-01-01\""); assertThatThrownBy(() -> getQueryRunner().execute( "CREATE TABLE " + getFullyQualifiedTestTableName("nation_" + randomNameSuffix()) + " ( " + @@ -1436,7 +1436,7 @@ public void testPartitionProjectionInvalidTableProperties() " partition_projection_enabled=true " + ")")) .hasMessage("Column projection for column 'short_name1' failed. Property: 'partition_projection_range' needs to be a list of 2 valid dates formatted as 'yyyy-MM-dd' " + - "or '^\\s*NOW\\s*(([+-])\\s*([0-9]+)\\s*(DAY|HOUR|MINUTE|SECOND)S?\\s*)?$' that are sequential. Unparseable date: \"NOW*3DAYS\""); + "or '^\\s*NOW\\s*(([+-])\\s*([0-9]+)\\s*(DAY|HOUR|MINUTE|SECOND)S?\\s*)?$' that are sequential: Unparseable date: \"NOW*3DAYS\""); assertThatThrownBy(() -> getQueryRunner().execute( "CREATE TABLE " + getFullyQualifiedTestTableName("nation_" + randomNameSuffix()) + " ( " + @@ -1496,7 +1496,7 @@ public void testPartitionProjectionInvalidTableProperties() ") WITH ( " + " partitioned_by=ARRAY['short_name1'] " + ")")) - .hasMessage("Columns ['short_name1'] projections are disallowed when partition projection property 'partition_projection_enabled' is missing"); + .hasMessage("Columns partition projection properties cannot be set when 'partition_projection_enabled' is not set"); // Verify that ignored flag is only interpreted for pre-existing tables where configuration is loaded from metastore. // It should not allow creating corrupted config via Trino. It's a kill switch to run away when we have compatibility issues. @@ -1510,7 +1510,7 @@ public void testPartitionProjectionInvalidTableProperties() " )" + ") WITH ( " + " partitioned_by=ARRAY['short_name1'], " + - " partition_projection_enabled=false, " + + " partition_projection_enabled=true, " + " partition_projection_ignore=true " + // <-- Even if this is set we disallow creating corrupted configuration via Trino ")")) .hasMessage("Column projection for column 'short_name1' failed. Property: 'partition_projection_interval_unit' " + @@ -1542,7 +1542,7 @@ public void testPartitionProjectionIgnore() // Expect invalid Partition Projection properties to fail assertThatThrownBy(() -> getQueryRunner().execute("SELECT * FROM " + fullyQualifiedTestTableName)) .hasMessage("Column projection for column 'date_time' failed. Property: 'partition_projection_range' needs to be a list of 2 valid dates formatted as 'yyyy-MM-dd HH' " + - "or '^\\s*NOW\\s*(([+-])\\s*([0-9]+)\\s*(DAY|HOUR|MINUTE|SECOND)S?\\s*)?$' that are sequential. Unparseable date: \"2001-01-01\""); + "or '^\\s*NOW\\s*(([+-])\\s*([0-9]+)\\s*(DAY|HOUR|MINUTE|SECOND)S?\\s*)?$' that are sequential: Unparseable date: \"2001-01-01\""); // Append kill switch table property to ignore Partition Projection properties hiveMinioDataLake.getHiveHadoop().runOnHive( diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/TestSemiTransactionalHiveMetastore.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/TestSemiTransactionalHiveMetastore.java index 63f1346c9726..765619228c86 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/TestSemiTransactionalHiveMetastore.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/TestSemiTransactionalHiveMetastore.java @@ -39,6 +39,7 @@ import static io.trino.plugin.hive.acid.AcidOperation.INSERT; import static io.trino.plugin.hive.util.HiveBucketing.BucketingVersion.BUCKETING_V1; import static io.trino.testing.TestingConnectorSession.SESSION; +import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; import static java.util.concurrent.Executors.newFixedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; import static org.testng.Assert.assertTrue; @@ -80,7 +81,7 @@ private SemiTransactionalHiveMetastore getSemiTransactionalHiveMetastoreWithDrop { return new SemiTransactionalHiveMetastore( HDFS_FILE_SYSTEM_FACTORY, - new HiveMetastoreClosure(new TestingHiveMetastore()), + new HiveMetastoreClosure(new TestingHiveMetastore(), TESTING_TYPE_MANAGER, false), directExecutor(), dropExecutor, directExecutor(), @@ -121,7 +122,7 @@ private SemiTransactionalHiveMetastore getSemiTransactionalHiveMetastoreWithUpda { return new SemiTransactionalHiveMetastore( HDFS_FILE_SYSTEM_FACTORY, - new HiveMetastoreClosure(new TestingHiveMetastore()), + new HiveMetastoreClosure(new TestingHiveMetastore(), TESTING_TYPE_MANAGER, false), directExecutor(), directExecutor(), updateExecutor, diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastore.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastore.java index a4d65e13e014..117b8384dcfc 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastore.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastore.java @@ -111,6 +111,7 @@ import static io.trino.spi.predicate.TupleDomain.withColumnDomains; import static io.trino.spi.security.PrincipalType.USER; import static io.trino.spi.type.VarcharType.VARCHAR; +import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; import static java.lang.String.format; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.TimeUnit.SECONDS; @@ -845,7 +846,7 @@ public void testUpdatePartitionStatistics() { assertEquals(mockClient.getAccessCount(), 0); - HiveMetastoreClosure hiveMetastoreClosure = new HiveMetastoreClosure(metastore); + HiveMetastoreClosure hiveMetastoreClosure = new HiveMetastoreClosure(metastore, TESTING_TYPE_MANAGER, false); Table table = hiveMetastoreClosure.getTable(TEST_DATABASE, TEST_TABLE).orElseThrow(); assertEquals(mockClient.getAccessCount(), 1); diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/glue/TestHiveGlueMetastore.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/glue/TestHiveGlueMetastore.java index cfef79f5ffae..091338b1b4a3 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/glue/TestHiveGlueMetastore.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/glue/TestHiveGlueMetastore.java @@ -114,6 +114,7 @@ import static io.trino.spi.type.VarcharType.VARCHAR; import static io.trino.spi.type.VarcharType.createUnboundedVarcharType; import static io.trino.testing.TestingPageSinkId.TESTING_PAGE_SINK_ID; +import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; import static java.lang.String.format; import static java.lang.System.currentTimeMillis; import static java.util.Collections.unmodifiableList; @@ -212,7 +213,7 @@ public void initialize() // Logging logging = Logging.initialize(); // logging.setLevel("com.amazonaws.request", Level.DEBUG); - metastore = new HiveMetastoreClosure(metastoreClient); + metastore = new HiveMetastoreClosure(metastoreClient, TESTING_TYPE_MANAGER, false); glueClient = AWSGlueAsyncClientBuilder.defaultClient(); } @@ -384,7 +385,7 @@ public void testGetPartitionsWithFilterUsingReservedKeywordsAsColumnName() doCreateEmptyTable(tableName, ORC, columns, partitionedBy); - HiveMetastoreClosure metastoreClient = new HiveMetastoreClosure(getMetastoreClient()); + HiveMetastoreClosure metastoreClient = new HiveMetastoreClosure(getMetastoreClient(), TESTING_TYPE_MANAGER, false); Table table = metastoreClient.getTable(tableName.getSchemaName(), tableName.getTableName()) .orElseThrow(() -> new TableNotFoundException(tableName)); @@ -1542,7 +1543,7 @@ private void createDummyPartitionedTable(SchemaTableName tableName, List new TableNotFoundException(tableName)); List partitions = new ArrayList<>(); diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreAccessOperations.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreAccessOperations.java index 50e5d054a617..35dce20c9f3e 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreAccessOperations.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreAccessOperations.java @@ -120,7 +120,7 @@ public void testSelectPartitionedTable() assertMetastoreInvocations("SELECT * FROM test_select_partition", ImmutableMultiset.builder() - .addCopies(GET_TABLE, 2) + .add(GET_TABLE) .add(GET_PARTITION_NAMES_BY_FILTER) .add(GET_PARTITIONS_BY_NAMES) .build()); @@ -128,7 +128,7 @@ public void testSelectPartitionedTable() assertUpdate("INSERT INTO test_select_partition SELECT 2 AS data, 20 AS part", 1); assertMetastoreInvocations("SELECT * FROM test_select_partition", ImmutableMultiset.builder() - .addCopies(GET_TABLE, 2) + .add(GET_TABLE) .add(GET_PARTITION_NAMES_BY_FILTER) .add(GET_PARTITIONS_BY_NAMES) .build()); @@ -136,7 +136,7 @@ public void testSelectPartitionedTable() // Specify a specific partition assertMetastoreInvocations("SELECT * FROM test_select_partition WHERE part = 10", ImmutableMultiset.builder() - .addCopies(GET_TABLE, 2) + .add(GET_TABLE) .add(GET_PARTITION_NAMES_BY_FILTER) .add(GET_PARTITIONS_BY_NAMES) .build()); @@ -269,7 +269,7 @@ public void testAnalyzePartitionedTable() assertMetastoreInvocations("ANALYZE test_analyze_partition", ImmutableMultiset.builder() - .addCopies(GET_TABLE, 2) + .addCopies(GET_TABLE, 1) .add(GET_PARTITION_NAMES_BY_FILTER) .add(GET_PARTITIONS_BY_NAMES) .add(GET_PARTITION_STATISTICS) @@ -280,7 +280,7 @@ public void testAnalyzePartitionedTable() assertMetastoreInvocations("ANALYZE test_analyze_partition", ImmutableMultiset.builder() - .addCopies(GET_TABLE, 2) + .add(GET_TABLE) .add(GET_PARTITION_NAMES_BY_FILTER) .add(GET_PARTITIONS_BY_NAMES) .add(GET_PARTITION_STATISTICS) @@ -307,7 +307,7 @@ public void testDropStatsPartitionedTable() assertMetastoreInvocations("CALL system.drop_stats('test_schema', 'drop_stats_partition')", ImmutableMultiset.builder() - .addCopies(GET_TABLE, 2) + .add(GET_TABLE) .add(GET_PARTITION_NAMES_BY_FILTER) .add(UPDATE_PARTITION_STATISTICS) .build()); @@ -316,7 +316,7 @@ public void testDropStatsPartitionedTable() assertMetastoreInvocations("CALL system.drop_stats('test_schema', 'drop_stats_partition')", ImmutableMultiset.builder() - .addCopies(GET_TABLE, 2) + .add(GET_TABLE) .add(GET_PARTITION_NAMES_BY_FILTER) .addCopies(UPDATE_PARTITION_STATISTICS, 2) .build()); diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/projection/TestDateProjectionFactory.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/projection/TestDateProjectionFactory.java new file mode 100644 index 000000000000..03cc6b724c8c --- /dev/null +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/projection/TestDateProjectionFactory.java @@ -0,0 +1,66 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive.projection; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.airlift.slice.Slices; +import io.trino.spi.predicate.Domain; +import org.junit.jupiter.api.Test; + +import java.util.Optional; + +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.COLUMN_PROJECTION_FORMAT; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.COLUMN_PROJECTION_RANGE; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.DateType.DATE; +import static io.trino.spi.type.TimestampType.TIMESTAMP_MICROS; +import static io.trino.spi.type.TimestampType.TIMESTAMP_NANOS; +import static io.trino.spi.type.TimestampType.TIMESTAMP_SECONDS; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class TestDateProjectionFactory +{ + @Test + void testTypeSupport() + { + new DateProjection("test", VARCHAR, ImmutableMap.of(COLUMN_PROJECTION_FORMAT, "yyyy-MM-dd", COLUMN_PROJECTION_RANGE, ImmutableList.of("2020-01-01", "2020-01-03"))); + new DateProjection("test", DATE, ImmutableMap.of(COLUMN_PROJECTION_FORMAT, "yyyy-MM-dd", COLUMN_PROJECTION_RANGE, ImmutableList.of("2020-01-01", "2020-01-03"))); + new DateProjection("test", TIMESTAMP_SECONDS, ImmutableMap.of(COLUMN_PROJECTION_FORMAT, "yyyy-MM-dd", COLUMN_PROJECTION_RANGE, ImmutableList.of("2020-01-01", "2020-01-03"))); + new DateProjection("test", TIMESTAMP_MICROS, ImmutableMap.of(COLUMN_PROJECTION_FORMAT, "yyyy-MM-dd", COLUMN_PROJECTION_RANGE, ImmutableList.of("2020-01-01", "2020-01-03"))); + assertThatThrownBy(() -> new DateProjection("test", TIMESTAMP_NANOS, ImmutableMap.of(COLUMN_PROJECTION_FORMAT, "yyyy-MM-dd", COLUMN_PROJECTION_RANGE, ImmutableList.of("2020-01-01", "2020-01-03")))) + .isInstanceOf(InvalidProjectionException.class) + .hasMessage("Column projection for column 'test' failed. Unsupported column type: timestamp(9)"); + assertThatThrownBy(() -> new DateProjection("test", BIGINT, ImmutableMap.of(COLUMN_PROJECTION_FORMAT, "yyyy-MM-dd", COLUMN_PROJECTION_RANGE, ImmutableList.of("2020-01-01", "2020-01-03")))) + .isInstanceOf(InvalidProjectionException.class) + .hasMessage("Column projection for column 'test' failed. Unsupported column type: bigint"); + } + + @Test + void testCreate() + { + Projection projection = new DateProjection("test", VARCHAR, ImmutableMap.of(COLUMN_PROJECTION_FORMAT, "yyyy-MM-dd", COLUMN_PROJECTION_RANGE, ImmutableList.of("2020-01-01", "2020-01-03"))); + assertThat(projection.getProjectedValues(Optional.empty())).containsExactly("2020-01-01", "2020-01-02", "2020-01-03"); + assertThat(projection.getProjectedValues(Optional.of(Domain.all(VARCHAR)))).containsExactly("2020-01-01", "2020-01-02", "2020-01-03"); + assertThat(projection.getProjectedValues(Optional.of(Domain.none(VARCHAR)))).isEmpty(); + assertThat(projection.getProjectedValues(Optional.of(Domain.singleValue(VARCHAR, Slices.utf8Slice("2020-01-02"))))).containsExactly("2020-01-02"); + assertThat(projection.getProjectedValues(Optional.of(Domain.singleValue(VARCHAR, Slices.utf8Slice("2222-01-01"))))).isEmpty(); + + assertThatThrownBy(() -> new DateProjection("test", VARCHAR, ImmutableMap.of("ignored", ImmutableList.of("2020-01-01", "2020-01-02", "2020-01-03")))) + .isInstanceOf(InvalidProjectionException.class) + .hasMessage("Column projection for column 'test' failed. Missing required property: 'partition_projection_format'"); + } +} diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/projection/TestEnumProjectionFactory.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/projection/TestEnumProjectionFactory.java new file mode 100644 index 000000000000..a3e10d53f41e --- /dev/null +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/projection/TestEnumProjectionFactory.java @@ -0,0 +1,58 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive.projection; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.airlift.slice.Slices; +import io.trino.spi.predicate.Domain; +import org.junit.jupiter.api.Test; + +import java.util.Optional; + +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.COLUMN_PROJECTION_VALUES; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class TestEnumProjectionFactory +{ + @Test + void testIsSupported() + { + new EnumProjection("test", VARCHAR, ImmutableMap.of(COLUMN_PROJECTION_VALUES, ImmutableList.of("a", "b", "c"))); + assertThatThrownBy(() -> new EnumProjection("test", BIGINT, ImmutableMap.of(COLUMN_PROJECTION_VALUES, ImmutableList.of("a", "b", "c")))) + .isInstanceOf(InvalidProjectionException.class) + .hasMessage("Column projection for column 'test' failed. Unsupported column type: bigint"); + } + + @Test + void testCreate() + { + Projection projection = new EnumProjection("test", VARCHAR, ImmutableMap.of(COLUMN_PROJECTION_VALUES, ImmutableList.of("a", "b", "c"))); + assertThat(projection.getProjectedValues(Optional.empty())).containsExactly("a", "b", "c"); + assertThat(projection.getProjectedValues(Optional.of(Domain.all(VARCHAR)))).containsExactly("a", "b", "c"); + assertThat(projection.getProjectedValues(Optional.of(Domain.none(VARCHAR)))).isEmpty(); + assertThat(projection.getProjectedValues(Optional.of(Domain.singleValue(VARCHAR, Slices.utf8Slice("b"))))).containsExactly("b"); + assertThat(projection.getProjectedValues(Optional.of(Domain.singleValue(VARCHAR, Slices.utf8Slice("x"))))).isEmpty(); + + assertThatThrownBy(() -> new EnumProjection("test", VARCHAR, ImmutableMap.of("ignored", ImmutableList.of("a", "b", "c")))) + .isInstanceOf(InvalidProjectionException.class) + .hasMessage("Column projection for column 'test' failed. Missing required property: 'partition_projection_values'"); + + assertThatThrownBy(() -> new EnumProjection("test", VARCHAR, ImmutableMap.of(COLUMN_PROJECTION_VALUES, "invalid"))) + .isInstanceOf(ClassCastException.class); + } +} diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/projection/TestInjectedProjectionFactory.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/projection/TestInjectedProjectionFactory.java new file mode 100644 index 000000000000..8591cfcff272 --- /dev/null +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/projection/TestInjectedProjectionFactory.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive.projection; + +import io.airlift.slice.Slices; +import io.trino.spi.predicate.Domain; +import org.junit.jupiter.api.Test; + +import java.util.Optional; + +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.CharType.createCharType; +import static io.trino.spi.type.TimestampType.TIMESTAMP_PICOS; +import static io.trino.spi.type.TimestampType.TIMESTAMP_SECONDS; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class TestInjectedProjectionFactory +{ + @Test + void testIsSupported() + { + new InjectedProjection("test", VARCHAR); + new InjectedProjection("test", createCharType(10)); + new InjectedProjection("test", BIGINT); + assertThatThrownBy(() -> new InjectedProjection("test", TIMESTAMP_SECONDS)) + .isInstanceOf(InvalidProjectionException.class) + .hasMessage("Column projection for column 'test' failed. Unsupported column type: timestamp(0)"); + assertThatThrownBy(() -> new InjectedProjection("test", TIMESTAMP_PICOS)) + .isInstanceOf(InvalidProjectionException.class) + .hasMessage("Column projection for column 'test' failed. Unsupported column type: timestamp(12)"); + } + + @Test + void testCreate() + { + Projection projection = new InjectedProjection("test", VARCHAR); + assertThat(projection.getProjectedValues(Optional.of(Domain.singleValue(VARCHAR, Slices.utf8Slice("b"))))).containsExactly("b"); + assertThat(projection.getProjectedValues(Optional.of(Domain.singleValue(VARCHAR, Slices.utf8Slice("x"))))).containsExactly("x"); + + assertThatThrownBy(() -> assertThat(projection.getProjectedValues(Optional.empty())).containsExactly("a", "b", "c")) + .isInstanceOf(InvalidProjectionException.class) + .hasMessage("Column projection for column 'test' failed. Injected projection requires single predicate for it's column in where clause"); + assertThatThrownBy(() -> assertThat(projection.getProjectedValues(Optional.of(Domain.all(VARCHAR)))).containsExactly("a", "b", "c")) + .isInstanceOf(InvalidProjectionException.class) + .hasMessage("Column projection for column 'test' failed. Injected projection requires single predicate for it's column in where clause. Currently provided can't be converted to single partition."); + assertThatThrownBy(() -> assertThat(projection.getProjectedValues(Optional.of(Domain.none(VARCHAR)))).isEmpty()) + .isInstanceOf(InvalidProjectionException.class) + .hasMessage("Column projection for column 'test' failed. Injected projection requires single predicate for it's column in where clause. Currently provided can't be converted to single partition."); + } +} diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/projection/TestIntegerProjectionFactory.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/projection/TestIntegerProjectionFactory.java new file mode 100644 index 000000000000..345cda943088 --- /dev/null +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/projection/TestIntegerProjectionFactory.java @@ -0,0 +1,85 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive.projection; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.trino.spi.predicate.Domain; +import org.junit.jupiter.api.Test; + +import java.util.Optional; + +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.COLUMN_PROJECTION_DIGITS; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.COLUMN_PROJECTION_INTERVAL; +import static io.trino.plugin.hive.projection.PartitionProjectionProperties.COLUMN_PROJECTION_RANGE; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.DateType.DATE; +import static io.trino.spi.type.IntegerType.INTEGER; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class TestIntegerProjectionFactory +{ + @Test + void testIsSupported() + { + new IntegerProjection("test", VARCHAR, ImmutableMap.of(COLUMN_PROJECTION_RANGE, ImmutableList.of("1", "3"))); + new IntegerProjection("test", INTEGER, ImmutableMap.of(COLUMN_PROJECTION_RANGE, ImmutableList.of("1", "3"))); + new IntegerProjection("test", BIGINT, ImmutableMap.of(COLUMN_PROJECTION_RANGE, ImmutableList.of("1", "3"))); + assertThatThrownBy(() -> new IntegerProjection("test", DATE, ImmutableMap.of(COLUMN_PROJECTION_RANGE, ImmutableList.of("1", "3")))) + .isInstanceOf(InvalidProjectionException.class) + .hasMessage("Column projection for column 'test' failed. Unsupported column type: date"); + } + + @Test + void testCreateBasic() + { + Projection projection = new IntegerProjection("test", INTEGER, ImmutableMap.of(COLUMN_PROJECTION_RANGE, ImmutableList.of("1", "3"))); + assertThat(projection.getProjectedValues(Optional.empty())).containsExactly("1", "2", "3"); + assertThat(projection.getProjectedValues(Optional.of(Domain.all(INTEGER)))).containsExactly("1", "2", "3"); + assertThat(projection.getProjectedValues(Optional.of(Domain.none(INTEGER)))).isEmpty(); + assertThat(projection.getProjectedValues(Optional.of(Domain.singleValue(INTEGER, 2L)))).containsExactly("2"); + assertThat(projection.getProjectedValues(Optional.of(Domain.singleValue(INTEGER, 7L)))).isEmpty(); + + assertThatThrownBy(() -> new IntegerProjection("test", INTEGER, ImmutableMap.of("ignored", ImmutableList.of("1", "3")))) + .isInstanceOf(InvalidProjectionException.class) + .hasMessage("Column projection for column 'test' failed. Missing required property: 'partition_projection_range'"); + + assertThatThrownBy(() -> new IntegerProjection("test", INTEGER, ImmutableMap.of(COLUMN_PROJECTION_RANGE, "invalid"))) + .isInstanceOf(ClassCastException.class); + } + + @Test + void testInterval() + { + Projection projection = new IntegerProjection("test", INTEGER, ImmutableMap.of(COLUMN_PROJECTION_RANGE, ImmutableList.of("10", "30"), COLUMN_PROJECTION_INTERVAL, 10)); + assertThat(projection.getProjectedValues(Optional.empty())).containsExactly("10", "20", "30"); + assertThat(projection.getProjectedValues(Optional.of(Domain.all(INTEGER)))).containsExactly("10", "20", "30"); + assertThat(projection.getProjectedValues(Optional.of(Domain.none(INTEGER)))).isEmpty(); + assertThat(projection.getProjectedValues(Optional.of(Domain.singleValue(INTEGER, 20L)))).containsExactly("20"); + assertThat(projection.getProjectedValues(Optional.of(Domain.singleValue(INTEGER, 70L)))).isEmpty(); + } + + @Test + void testCreateDigits() + { + Projection projection = new IntegerProjection("test", INTEGER, ImmutableMap.of(COLUMN_PROJECTION_RANGE, ImmutableList.of("1", "3"), COLUMN_PROJECTION_DIGITS, 3)); + assertThat(projection.getProjectedValues(Optional.empty())).containsExactly("001", "002", "003"); + assertThat(projection.getProjectedValues(Optional.of(Domain.all(INTEGER)))).containsExactly("001", "002", "003"); + assertThat(projection.getProjectedValues(Optional.of(Domain.none(INTEGER)))).isEmpty(); + assertThat(projection.getProjectedValues(Optional.of(Domain.singleValue(INTEGER, 2L)))).containsExactly("002"); + assertThat(projection.getProjectedValues(Optional.of(Domain.singleValue(INTEGER, 7L)))).isEmpty(); + } +}