Skip to content

Rename hive.s3.storage-class-filter and add documentation #25633

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pettyjamesm merged 1 commit into from
Apr 22, 2025
Merged

Rename hive.s3.storage-class-filter and add documentation #25633

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions docs/src/main/sphinx/connector/hive.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,9 @@ Hive connector documentation.
* - `hive.partition-projection-enabled`
- Enables Athena partition projection support
- `true`
* - `hive.s3-glacier-filter`
- Filter S3 objects based on their storage class and restored status if applicable
- `true`
* - `hive.max-partition-drops-per-query`
- Maximum number of partitions to drop in a single query.
- 100,000
Expand Down
13 changes: 7 additions & 6 deletions plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ public class HiveConfig

private boolean partitionProjectionEnabled = true;

private S3StorageClassFilter s3StorageClassFilter = S3StorageClassFilter.READ_ALL;
private S3GlacierFilter s3GlacierFilter = S3GlacierFilter.READ_ALL;

private int metadataParallelism = 8;

Expand Down Expand Up @@ -1255,16 +1255,17 @@ public HiveConfig setPartitionProjectionEnabled(boolean enabledAthenaPartitionPr
return this;
}

public S3StorageClassFilter getS3StorageClassFilter()
public S3GlacierFilter getS3GlacierFilter()
{
return s3StorageClassFilter;
return s3GlacierFilter;
}

@Config("hive.s3.storage-class-filter")
@LegacyConfig("hive.s3.storage-class-filter")
@Config("hive.s3-glacier-filter")
@ConfigDescription("Filter based on storage class of S3 object")
public HiveConfig setS3StorageClassFilter(S3StorageClassFilter s3StorageClassFilter)
public HiveConfig setS3GlacierFilter(S3GlacierFilter s3GlacierFilter)
{
this.s3StorageClassFilter = s3StorageClassFilter;
this.s3GlacierFilter = s3GlacierFilter;
return this;
}

Expand Down
6 changes: 3 additions & 3 deletions ...ino/plugin/hive/S3StorageClassFilter.java → ...io/trino/plugin/hive/S3GlacierFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

import static com.google.common.base.Predicates.alwaysTrue;

public enum S3StorageClassFilter {
public enum S3GlacierFilter {
READ_ALL,
READ_NON_GLACIER,
READ_NON_GLACIER_AND_RESTORED;
Expand Down Expand Up @@ -52,8 +52,8 @@ public Predicate<FileEntry> toFileEntryPredicate()
{
return switch (this) {
case READ_ALL -> alwaysTrue();
case READ_NON_GLACIER -> S3StorageClassFilter::isNotGlacierObject;
case READ_NON_GLACIER_AND_RESTORED -> S3StorageClassFilter::isCompletedRestoredObject;
case READ_NON_GLACIER -> S3GlacierFilter::isNotGlacierObject;
case READ_NON_GLACIER_AND_RESTORED -> S3GlacierFilter::isCompletedRestoredObject;
};
}
}
2 changes: 1 addition & 1 deletion plugin/trino-hive/src/main/java/io/trino/plugin/hive/fs/CachingDirectoryLister.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ public CachingDirectoryLister(HiveConfig hiveClientConfig)
hiveClientConfig.getFileStatusCacheExpireAfterWrite(),
hiveClientConfig.getFileStatusCacheMaxRetainedSize(),
hiveClientConfig.getFileStatusCacheTables(),
hiveClientConfig.getS3StorageClassFilter().toFileEntryPredicate());
hiveClientConfig.getS3GlacierFilter().toFileEntryPredicate());
}

public CachingDirectoryLister(
Expand Down
6 changes: 3 additions & 3 deletions plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ public void testDefaults()
.setHudiCatalogName(null)
.setAutoPurge(false)
.setPartitionProjectionEnabled(true)
.setS3StorageClassFilter(S3StorageClassFilter.READ_ALL)
.setS3GlacierFilter(S3GlacierFilter.READ_ALL)
.setMetadataParallelism(8));
}

Expand Down Expand Up @@ -203,7 +203,7 @@ public void testExplicitPropertyMappings()
.put("hive.hudi-catalog-name", "hudi")
.put("hive.auto-purge", "true")
.put("hive.partition-projection-enabled", "false")
.put("hive.s3.storage-class-filter", "READ_NON_GLACIER_AND_RESTORED")
.put("hive.s3-glacier-filter", "READ_NON_GLACIER_AND_RESTORED")
.put("hive.metadata.parallelism", "10")
.buildOrThrow();

Expand Down Expand Up @@ -286,7 +286,7 @@ public void testExplicitPropertyMappings()
.setHudiCatalogName("hudi")
.setAutoPurge(true)
.setPartitionProjectionEnabled(false)
.setS3StorageClassFilter(S3StorageClassFilter.READ_NON_GLACIER_AND_RESTORED)
.setS3GlacierFilter(S3GlacierFilter.READ_NON_GLACIER_AND_RESTORED)
.setMetadataParallelism(10);

assertFullMapping(properties, expected);
Expand Down