diff --git a/docs/src/main/sphinx/connector/iceberg.md b/docs/src/main/sphinx/connector/iceberg.md index 29cc27b3fd86..ea3cc34caf60 100644 --- a/docs/src/main/sphinx/connector/iceberg.md +++ b/docs/src/main/sphinx/connector/iceberg.md @@ -198,7 +198,7 @@ implementation is used: - Set to `false` to disable in-memory caching of metadata files on the coordinator. This cache is not used when `fs.cache.enabled` is set to true. - `true` -* - `iceberg.object-store.enabled` +* - `iceberg.object-store-layout.enabled` - Set to `true` to enable Iceberg's [object store file layout](https://iceberg.apache.org/docs/latest/aws/#object-store-file-layout). Enabling the object store file layout appends a deterministic hash directly after the data write path. @@ -816,7 +816,7 @@ The following table properties can be updated after a table is created: - `format_version` - `partitioning` - `sorted_by` -- `object_store_enabled` +- `object_store_layout_enabled` - `data_location` For example, to update a table from v1 of the Iceberg specification to v2: @@ -880,7 +880,7 @@ connector using a {doc}`WITH ` clause. - Comma-separated list of columns to use for Parquet bloom filter. It improves the performance of queries using Equality and IN predicates when reading Parquet files. Requires Parquet format. Defaults to `[]`. -* - `object_store_enabled` +* - `object_store_layout_enabled` - Whether Iceberg's [object store file layout](https://iceberg.apache.org/docs/latest/aws/#object-store-file-layout) is enabled. * - `data_location` - Optionally specifies the file system location URI for the table's data files diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java index 1ef5dc79d63a..395b19bb40e9 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java @@ -91,7 +91,7 @@ public class IcebergConfig private List allowedExtraProperties = ImmutableList.of(); private boolean incrementalRefreshEnabled = true; private boolean metadataCacheEnabled = true; - private boolean objectStoreEnabled; + private boolean objectStoreLayoutEnabled; private int metadataParallelism = 8; public CatalogType getCatalogType() @@ -522,16 +522,16 @@ public IcebergConfig setMetadataCacheEnabled(boolean metadataCacheEnabled) return this; } - public boolean isObjectStoreEnabled() + public boolean isObjectStoreLayoutEnabled() { - return objectStoreEnabled; + return objectStoreLayoutEnabled; } - @Config("iceberg.object-store.enabled") + @Config("iceberg.object-store-layout.enabled") @ConfigDescription("Enable the Iceberg object store file layout") - public IcebergConfig setObjectStoreEnabled(boolean objectStoreEnabled) + public IcebergConfig setObjectStoreLayoutEnabled(boolean objectStoreLayoutEnabled) { - this.objectStoreEnabled = objectStoreEnabled; + this.objectStoreLayoutEnabled = objectStoreLayoutEnabled; return this; } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java index c3bbc0f5db5c..81653b5e20df 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java @@ -271,7 +271,7 @@ import static io.trino.plugin.iceberg.IcebergTableProperties.EXTRA_PROPERTIES_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.FILE_FORMAT_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.FORMAT_VERSION_PROPERTY; -import static io.trino.plugin.iceberg.IcebergTableProperties.OBJECT_STORE_ENABLED_PROPERTY; +import static io.trino.plugin.iceberg.IcebergTableProperties.OBJECT_STORE_LAYOUT_ENABLED_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.PARTITIONING_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.SORTED_BY_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.getPartitioning; @@ -380,7 +380,7 @@ public class IcebergMetadata .add(EXTRA_PROPERTIES_PROPERTY) .add(FILE_FORMAT_PROPERTY) .add(FORMAT_VERSION_PROPERTY) - .add(OBJECT_STORE_ENABLED_PROPERTY) + .add(OBJECT_STORE_LAYOUT_ENABLED_PROPERTY) .add(DATA_LOCATION_PROPERTY) .add(PARTITIONING_PROPERTY) .add(SORTED_BY_PROPERTY) @@ -2161,8 +2161,8 @@ public void setTableProperties(ConnectorSession session, ConnectorTableHandle ta updateProperties.set(FORMAT_VERSION, Integer.toString(formatVersion)); } - if (properties.containsKey(OBJECT_STORE_ENABLED_PROPERTY)) { - boolean objectStoreEnabled = (boolean) properties.get(OBJECT_STORE_ENABLED_PROPERTY) + if (properties.containsKey(OBJECT_STORE_LAYOUT_ENABLED_PROPERTY)) { + boolean objectStoreEnabled = (boolean) properties.get(OBJECT_STORE_LAYOUT_ENABLED_PROPERTY) .orElseThrow(() -> new IllegalArgumentException("The object_store_enabled property cannot be empty")); updateProperties.set(OBJECT_STORE_ENABLED, Boolean.toString(objectStoreEnabled)); } @@ -2171,10 +2171,10 @@ public void setTableProperties(ConnectorSession session, ConnectorTableHandle ta String dataLocation = (String) properties.get(DATA_LOCATION_PROPERTY) .orElseThrow(() -> new IllegalArgumentException("The data_location property cannot be empty")); boolean objectStoreEnabled = (boolean) properties.getOrDefault( - OBJECT_STORE_ENABLED_PROPERTY, + OBJECT_STORE_LAYOUT_ENABLED_PROPERTY, Optional.of(Boolean.parseBoolean(icebergTable.properties().get(OBJECT_STORE_ENABLED)))).orElseThrow(); if (!objectStoreEnabled) { - throw new TrinoException(INVALID_TABLE_PROPERTY, "Data location can only be set when object store is enabled"); + throw new TrinoException(INVALID_TABLE_PROPERTY, "Data location can only be set when object store layout is enabled"); } updateProperties.set(WRITE_DATA_LOCATION, dataLocation); } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergTableProperties.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergTableProperties.java index 532650d88c39..ec14b2585836 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergTableProperties.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergTableProperties.java @@ -59,7 +59,7 @@ public class IcebergTableProperties public static final String ORC_BLOOM_FILTER_COLUMNS_PROPERTY = "orc_bloom_filter_columns"; public static final String ORC_BLOOM_FILTER_FPP_PROPERTY = "orc_bloom_filter_fpp"; public static final String PARQUET_BLOOM_FILTER_COLUMNS_PROPERTY = "parquet_bloom_filter_columns"; - public static final String OBJECT_STORE_ENABLED_PROPERTY = "object_store_enabled"; + public static final String OBJECT_STORE_LAYOUT_ENABLED_PROPERTY = "object_store_layout_enabled"; public static final String DATA_LOCATION_PROPERTY = "data_location"; public static final String EXTRA_PROPERTIES_PROPERTY = "extra_properties"; @@ -71,7 +71,7 @@ public class IcebergTableProperties .add(FORMAT_VERSION_PROPERTY) .add(ORC_BLOOM_FILTER_COLUMNS_PROPERTY) .add(ORC_BLOOM_FILTER_FPP_PROPERTY) - .add(OBJECT_STORE_ENABLED_PROPERTY) + .add(OBJECT_STORE_LAYOUT_ENABLED_PROPERTY) .add(DATA_LOCATION_PROPERTY) .add(EXTRA_PROPERTIES_PROPERTY) .add(PARQUET_BLOOM_FILTER_COLUMNS_PROPERTY) @@ -181,9 +181,9 @@ public IcebergTableProperties( }, value -> value)) .add(booleanProperty( - OBJECT_STORE_ENABLED_PROPERTY, + OBJECT_STORE_LAYOUT_ENABLED_PROPERTY, "Set to true to enable Iceberg object store file layout", - icebergConfig.isObjectStoreEnabled(), + icebergConfig.isObjectStoreLayoutEnabled(), false)) .add(stringProperty( DATA_LOCATION_PROPERTY, @@ -264,9 +264,9 @@ public static List getParquetBloomFilterColumns(Map tabl return parquetBloomFilterColumns == null ? ImmutableList.of() : ImmutableList.copyOf(parquetBloomFilterColumns); } - public static boolean getObjectStoreEnabled(Map tableProperties) + public static boolean getObjectStoreLayoutEnabled(Map tableProperties) { - return (boolean) tableProperties.getOrDefault(OBJECT_STORE_ENABLED_PROPERTY, false); + return (boolean) tableProperties.getOrDefault(OBJECT_STORE_LAYOUT_ENABLED_PROPERTY, false); } public static Optional getDataLocation(Map tableProperties) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java index 7cd1ba1dc23c..ecd82d188260 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java @@ -123,7 +123,7 @@ import static io.trino.plugin.iceberg.IcebergTableProperties.FILE_FORMAT_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.FORMAT_VERSION_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.LOCATION_PROPERTY; -import static io.trino.plugin.iceberg.IcebergTableProperties.OBJECT_STORE_ENABLED_PROPERTY; +import static io.trino.plugin.iceberg.IcebergTableProperties.OBJECT_STORE_LAYOUT_ENABLED_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.ORC_BLOOM_FILTER_COLUMNS_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.ORC_BLOOM_FILTER_FPP_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.PARQUET_BLOOM_FILTER_COLUMNS_PROPERTY; @@ -337,7 +337,7 @@ public static Map getIcebergTableProperties(Table icebergTable) } if (parseBoolean(icebergTable.properties().getOrDefault(OBJECT_STORE_ENABLED, "false"))) { - properties.put(OBJECT_STORE_ENABLED_PROPERTY, true); + properties.put(OBJECT_STORE_LAYOUT_ENABLED_PROPERTY, true); } Optional dataLocation = Optional.ofNullable(icebergTable.properties().get(WRITE_DATA_LOCATION)); @@ -851,14 +851,14 @@ public static Map createTableProperties(ConnectorTableMetadata t propertiesBuilder.put(DEFAULT_FILE_FORMAT, fileFormat.toIceberg().toString()); propertiesBuilder.put(FORMAT_VERSION, Integer.toString(IcebergTableProperties.getFormatVersion(tableMetadata.getProperties()))); - boolean objectStoreEnabled = IcebergTableProperties.getObjectStoreEnabled(tableMetadata.getProperties()); - if (objectStoreEnabled) { + boolean objectStoreLayoutEnabled = IcebergTableProperties.getObjectStoreLayoutEnabled(tableMetadata.getProperties()); + if (objectStoreLayoutEnabled) { propertiesBuilder.put(OBJECT_STORE_ENABLED, "true"); } Optional dataLocation = IcebergTableProperties.getDataLocation(tableMetadata.getProperties()); dataLocation.ifPresent(location -> { - if (!objectStoreEnabled) { - throw new TrinoException(INVALID_TABLE_PROPERTY, "Data location can only be set when object store is enabled"); + if (!objectStoreLayoutEnabled) { + throw new TrinoException(INVALID_TABLE_PROPERTY, "Data location can only be set when object store layout is enabled"); } propertiesBuilder.put(WRITE_DATA_LOCATION, location); }); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java index a7c3457b1c26..bc722b4c8e01 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java @@ -8540,11 +8540,11 @@ public void testSetIllegalExtraPropertyKey() } @Test - public void testObjectStoreEnabledAndDataLocation() + public void testObjectStoreLayoutEnabledAndDataLocation() throws Exception { - String tableName = "test_object_store_enabled_data_location" + randomNameSuffix(); - assertUpdate("CREATE TABLE " + tableName + " WITH (object_store_enabled = true, data_location = 'local:///data-location/xyz') AS SELECT 1 AS val", 1); + String tableName = "test_object_store_layout_enabled_data_location" + randomNameSuffix(); + assertUpdate("CREATE TABLE " + tableName + " WITH (object_store_layout_enabled = true, data_location = 'local:///data-location/xyz') AS SELECT 1 AS val", 1); Location tableLocation = Location.of(getTableLocation(tableName)); assertThat(fileSystem.directoryExists(tableLocation).get()).isTrue(); @@ -8560,11 +8560,11 @@ public void testObjectStoreEnabledAndDataLocation() } @Test - public void testCreateTableWithDataLocationButObjectStoreDisabled() + public void testCreateTableWithDataLocationButObjectStoreLayoutDisabled() { assertQueryFails( "CREATE TABLE test_data_location WITH (data_location = 'local:///data-location/xyz') AS SELECT 1 AS val", - "Data location can only be set when object store is enabled"); + "Data location can only be set when object store layout is enabled"); } @Override diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConfig.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConfig.java index f097dfba0678..bf849ad9f4c0 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConfig.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConfig.java @@ -76,7 +76,7 @@ public void testDefaults() .setIncrementalRefreshEnabled(true) .setMetadataCacheEnabled(true) .setIncrementalRefreshEnabled(true) - .setObjectStoreEnabled(false) + .setObjectStoreLayoutEnabled(false) .setMetadataParallelism(8)); } @@ -114,7 +114,7 @@ public void testExplicitPropertyMappings() .put("iceberg.allowed-extra-properties", "propX,propY") .put("iceberg.incremental-refresh-enabled", "false") .put("iceberg.metadata-cache.enabled", "false") - .put("iceberg.object-store.enabled", "true") + .put("iceberg.object-store-layout.enabled", "true") .put("iceberg.metadata.parallelism", "10") .buildOrThrow(); @@ -150,7 +150,7 @@ public void testExplicitPropertyMappings() .setIncrementalRefreshEnabled(false) .setMetadataCacheEnabled(false) .setIncrementalRefreshEnabled(false) - .setObjectStoreEnabled(true) + .setObjectStoreLayoutEnabled(true) .setMetadataParallelism(10); assertFullMapping(properties, expected); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergTableWithObjectStore.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergTableWithObjectStoreLayout.java similarity index 96% rename from plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergTableWithObjectStore.java rename to plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergTableWithObjectStoreLayout.java index f2fd81e58a63..3cea174502a8 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergTableWithObjectStore.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergTableWithObjectStoreLayout.java @@ -30,7 +30,7 @@ import static io.trino.testing.TestingConnectorSession.SESSION; import static org.assertj.core.api.Assertions.assertThat; -final class TestIcebergTableWithObjectStore +final class TestIcebergTableWithObjectStoreLayout extends AbstractTestQueryFramework { private HiveMetastore metastore; @@ -41,7 +41,7 @@ protected DistributedQueryRunner createQueryRunner() throws Exception { DistributedQueryRunner queryRunner = IcebergQueryRunner.builder() - .addIcebergProperty("iceberg.object-store.enabled", "true") + .addIcebergProperty("iceberg.object-store-layout.enabled", "true") .build(); metastore = ((IcebergConnector) queryRunner.getCoordinator().getConnector(ICEBERG_CATALOG)).getInjector() diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java index 0a1b689abfa0..af1f410a9485 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java @@ -183,17 +183,17 @@ public void testDefaultFormatVersion() } @Test - public void testSetPropertiesObjectStoreEnabled() + public void testSetPropertiesObjectStoreLayoutEnabled() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_object_store", "(x int) WITH (object_store_enabled = false)")) { + try (TestTable table = new TestTable(getQueryRunner()::execute, "test_object_store", "(x int) WITH (object_store_layout_enabled = false)")) { assertThat((String) computeScalar("SHOW CREATE TABLE " + table.getName())) - .doesNotContain("object_store_enabled"); + .doesNotContain("object_store_layout_enabled"); assertThat(loadTable(table.getName()).properties()) .doesNotContainKey("write.object-storage.enabled"); - assertUpdate("ALTER TABLE " + table.getName() + " SET PROPERTIES object_store_enabled = true"); + assertUpdate("ALTER TABLE " + table.getName() + " SET PROPERTIES object_store_layout_enabled = true"); assertThat((String) computeScalar("SHOW CREATE TABLE " + table.getName())) - .contains("object_store_enabled = true"); + .contains("object_store_layout_enabled = true"); assertThat(loadTable(table.getName()).properties()) .containsEntry("write.object-storage.enabled", "true"); } @@ -210,11 +210,11 @@ public void testSetPropertiesDataLocation() assertQueryFails( "ALTER TABLE " + table.getName() + " SET PROPERTIES data_location = 'local:///data-location'", - "Data location can only be set when object store is enabled"); + "Data location can only be set when object store layout is enabled"); - assertUpdate("ALTER TABLE " + table.getName() + " SET PROPERTIES object_store_enabled = true, data_location = 'local:///data-location'"); + assertUpdate("ALTER TABLE " + table.getName() + " SET PROPERTIES object_store_layout_enabled = true, data_location = 'local:///data-location'"); assertThat((String) computeScalar("SHOW CREATE TABLE " + table.getName())) - .contains("object_store_enabled = true") + .contains("object_store_layout_enabled = true") .contains("data_location = 'local:///data-location'"); assertThat(loadTable(table.getName()).properties()) .containsEntry("write.object-storage.enabled", "true") diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergSparkCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergSparkCompatibility.java index 925f3df24be3..2254de648e4b 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergSparkCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergSparkCompatibility.java @@ -737,7 +737,7 @@ public void testSparkReadingTrinoIcebergTablePropertiesData() onTrino().executeQuery( "CREATE TABLE " + trinoTableName + " (doc_id VARCHAR)\n" + " WITH (" + - " object_store_enabled = true," + + " object_store_layout_enabled = true," + " data_location = 'local:///write-data-path'," + " extra_properties = MAP(ARRAY['custom.table-property'], ARRAY['my_custom_value'])" + " )"); @@ -1203,7 +1203,7 @@ public void testSparkReadingTrinoObjectStorage(StorageFormat storageFormat, int onTrino().executeQuery(format( "CREATE TABLE %s (_string VARCHAR, _bigint BIGINT) WITH (" + - "object_store_enabled = true," + + "object_store_layout_enabled = true," + "data_location = '%s'," + "format = '%s'," + "format_version = %s)",