diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml index 18c63fbe7bb1..6aec3a6d2745 100644 --- a/.palantir/revapi.yml +++ b/.palantir/revapi.yml @@ -1147,6 +1147,12 @@ acceptedBreaks: justification: "Removing deprecated code" "1.7.0": org.apache.iceberg:iceberg-core: + - code: "java.field.removedWithConstant" + old: "field org.apache.iceberg.TableProperties.OBJECT_STORE_PATH" + justification: "Deprecated since 0.12.0" + - code: "java.field.removedWithConstant" + old: "field org.apache.iceberg.TableProperties.WRITE_FOLDER_STORAGE_LOCATION" + justification: "Deprecated since 0.12.0" - code: "java.method.removed" old: "method org.apache.iceberg.deletes.PositionDeleteIndex\ \ org.apache.iceberg.deletes.Deletes::toPositionIndex(java.lang.CharSequence,\ diff --git a/aws/src/integration/java/org/apache/iceberg/aws/glue/GlueTestBase.java b/aws/src/integration/java/org/apache/iceberg/aws/glue/GlueTestBase.java index 65e37eba4cd3..c3c026a075e4 100644 --- a/aws/src/integration/java/org/apache/iceberg/aws/glue/GlueTestBase.java +++ b/aws/src/integration/java/org/apache/iceberg/aws/glue/GlueTestBase.java @@ -73,9 +73,8 @@ public class GlueTestBase { static final Map TABLE_LOCATION_PROPERTIES = ImmutableMap.of( TableProperties.WRITE_DATA_LOCATION, "s3://" + TEST_BUCKET_NAME + "/writeDataLoc", - TableProperties.WRITE_METADATA_LOCATION, "s3://" + TEST_BUCKET_NAME + "/writeMetaDataLoc", - TableProperties.WRITE_FOLDER_STORAGE_LOCATION, - "s3://" + TEST_BUCKET_NAME + "/writeFolderStorageLoc"); + TableProperties.WRITE_METADATA_LOCATION, + "s3://" + TEST_BUCKET_NAME + "/writeMetaDataLoc"); static final String TEST_BUCKET_PATH = "s3://" + TEST_BUCKET_NAME + "/" + TEST_PATH_PREFIX; diff --git a/aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java b/aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java index 56b38a47e968..0497601bec6d 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java +++ b/aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java @@ -69,10 +69,7 @@ private IcebergToGlueConverter() {} public static final String ICEBERG_FIELD_CURRENT = "iceberg.field.current"; private static final List ADDITIONAL_LOCATION_PROPERTIES = ImmutableList.of( - TableProperties.WRITE_DATA_LOCATION, - TableProperties.WRITE_METADATA_LOCATION, - TableProperties.OBJECT_STORE_PATH, - TableProperties.WRITE_FOLDER_STORAGE_LOCATION); + TableProperties.WRITE_DATA_LOCATION, TableProperties.WRITE_METADATA_LOCATION); // Attempt to set additionalLocations if available on the given AWS SDK version private static final DynMethods.UnboundMethod SET_ADDITIONAL_LOCATIONS = diff --git a/aws/src/test/java/org/apache/iceberg/aws/glue/TestIcebergToGlueConverter.java b/aws/src/test/java/org/apache/iceberg/aws/glue/TestIcebergToGlueConverter.java index edebfd3420e2..fdc9569d829b 100644 --- a/aws/src/test/java/org/apache/iceberg/aws/glue/TestIcebergToGlueConverter.java +++ b/aws/src/test/java/org/apache/iceberg/aws/glue/TestIcebergToGlueConverter.java @@ -47,8 +47,7 @@ public class TestIcebergToGlueConverter { private final Map tableLocationProperties = ImmutableMap.of( TableProperties.WRITE_DATA_LOCATION, "s3://writeDataLoc", - TableProperties.WRITE_METADATA_LOCATION, "s3://writeMetaDataLoc", - TableProperties.WRITE_FOLDER_STORAGE_LOCATION, "s3://writeFolderStorageLoc"); + TableProperties.WRITE_METADATA_LOCATION, "s3://writeMetaDataLoc"); @Test public void testToDatabaseName() { diff --git a/core/src/main/java/org/apache/iceberg/LocationProviders.java b/core/src/main/java/org/apache/iceberg/LocationProviders.java index 68bec2f4e4fc..2afe2a52f181 100644 --- a/core/src/main/java/org/apache/iceberg/LocationProviders.java +++ b/core/src/main/java/org/apache/iceberg/LocationProviders.java @@ -85,10 +85,7 @@ static class DefaultLocationProvider implements LocationProvider { private static String dataLocation(Map properties, String tableLocation) { String dataLocation = properties.get(TableProperties.WRITE_DATA_LOCATION); if (dataLocation == null) { - dataLocation = properties.get(TableProperties.WRITE_FOLDER_STORAGE_LOCATION); - if (dataLocation == null) { - dataLocation = String.format("%s/data", tableLocation); - } + dataLocation = String.format("%s/data", tableLocation); } return dataLocation; } @@ -137,13 +134,7 @@ static class ObjectStoreLocationProvider implements LocationProvider { private static String dataLocation(Map properties, String tableLocation) { String dataLocation = properties.get(TableProperties.WRITE_DATA_LOCATION); if (dataLocation == null) { - dataLocation = properties.get(TableProperties.OBJECT_STORE_PATH); - if (dataLocation == null) { - dataLocation = properties.get(TableProperties.WRITE_FOLDER_STORAGE_LOCATION); - if (dataLocation == null) { - dataLocation = String.format("%s/data", tableLocation); - } - } + dataLocation = String.format("%s/data", tableLocation); } return dataLocation; } diff --git a/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java b/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java index 714c0b3bfe67..2e4b21b3d9a1 100644 --- a/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java +++ b/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java @@ -139,9 +139,6 @@ public static TableMetadata replacePaths( private static Map updateProperties( Map tableProperties, String sourcePrefix, String targetPrefix) { Map properties = Maps.newHashMap(tableProperties); - updatePathInProperty(properties, sourcePrefix, targetPrefix, TableProperties.OBJECT_STORE_PATH); - updatePathInProperty( - properties, sourcePrefix, targetPrefix, TableProperties.WRITE_FOLDER_STORAGE_LOCATION); updatePathInProperty( properties, sourcePrefix, targetPrefix, TableProperties.WRITE_DATA_LOCATION); updatePathInProperty( diff --git a/core/src/main/java/org/apache/iceberg/TableProperties.java b/core/src/main/java/org/apache/iceberg/TableProperties.java index cd7cda23c2d3..acacb964f39c 100644 --- a/core/src/main/java/org/apache/iceberg/TableProperties.java +++ b/core/src/main/java/org/apache/iceberg/TableProperties.java @@ -249,19 +249,8 @@ private TableProperties() {} "write.object-storage.partitioned-paths"; public static final boolean WRITE_OBJECT_STORE_PARTITIONED_PATHS_DEFAULT = true; - /** - * @deprecated Use {@link #WRITE_DATA_LOCATION} instead. - */ - @Deprecated public static final String OBJECT_STORE_PATH = "write.object-storage.path"; - public static final String WRITE_LOCATION_PROVIDER_IMPL = "write.location-provider.impl"; - /** - * @deprecated Use {@link #WRITE_DATA_LOCATION} instead. - */ - @Deprecated - public static final String WRITE_FOLDER_STORAGE_LOCATION = "write.folder-storage.path"; - // This only applies to files written after this property is set. Files previously written aren't // relocated to reflect this parameter. // If not set, defaults to a "data" folder underneath the root path of the table. diff --git a/core/src/test/java/org/apache/iceberg/TestLocationProvider.java b/core/src/test/java/org/apache/iceberg/TestLocationProvider.java index 7edba51c3d85..5f60eaa1b2e4 100644 --- a/core/src/test/java/org/apache/iceberg/TestLocationProvider.java +++ b/core/src/test/java/org/apache/iceberg/TestLocationProvider.java @@ -221,23 +221,6 @@ public void testObjectStorageLocationProviderPathResolution() { .as("default data location should be used when object storage path not set") .contains(table.location() + "/data"); - String folderPath = "s3://random/folder/location"; - table - .updateProperties() - .set(TableProperties.WRITE_FOLDER_STORAGE_LOCATION, folderPath) - .commit(); - - assertThat(table.locationProvider().newDataLocation("file")) - .as("folder storage path should be used when set") - .contains(folderPath); - - String objectPath = "s3://random/object/location"; - table.updateProperties().set(TableProperties.OBJECT_STORE_PATH, objectPath).commit(); - - assertThat(table.locationProvider().newDataLocation("file")) - .as("object storage path should be used when set") - .contains(objectPath); - String dataPath = "s3://random/data/location"; table.updateProperties().set(TableProperties.WRITE_DATA_LOCATION, dataPath).commit(); assertThat(table.locationProvider().newDataLocation("file")) @@ -253,16 +236,6 @@ public void testDefaultStorageLocationProviderPathResolution() { .as("default data location should be used when object storage path not set") .contains(table.location() + "/data"); - String folderPath = "s3://random/folder/location"; - table - .updateProperties() - .set(TableProperties.WRITE_FOLDER_STORAGE_LOCATION, folderPath) - .commit(); - - assertThat(table.locationProvider().newDataLocation("file")) - .as("folder storage path should be used when set") - .contains(folderPath); - String dataPath = "s3://random/data/location"; table.updateProperties().set(TableProperties.WRITE_DATA_LOCATION, dataPath).commit(); diff --git a/docs/docs/aws.md b/docs/docs/aws.md index 1a98a4d18e5b..5e9bf19b9c41 100644 --- a/docs/docs/aws.md +++ b/docs/docs/aws.md @@ -381,11 +381,6 @@ the end. s3://my-table-data-bucket/my_ns.db/my_table/0101/0110/1001/10110010/category=orders/00000-0-5affc076-96a4-48f2-9cd2-d5efbc9f0c94-00001.parquet ``` -Note, the path resolution logic for `ObjectStoreLocationProvider` is `write.data.path` then `/data`. -However, for the older versions up to 0.12.0, the logic is as follows: -- before 0.12.0, `write.object-storage.path` must be set. -- at 0.12.0, `write.object-storage.path` then `write.folder-storage.path` then `/data`. - For more details, please refer to the [LocationProvider Configuration](custom-catalog.md#custom-location-provider-implementation) section. We have also added a new table property `write.object-storage.partitioned-paths` that if set to false(default=true), this will diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/actions/SnapshotTableSparkAction.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/actions/SnapshotTableSparkAction.java index 5f7f408cb099..1ef4880144a9 100644 --- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/actions/SnapshotTableSparkAction.java +++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/actions/SnapshotTableSparkAction.java @@ -175,8 +175,6 @@ protected Map destTableProps() { // remove any possible location properties from origin properties properties.remove(LOCATION); properties.remove(TableProperties.WRITE_METADATA_LOCATION); - properties.remove(TableProperties.WRITE_FOLDER_STORAGE_LOCATION); - properties.remove(TableProperties.OBJECT_STORE_PATH); properties.remove(TableProperties.WRITE_DATA_LOCATION); // set default and user-provided props diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/actions/SnapshotTableSparkAction.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/actions/SnapshotTableSparkAction.java index 5f7f408cb099..1ef4880144a9 100644 --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/actions/SnapshotTableSparkAction.java +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/actions/SnapshotTableSparkAction.java @@ -175,8 +175,6 @@ protected Map destTableProps() { // remove any possible location properties from origin properties properties.remove(LOCATION); properties.remove(TableProperties.WRITE_METADATA_LOCATION); - properties.remove(TableProperties.WRITE_FOLDER_STORAGE_LOCATION); - properties.remove(TableProperties.OBJECT_STORE_PATH); properties.remove(TableProperties.WRITE_DATA_LOCATION); // set default and user-provided props diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/SnapshotTableSparkAction.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/SnapshotTableSparkAction.java index 5f7f408cb099..1ef4880144a9 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/SnapshotTableSparkAction.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/SnapshotTableSparkAction.java @@ -175,8 +175,6 @@ protected Map destTableProps() { // remove any possible location properties from origin properties properties.remove(LOCATION); properties.remove(TableProperties.WRITE_METADATA_LOCATION); - properties.remove(TableProperties.WRITE_FOLDER_STORAGE_LOCATION); - properties.remove(TableProperties.OBJECT_STORE_PATH); properties.remove(TableProperties.WRITE_DATA_LOCATION); // set default and user-provided props