diff --git a/docs/src/main/sphinx/connector/iceberg.rst b/docs/src/main/sphinx/connector/iceberg.rst index c017dc145055..89cf363a30f1 100644 --- a/docs/src/main/sphinx/connector/iceberg.rst +++ b/docs/src/main/sphinx/connector/iceberg.rst @@ -216,25 +216,25 @@ The procedure affects all snapshots that are older than the time period configur ALTER TABLE test_table EXECUTE expire_snapshots(retention_threshold => '7d') -The value for ``retention_threshold`` must be higher than ``iceberg.expire_snapshots.min-retention`` in the catalog +The value for ``retention_threshold`` must be higher than or equal to ``iceberg.expire_snapshots.min-retention`` in the catalog otherwise the procedure will fail with similar message: ``Retention specified (1.00d) is shorter than the minimum retention configured in the system (7.00d)``. The default value for this property is ``7d``. -delete_orphan_files +remove_orphan_files ~~~~~~~~~~~~~~~~~~~ -The ``delete_orphan_files`` command removes all files from table's data directory which are +The ``remove_orphan_files`` command removes all files from table's data directory which are not linked from metadata files and that are older than the value of ``retention_threshold`` parameter. Deleting orphan files from time to time is recommended to keep size of table's data directory under control. -``delete_orphan_files`` can be run as follows: +``remove_orphan_files`` can be run as follows: .. code-block:: sql - ALTER TABLE test_table EXECUTE delete_orphan_files(retention_threshold => '7d') + ALTER TABLE test_table EXECUTE remove_orphan_files(retention_threshold => '7d') -The value for ``retention_threshold`` must be higher than ``iceberg.delete_orphan_files.min-retention`` in the catalog +The value for ``retention_threshold`` must be higher than or equal to ``iceberg.remove_orphan_files.min-retention`` in the catalog otherwise the procedure will fail with similar message: ``Retention specified (1.00d) is shorter than the minimum retention configured in the system (7.00d)``. The default value for this property is ``7d``. diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java index cb975725287c..843a0c1b117a 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java @@ -35,7 +35,7 @@ public class IcebergConfig public static final int FORMAT_VERSION_SUPPORT_MIN = 1; public static final int FORMAT_VERSION_SUPPORT_MAX = 2; public static final String EXPIRE_SNAPSHOTS_MIN_RETENTION = "iceberg.expire_snapshots.min-retention"; - public static final String DELETE_ORPHAN_FILES_MIN_RETENTION = "iceberg.delete_orphan_files.min-retention"; + public static final String REMOVE_ORPHAN_FILES_MIN_RETENTION = "iceberg.remove_orphan_files.min-retention"; private IcebergFileFormat fileFormat = ORC; private HiveCompressionCodec compressionCodec = ZSTD; @@ -49,7 +49,7 @@ public class IcebergConfig private Optional hiveCatalogName = Optional.empty(); private int formatVersion = FORMAT_VERSION_SUPPORT_MAX; private Duration expireSnapshotsMinRetention = new Duration(7, DAYS); - private Duration deleteOrphanFilesMinRetention = new Duration(7, DAYS); + private Duration removeOrphanFilesMinRetention = new Duration(7, DAYS); public CatalogType getCatalogType() { @@ -223,16 +223,16 @@ public IcebergConfig setExpireSnapshotsMinRetention(Duration expireSnapshotsMinR } @NotNull - public Duration getDeleteOrphanFilesMinRetention() + public Duration getRemoveOrphanFilesMinRetention() { - return deleteOrphanFilesMinRetention; + return removeOrphanFilesMinRetention; } - @Config(DELETE_ORPHAN_FILES_MIN_RETENTION) - @ConfigDescription("Minimal retention period for delete_orphan_files procedure") - public IcebergConfig setDeleteOrphanFilesMinRetention(Duration deleteOrphanFilesMinRetention) + @Config(REMOVE_ORPHAN_FILES_MIN_RETENTION) + @ConfigDescription("Minimal retention period for remove_orphan_files procedure") + public IcebergConfig setRemoveOrphanFilesMinRetention(Duration removeOrphanFilesMinRetention) { - this.deleteOrphanFilesMinRetention = deleteOrphanFilesMinRetention; + this.removeOrphanFilesMinRetention = removeOrphanFilesMinRetention; return this; } } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java index 0afc8cf41af8..c9afc9585344 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java @@ -33,9 +33,9 @@ import io.trino.plugin.hive.HiveApplyProjectionUtil.ProjectedColumnRepresentation; import io.trino.plugin.hive.HiveWrittenPartitions; import io.trino.plugin.iceberg.catalog.TrinoCatalog; -import io.trino.plugin.iceberg.procedure.IcebergDeleteOrphanFilesHandle; import io.trino.plugin.iceberg.procedure.IcebergExpireSnapshotsHandle; import io.trino.plugin.iceberg.procedure.IcebergOptimizeHandle; +import io.trino.plugin.iceberg.procedure.IcebergRemoveOrphanFilesHandle; import io.trino.plugin.iceberg.procedure.IcebergTableExecuteHandle; import io.trino.plugin.iceberg.procedure.IcebergTableProcedureId; import io.trino.spi.TrinoException; @@ -156,8 +156,8 @@ import static io.trino.plugin.iceberg.IcebergErrorCode.ICEBERG_INVALID_METADATA; import static io.trino.plugin.iceberg.IcebergMetadataColumn.FILE_PATH; import static io.trino.plugin.iceberg.IcebergMetadataColumn.isMetadataColumnId; -import static io.trino.plugin.iceberg.IcebergSessionProperties.getDeleteOrphanFilesMinRetention; import static io.trino.plugin.iceberg.IcebergSessionProperties.getExpireSnapshotMinRetention; +import static io.trino.plugin.iceberg.IcebergSessionProperties.getRemoveOrphanFilesMinRetention; import static io.trino.plugin.iceberg.IcebergSessionProperties.isProjectionPushdownEnabled; import static io.trino.plugin.iceberg.IcebergSessionProperties.isStatisticsEnabled; import static io.trino.plugin.iceberg.IcebergTableProperties.FILE_FORMAT_PROPERTY; @@ -179,9 +179,9 @@ import static io.trino.plugin.iceberg.TypeConverter.toIcebergType; import static io.trino.plugin.iceberg.TypeConverter.toTrinoType; import static io.trino.plugin.iceberg.catalog.hms.TrinoHiveCatalog.DEPENDS_ON_TABLES; -import static io.trino.plugin.iceberg.procedure.IcebergTableProcedureId.DELETE_ORPHAN_FILES; import static io.trino.plugin.iceberg.procedure.IcebergTableProcedureId.EXPIRE_SNAPSHOTS; import static io.trino.plugin.iceberg.procedure.IcebergTableProcedureId.OPTIMIZE; +import static io.trino.plugin.iceberg.procedure.IcebergTableProcedureId.REMOVE_ORPHAN_FILES; import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; import static io.trino.spi.connector.RetryMode.NO_RETRIES; import static java.lang.String.format; @@ -760,8 +760,8 @@ public Optional getTableHandleForExecute( return getTableHandleForOptimize(session, tableHandle, executeProperties, retryMode); case EXPIRE_SNAPSHOTS: return getTableHandleForExpireSnapshots(session, tableHandle, executeProperties); - case DELETE_ORPHAN_FILES: - return getTableHandleForDeleteOrphanFiles(session, tableHandle, executeProperties); + case REMOVE_ORPHAN_FILES: + return getTableHandleForRemoveOrphanFiles(session, tableHandle, executeProperties); } throw new IllegalArgumentException("Unknown procedure: " + procedureId); @@ -798,15 +798,15 @@ private Optional getTableHandleForExpireSnapshots(C icebergTable.location())); } - private Optional getTableHandleForDeleteOrphanFiles(ConnectorSession session, IcebergTableHandle tableHandle, Map executeProperties) + private Optional getTableHandleForRemoveOrphanFiles(ConnectorSession session, IcebergTableHandle tableHandle, Map executeProperties) { Duration retentionThreshold = (Duration) executeProperties.get(RETENTION_THRESHOLD); Table icebergTable = catalog.loadTable(session, tableHandle.getSchemaTableName()); return Optional.of(new IcebergTableExecuteHandle( tableHandle.getSchemaTableName(), - DELETE_ORPHAN_FILES, - new IcebergDeleteOrphanFilesHandle(retentionThreshold), + REMOVE_ORPHAN_FILES, + new IcebergRemoveOrphanFilesHandle(retentionThreshold), icebergTable.location())); } @@ -818,7 +818,7 @@ public Optional getLayoutForTableExecute(ConnectorSession case OPTIMIZE: return getLayoutForOptimize(session, executeHandle); case EXPIRE_SNAPSHOTS: - case DELETE_ORPHAN_FILES: + case REMOVE_ORPHAN_FILES: // handled via executeTableExecute } throw new IllegalArgumentException("Unknown procedure '" + executeHandle.getProcedureId() + "'"); @@ -844,7 +844,7 @@ public BeginTableExecuteResult buildSetOfValidFiles(Table table) .collect(toImmutableSet()); } - public void executeDeleteOrphanFiles(ConnectorSession session, IcebergTableExecuteHandle executeHandle) + public void executeRemoveOrphanFiles(ConnectorSession session, IcebergTableExecuteHandle executeHandle) { - IcebergDeleteOrphanFilesHandle deleteOrphanFilesHandle = (IcebergDeleteOrphanFilesHandle) executeHandle.getProcedureHandle(); + IcebergRemoveOrphanFilesHandle removeOrphanFilesHandle = (IcebergRemoveOrphanFilesHandle) executeHandle.getProcedureHandle(); Table table = catalog.loadTable(session, executeHandle.getSchemaTableName()); - Duration retention = requireNonNull(deleteOrphanFilesHandle.getRetentionThreshold(), "retention is null"); + Duration retention = requireNonNull(removeOrphanFilesHandle.getRetentionThreshold(), "retention is null"); validateTableExecuteParameters( table, executeHandle.getSchemaTableName(), - DELETE_ORPHAN_FILES.name(), + REMOVE_ORPHAN_FILES.name(), retention, - getDeleteOrphanFilesMinRetention(session), - IcebergConfig.DELETE_ORPHAN_FILES_MIN_RETENTION, - IcebergSessionProperties.DELETE_ORPHAN_FILES_MIN_RETENTION); + getRemoveOrphanFilesMinRetention(session), + IcebergConfig.REMOVE_ORPHAN_FILES_MIN_RETENTION, + IcebergSessionProperties.REMOVE_ORPHAN_FILES_MIN_RETENTION); long expireTimestampMillis = session.getStart().toEpochMilli() - retention.toMillis(); - deleteOrphanFiles(table, session, executeHandle.getSchemaTableName(), expireTimestampMillis); - deleteOrphanMetadataFiles(table, session, executeHandle.getSchemaTableName(), expireTimestampMillis); + removeOrphanFiles(table, session, executeHandle.getSchemaTableName(), expireTimestampMillis); + removeOrphanMetadataFiles(table, session, executeHandle.getSchemaTableName(), expireTimestampMillis); } - private void deleteOrphanFiles(Table table, ConnectorSession session, SchemaTableName schemaTableName, long expireTimestamp) + private void removeOrphanFiles(Table table, ConnectorSession session, SchemaTableName schemaTableName, long expireTimestamp) { Set validDataFilePaths = stream(table.snapshots()) .map(Snapshot::snapshotId) @@ -1095,7 +1095,7 @@ private void deleteOrphanFiles(Table table, ConnectorSession session, SchemaTabl scanAndDeleteInvalidFiles(table, session, schemaTableName, expireTimestamp, union(validDataFilePaths, validDeleteFilePaths), "/data"); } - private void deleteOrphanMetadataFiles(Table table, ConnectorSession session, SchemaTableName schemaTableName, long expireTimestamp) + private void removeOrphanMetadataFiles(Table table, ConnectorSession session, SchemaTableName schemaTableName, long expireTimestamp) { ImmutableSet manifests = stream(table.snapshots()) .flatMap(snapshot -> snapshot.allManifests().stream()) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java index 32c3c61bf096..19763aaba540 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java @@ -25,9 +25,9 @@ import io.trino.plugin.hive.orc.OrcWriterConfig; import io.trino.plugin.hive.parquet.ParquetReaderConfig; import io.trino.plugin.hive.parquet.ParquetWriterConfig; -import io.trino.plugin.iceberg.procedure.DeleteOrphanFilesTableProcedure; import io.trino.plugin.iceberg.procedure.ExpireSnapshotsTableProcedure; import io.trino.plugin.iceberg.procedure.OptimizeTableProcedure; +import io.trino.plugin.iceberg.procedure.RemoveOrphanFilesTableProcedure; import io.trino.spi.connector.ConnectorNodePartitioningProvider; import io.trino.spi.connector.ConnectorPageSinkProvider; import io.trino.spi.connector.ConnectorPageSourceProvider; @@ -83,6 +83,6 @@ public void configure(Binder binder) Multibinder tableProcedures = newSetBinder(binder, TableProcedureMetadata.class); tableProcedures.addBinding().toProvider(OptimizeTableProcedure.class).in(Scopes.SINGLETON); tableProcedures.addBinding().toProvider(ExpireSnapshotsTableProcedure.class).in(Scopes.SINGLETON); - tableProcedures.addBinding().toProvider(DeleteOrphanFilesTableProcedure.class).in(Scopes.SINGLETON); + tableProcedures.addBinding().toProvider(RemoveOrphanFilesTableProcedure.class).in(Scopes.SINGLETON); } } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSinkProvider.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSinkProvider.java index 47a1ec9055ea..c0bf0e002211 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSinkProvider.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSinkProvider.java @@ -125,7 +125,7 @@ public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHa optimizeHandle.getTableStorageProperties(), maxOpenPartitions); case EXPIRE_SNAPSHOTS: - case DELETE_ORPHAN_FILES: + case REMOVE_ORPHAN_FILES: // handled via ConnectorMetadata.executeTableExecute } throw new IllegalArgumentException("Unknown procedure: " + executeHandle.getProcedureId()); diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSessionProperties.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSessionProperties.java index 9d9da4b174d8..09d94ff70bf7 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSessionProperties.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSessionProperties.java @@ -75,7 +75,7 @@ public final class IcebergSessionProperties private static final String TARGET_MAX_FILE_SIZE = "target_max_file_size"; private static final String HIVE_CATALOG_NAME = "hive_catalog_name"; public static final String EXPIRE_SNAPSHOTS_MIN_RETENTION = "expire_snapshots_min_retention"; - public static final String DELETE_ORPHAN_FILES_MIN_RETENTION = "delete_orphan_files_min_retention"; + public static final String REMOVE_ORPHAN_FILES_MIN_RETENTION = "remove_orphan_files_min_retention"; private final List> sessionProperties; @@ -237,9 +237,9 @@ public IcebergSessionProperties( icebergConfig.getExpireSnapshotsMinRetention(), false)) .add(durationProperty( - DELETE_ORPHAN_FILES_MIN_RETENTION, - "Minimal retention period for delete_orphan_files procedure", - icebergConfig.getDeleteOrphanFilesMinRetention(), + REMOVE_ORPHAN_FILES_MIN_RETENTION, + "Minimal retention period for remove_orphan_files procedure", + icebergConfig.getRemoveOrphanFilesMinRetention(), false)) .build(); } @@ -392,8 +392,8 @@ public static Duration getExpireSnapshotMinRetention(ConnectorSession session) return session.getProperty(EXPIRE_SNAPSHOTS_MIN_RETENTION, Duration.class); } - public static Duration getDeleteOrphanFilesMinRetention(ConnectorSession session) + public static Duration getRemoveOrphanFilesMinRetention(ConnectorSession session) { - return session.getProperty(DELETE_ORPHAN_FILES_MIN_RETENTION, Duration.class); + return session.getProperty(REMOVE_ORPHAN_FILES_MIN_RETENTION, Duration.class); } } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/IcebergProcedureHandle.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/IcebergProcedureHandle.java index 04339a0ad6bd..96e247926338 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/IcebergProcedureHandle.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/IcebergProcedureHandle.java @@ -22,6 +22,6 @@ @JsonSubTypes({ @JsonSubTypes.Type(value = IcebergOptimizeHandle.class, name = "optimize"), @JsonSubTypes.Type(value = IcebergExpireSnapshotsHandle.class, name = "expire_snapshots"), - @JsonSubTypes.Type(value = IcebergDeleteOrphanFilesHandle.class, name = "delete_orphan_files"), + @JsonSubTypes.Type(value = IcebergRemoveOrphanFilesHandle.class, name = "remove_orphan_files"), }) public abstract class IcebergProcedureHandle {} diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/IcebergDeleteOrphanFilesHandle.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/IcebergRemoveOrphanFilesHandle.java similarity index 92% rename from plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/IcebergDeleteOrphanFilesHandle.java rename to plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/IcebergRemoveOrphanFilesHandle.java index 37c18d88c8b5..e4ac5d9f5890 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/IcebergDeleteOrphanFilesHandle.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/IcebergRemoveOrphanFilesHandle.java @@ -20,13 +20,13 @@ import static com.google.common.base.MoreObjects.toStringHelper; import static java.util.Objects.requireNonNull; -public class IcebergDeleteOrphanFilesHandle +public class IcebergRemoveOrphanFilesHandle extends IcebergProcedureHandle { private final Duration retentionThreshold; @JsonCreator - public IcebergDeleteOrphanFilesHandle(Duration retentionThreshold) + public IcebergRemoveOrphanFilesHandle(Duration retentionThreshold) { this.retentionThreshold = requireNonNull(retentionThreshold, "retentionThreshold is null"); } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/IcebergTableProcedureId.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/IcebergTableProcedureId.java index 2c3f7a49e168..e81c8336fb04 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/IcebergTableProcedureId.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/IcebergTableProcedureId.java @@ -17,5 +17,5 @@ public enum IcebergTableProcedureId { OPTIMIZE, EXPIRE_SNAPSHOTS, - DELETE_ORPHAN_FILES, + REMOVE_ORPHAN_FILES, } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/DeleteOrphanFilesTableProcedure.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/RemoveOrphanFilesTableProcedure.java similarity index 92% rename from plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/DeleteOrphanFilesTableProcedure.java rename to plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/RemoveOrphanFilesTableProcedure.java index 8629cbe07b0a..7c02298abac9 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/DeleteOrphanFilesTableProcedure.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/RemoveOrphanFilesTableProcedure.java @@ -20,17 +20,17 @@ import javax.inject.Provider; import static io.trino.plugin.base.session.PropertyMetadataUtil.durationProperty; -import static io.trino.plugin.iceberg.procedure.IcebergTableProcedureId.DELETE_ORPHAN_FILES; +import static io.trino.plugin.iceberg.procedure.IcebergTableProcedureId.REMOVE_ORPHAN_FILES; import static io.trino.spi.connector.TableProcedureExecutionMode.coordinatorOnly; -public class DeleteOrphanFilesTableProcedure +public class RemoveOrphanFilesTableProcedure implements Provider { @Override public TableProcedureMetadata get() { return new TableProcedureMetadata( - DELETE_ORPHAN_FILES.name(), + REMOVE_ORPHAN_FILES.name(), coordinatorOnly(), ImmutableList.of( durationProperty( diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java index fbe3fc389fe7..0ee557ffc386 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java @@ -3314,7 +3314,7 @@ public void testExpireSnapshotsParameterValidation() } @Test - public void testDeleteOrphanFiles() + public void testRemoveOrphanFiles() throws Exception { String tableName = "test_deleting_orphan_files_unnecessary_files" + randomTableSuffix(); @@ -3324,7 +3324,7 @@ public void testDeleteOrphanFiles() Path orphanFile = Files.createFile(Path.of(getIcebergTableDataPath(tableName).toString(), "invalidData." + format)); List initialDataFiles = getAllDataFilesFromTableDirectory(tableName); - assertQuerySucceeds(sessionWithShortRetentionUnlocked, "ALTER TABLE " + tableName + " EXECUTE DELETE_ORPHAN_FILES (retention_threshold => '0s')"); + assertQuerySucceeds(sessionWithShortRetentionUnlocked, "ALTER TABLE " + tableName + " EXECUTE REMOVE_ORPHAN_FILES (retention_threshold => '0s')"); List updatedDataFiles = getAllDataFilesFromTableDirectory(tableName); assertThat(updatedDataFiles.size()).isLessThan(initialDataFiles.size()); @@ -3332,7 +3332,7 @@ public void testDeleteOrphanFiles() } @Test - public void testIfDeleteOrphanFilesCleansUnnecessaryDataFilesInPartitionedTable() + public void testIfRemoveOrphanFilesCleansUnnecessaryDataFilesInPartitionedTable() throws Exception { String tableName = "test_deleting_orphan_files_unnecessary_files" + randomTableSuffix(); @@ -3343,7 +3343,7 @@ public void testIfDeleteOrphanFilesCleansUnnecessaryDataFilesInPartitionedTable( Path orphanFile = Files.createFile(Path.of(getIcebergTableDataPath(tableName) + "/key=one/", "invalidData." + format)); List initialDataFiles = getAllDataFilesFromTableDirectory(tableName); - assertQuerySucceeds(sessionWithShortRetentionUnlocked, "ALTER TABLE " + tableName + " EXECUTE DELETE_ORPHAN_FILES (retention_threshold => '0s')"); + assertQuerySucceeds(sessionWithShortRetentionUnlocked, "ALTER TABLE " + tableName + " EXECUTE REMOVE_ORPHAN_FILES (retention_threshold => '0s')"); List updatedDataFiles = getAllDataFilesFromTableDirectory(tableName); assertThat(updatedDataFiles.size()).isLessThan(initialDataFiles.size()); @@ -3351,7 +3351,7 @@ public void testIfDeleteOrphanFilesCleansUnnecessaryDataFilesInPartitionedTable( } @Test - public void testIfDeleteOrphanFilesCleansUnnecessaryMetadataFilesInPartitionedTable() + public void testIfRemoveOrphanFilesCleansUnnecessaryMetadataFilesInPartitionedTable() throws Exception { String tableName = "test_deleting_orphan_files_unnecessary_files" + randomTableSuffix(); @@ -3362,7 +3362,7 @@ public void testIfDeleteOrphanFilesCleansUnnecessaryMetadataFilesInPartitionedTa Path orphanMetadataFile = Files.createFile(Path.of(getIcebergTableMetadataPath(tableName).toString(), "invalidData." + format)); List initialMetadataFiles = getAllMetadataFilesFromTableDirectoryForTable(tableName); - assertQuerySucceeds(sessionWithShortRetentionUnlocked, "ALTER TABLE " + tableName + " EXECUTE DELETE_ORPHAN_FILES (retention_threshold => '0s')"); + assertQuerySucceeds(sessionWithShortRetentionUnlocked, "ALTER TABLE " + tableName + " EXECUTE REMOVE_ORPHAN_FILES (retention_threshold => '0s')"); List updatedMetadataFiles = getAllMetadataFilesFromTableDirectoryForTable(tableName); assertThat(updatedMetadataFiles.size()).isLessThan(initialMetadataFiles.size()); @@ -3399,7 +3399,7 @@ private void testCleaningUpWithTableWithSpecifiedLocation(String suffix) Session sessionWithShortRetentionUnlocked = prepareCleanUpSession(); assertQuerySucceeds(sessionWithShortRetentionUnlocked, "ALTER TABLE " + tableName + " EXECUTE EXPIRE_SNAPSHOTS (retention_threshold => '0s')"); - assertQuerySucceeds(sessionWithShortRetentionUnlocked, "ALTER TABLE " + tableName + " EXECUTE DELETE_ORPHAN_FILES (retention_threshold => '0s')"); + assertQuerySucceeds(sessionWithShortRetentionUnlocked, "ALTER TABLE " + tableName + " EXECUTE REMOVE_ORPHAN_FILES (retention_threshold => '0s')"); List updatedFiles = getAllMetadataFilesFromTableDirectory(tempDirPath); List updatedSnapshots = getSnapshotIds(tableName); assertThat(updatedFiles.size()).isEqualTo(initialFiles.size() - 1); @@ -3409,32 +3409,32 @@ private void testCleaningUpWithTableWithSpecifiedLocation(String suffix) } @Test - public void testExplainDeleteOrphanFilesOutput() + public void testExplainRemoveOrphanFilesOutput() { - String tableName = "test_delete_orphan_files_output" + randomTableSuffix(); + String tableName = "test_remove_orphan_files_output" + randomTableSuffix(); assertUpdate("CREATE TABLE " + tableName + " (key varchar, value integer) WITH (partitioning = ARRAY['key'])"); assertUpdate("INSERT INTO " + tableName + " VALUES ('one', 1)", 1); assertUpdate("INSERT INTO " + tableName + " VALUES ('two', 2)", 1); - assertExplain("EXPLAIN ALTER TABLE " + tableName + " EXECUTE DELETE_ORPHAN_FILES (retention_threshold => '0s')", - "SimpleTableExecute\\[iceberg:schemaTableName:tpch.test_delete_orphan_files.*\\{retentionThreshold=0\\.00s}.*"); + assertExplain("EXPLAIN ALTER TABLE " + tableName + " EXECUTE REMOVE_ORPHAN_FILES (retention_threshold => '0s')", + "SimpleTableExecute\\[iceberg:schemaTableName:tpch.test_remove_orphan_files.*\\{retentionThreshold=0\\.00s}.*"); } @Test - public void testDeleteOrphanFilesParameterValidation() + public void testRemoveOrphanFilesParameterValidation() { assertQueryFails( - "ALTER TABLE no_such_table_exists EXECUTE DELETE_ORPHAN_FILES", + "ALTER TABLE no_such_table_exists EXECUTE REMOVE_ORPHAN_FILES", "\\Qline 1:1: Table 'iceberg.tpch.no_such_table_exists' does not exist"); assertQueryFails( - "ALTER TABLE nation EXECUTE DELETE_ORPHAN_FILES (retention_threshold => '33')", - "\\QUnable to set catalog 'iceberg' table procedure 'DELETE_ORPHAN_FILES' property 'retention_threshold' to ['33']: duration is not a valid data duration string: 33"); + "ALTER TABLE nation EXECUTE REMOVE_ORPHAN_FILES (retention_threshold => '33')", + "\\QUnable to set catalog 'iceberg' table procedure 'REMOVE_ORPHAN_FILES' property 'retention_threshold' to ['33']: duration is not a valid data duration string: 33"); assertQueryFails( - "ALTER TABLE nation EXECUTE DELETE_ORPHAN_FILES (retention_threshold => '33mb')", - "\\QUnable to set catalog 'iceberg' table procedure 'DELETE_ORPHAN_FILES' property 'retention_threshold' to ['33mb']: Unknown time unit: mb"); + "ALTER TABLE nation EXECUTE REMOVE_ORPHAN_FILES (retention_threshold => '33mb')", + "\\QUnable to set catalog 'iceberg' table procedure 'REMOVE_ORPHAN_FILES' property 'retention_threshold' to ['33mb']: Unknown time unit: mb"); assertQueryFails( - "ALTER TABLE nation EXECUTE DELETE_ORPHAN_FILES (retention_threshold => '33s')", - "\\QRetention specified (33.00s) is shorter than the minimum retention configured in the system (7.00d). Minimum retention can be changed with iceberg.delete_orphan_files.min-retention configuration property or iceberg.delete_orphan_files_min_retention session property"); + "ALTER TABLE nation EXECUTE REMOVE_ORPHAN_FILES (retention_threshold => '33s')", + "\\QRetention specified (33.00s) is shorter than the minimum retention configured in the system (7.00d). Minimum retention can be changed with iceberg.remove_orphan_files.min-retention configuration property or iceberg.remove_orphan_files_min_retention session property"); } @Test @@ -3486,7 +3486,7 @@ private Session prepareCleanUpSession() { return Session.builder(getSession()) .setCatalogSessionProperty("iceberg", "expire_snapshots_min_retention", "0s") - .setCatalogSessionProperty("iceberg", "delete_orphan_files_min_retention", "0s") + .setCatalogSessionProperty("iceberg", "remove_orphan_files_min_retention", "0s") .build(); } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConfig.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConfig.java index 65f3a31f4f95..b4e653be808d 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConfig.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConfig.java @@ -50,7 +50,7 @@ public void testDefaults() .setHiveCatalogName(null) .setFormatVersion(2) .setExpireSnapshotsMinRetention(new Duration(7, DAYS)) - .setDeleteOrphanFilesMinRetention(new Duration(7, DAYS))); + .setRemoveOrphanFilesMinRetention(new Duration(7, DAYS))); } @Test @@ -69,7 +69,7 @@ public void testExplicitPropertyMappings() .put("iceberg.hive-catalog-name", "hive") .put("iceberg.format-version", "1") .put("iceberg.expire_snapshots.min-retention", "13h") - .put("iceberg.delete_orphan_files.min-retention", "14h") + .put("iceberg.remove_orphan_files.min-retention", "14h") .buildOrThrow(); IcebergConfig expected = new IcebergConfig() @@ -85,7 +85,7 @@ public void testExplicitPropertyMappings() .setHiveCatalogName("hive") .setFormatVersion(1) .setExpireSnapshotsMinRetention(new Duration(13, HOURS)) - .setDeleteOrphanFilesMinRetention(new Duration(14, HOURS)); + .setRemoveOrphanFilesMinRetention(new Duration(14, HOURS)); assertFullMapping(properties, expected); } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergSparkCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergSparkCompatibility.java index 2e9b18a7d8b5..421dd7b51123 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergSparkCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergSparkCompatibility.java @@ -1721,9 +1721,9 @@ public void testSparkReadsTrinoTableAfterCleaningUp(StorageFormat storageFormat) int initialNumberOfMetadataFiles = calculateMetadataFilesForPartitionedTable(baseTableName); onTrino().executeQuery("SET SESSION iceberg.expire_snapshots_min_retention = '0s'"); - onTrino().executeQuery("SET SESSION iceberg.delete_orphan_files_min_retention = '0s'"); + onTrino().executeQuery("SET SESSION iceberg.remove_orphan_files_min_retention = '0s'"); onTrino().executeQuery(format("ALTER TABLE %s EXECUTE EXPIRE_SNAPSHOTS (retention_threshold => '0s')", trinoTableName)); - onTrino().executeQuery(format("ALTER TABLE %s EXECUTE DELETE_ORPHAN_FILES (retention_threshold => '0s')", trinoTableName)); + onTrino().executeQuery(format("ALTER TABLE %s EXECUTE REMOVE_ORPHAN_FILES (retention_threshold => '0s')", trinoTableName)); int updatedNumberOfMetadataFiles = calculateMetadataFilesForPartitionedTable(baseTableName); Assertions.assertThat(updatedNumberOfMetadataFiles).isLessThan(initialNumberOfMetadataFiles); @@ -1763,9 +1763,9 @@ public void testSparkReadsTrinoTableAfterOptimizeAndCleaningUp(StorageFormat sto onTrino().executeQuery(format("ALTER TABLE %s EXECUTE OPTIMIZE", trinoTableName)); onTrino().executeQuery("SET SESSION iceberg.expire_snapshots_min_retention = '0s'"); - onTrino().executeQuery("SET SESSION iceberg.delete_orphan_files_min_retention = '0s'"); + onTrino().executeQuery("SET SESSION iceberg.remove_orphan_files_min_retention = '0s'"); onTrino().executeQuery(format("ALTER TABLE %s EXECUTE EXPIRE_SNAPSHOTS (retention_threshold => '0s')", trinoTableName)); - onTrino().executeQuery(format("ALTER TABLE %s EXECUTE DELETE_ORPHAN_FILES (retention_threshold => '0s')", trinoTableName)); + onTrino().executeQuery(format("ALTER TABLE %s EXECUTE REMOVE_ORPHAN_FILES (retention_threshold => '0s')", trinoTableName)); int updatedNumberOfFiles = onTrino().executeQuery(format("SELECT * FROM iceberg.default.\"%s$files\"", baseTableName)).getRowsCount(); Assertions.assertThat(updatedNumberOfFiles).isLessThan(initialNumberOfFiles); @@ -1800,9 +1800,9 @@ public void testTrinoReadsTrinoTableWithSparkDeletesAfterOptimizeAndCleanUp(Stor onTrino().executeQuery(format("ALTER TABLE %s EXECUTE OPTIMIZE", trinoTableName)); onTrino().executeQuery("SET SESSION iceberg.expire_snapshots_min_retention = '0s'"); - onTrino().executeQuery("SET SESSION iceberg.delete_orphan_files_min_retention = '0s'"); + onTrino().executeQuery("SET SESSION iceberg.remove_orphan_files_min_retention = '0s'"); onTrino().executeQuery(format("ALTER TABLE %s EXECUTE EXPIRE_SNAPSHOTS (retention_threshold => '0s')", trinoTableName)); - onTrino().executeQuery(format("ALTER TABLE %s EXECUTE DELETE_ORPHAN_FILES (retention_threshold => '0s')", trinoTableName)); + onTrino().executeQuery(format("ALTER TABLE %s EXECUTE REMOVE_ORPHAN_FILES (retention_threshold => '0s')", trinoTableName)); Row row = row(3008); String selectByString = "SELECT SUM(_bigint) FROM %s WHERE _string = 'a'"; @@ -1842,8 +1842,8 @@ public void testCleaningUpIcebergTableWithRowLevelDeletes(StorageFormat tableSto onTrino().executeQuery("SET SESSION iceberg.expire_snapshots_min_retention = '0s'"); onTrino().executeQuery(format("ALTER TABLE %s EXECUTE EXPIRE_SNAPSHOTS (retention_threshold => '0s')", trinoTableName)); - onTrino().executeQuery("SET SESSION iceberg.delete_orphan_files_min_retention = '0s'"); - onTrino().executeQuery(format("ALTER TABLE %s EXECUTE DELETE_ORPHAN_FILES (retention_threshold => '0s')", trinoTableName)); + onTrino().executeQuery("SET SESSION iceberg.remove_orphan_files_min_retention = '0s'"); + onTrino().executeQuery(format("ALTER TABLE %s EXECUTE REMOVE_ORPHAN_FILES (retention_threshold => '0s')", trinoTableName)); assertThat(onTrino().executeQuery(format(selectByString, trinoTableName))) .containsOnly(row);