diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteDataFilesSparkAction.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteDataFilesSparkAction.java index e04a0c88b4bb..cf570dbe929f 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteDataFilesSparkAction.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteDataFilesSparkAction.java @@ -377,7 +377,8 @@ private Builder doExecuteWithPartialProgress( // stop commit service commitService.close(); - int failedCommits = maxCommits - commitService.succeededCommits(); + int totalCommits = groupsPerCommit == 1 ? ctx.totalGroupCount() : maxCommits; + int failedCommits = totalCommits - commitService.succeededCommits(); if (failedCommits > 0 && failedCommits <= maxFailedCommits) { LOG.warn( "{} is true but {} rewrite commits failed. Check the logs to determine why the individual " diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java index bdbb8c176812..95ca621e4671 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java @@ -1320,6 +1320,33 @@ public void testParallelPartialProgressWithMaxFailedCommits() { shouldHaveACleanCache(table); } + @TestTemplate + public void testParallelPartialProgressWithMaxFailedCommitsLargerThanTotalFileGroup() { + Table table = createTable(20); + int fileSize = averageFileSize(table); + + List originalData = currentData(); + + RewriteDataFilesSparkAction rewrite = + basicRewrite(table) + .option( + RewriteDataFiles.MAX_FILE_GROUP_SIZE_BYTES, Integer.toString(fileSize * 2 + 1000)) + .option(RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_REWRITES, "3") + .option(RewriteDataFiles.PARTIAL_PROGRESS_ENABLED, "true") + .option(RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS, "20") + .option(RewriteDataFiles.PARTIAL_PROGRESS_MAX_FAILED_COMMITS, "0"); + rewrite.execute(); + + table.refresh(); + + List postRewriteData = currentData(); + assertEquals("We shouldn't have changed the data", originalData, postRewriteData); + // With 10 original groups and max commits of 20, we have 1 group per commit. + shouldHaveSnapshots(table, 11); + shouldHaveNoOrphans(table); + shouldHaveACleanCache(table); + } + @TestTemplate public void testInvalidOptions() { Table table = createTable(20);