From dbd14f09d76778b0f68802cfb416ce17f73c55b9 Mon Sep 17 00:00:00 2001 From: zhangchen Date: Thu, 23 Dec 2021 22:47:31 +0800 Subject: [PATCH 1/2] Spark 3.0: Fix delete from snapshot of table --- .../main/java/org/apache/iceberg/spark/source/SparkTable.java | 1 + .../test/java/org/apache/iceberg/spark/sql/TestDeleteFrom.java | 1 + 2 files changed, 2 insertions(+) diff --git a/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java b/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java index 4031b89218d0..4fca26779000 100644 --- a/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java +++ b/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java @@ -244,6 +244,7 @@ private boolean requiresRewrite(Filter filter, Schema schema, Set ident @Override public void deleteWhere(Filter[] filters) { + canDeleteWhere(filters); Expression deleteExpr = SparkFilters.convert(filters); if (deleteExpr == Expressions.alwaysFalse()) { diff --git a/spark/v3.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestDeleteFrom.java b/spark/v3.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestDeleteFrom.java index cce987a781c0..75b7653fdb46 100644 --- a/spark/v3.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestDeleteFrom.java +++ b/spark/v3.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestDeleteFrom.java @@ -72,6 +72,7 @@ public void testDeleteFromUnpartitionedTable() throws NoSuchTableException { 0L, scalarSql("SELECT count(1) FROM %s", tableName)); } + @Test public void testDeleteFromTableAtSnapshot() throws NoSuchTableException { Assume.assumeFalse( "Spark session catalog does not support extended table names", From 97319bae46e3e63f795cb4ef07e83a47d79b9b5a Mon Sep 17 00:00:00 2001 From: zhangchen351 Date: Wed, 5 Jan 2022 02:25:19 +0800 Subject: [PATCH 2/2] Spark 3.0: Fix delete from snapshot of table --- .../java/org/apache/iceberg/spark/source/SparkTable.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java b/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java index 4fca26779000..2db252decbf0 100644 --- a/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java +++ b/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java @@ -244,7 +244,10 @@ private boolean requiresRewrite(Filter filter, Schema schema, Set ident @Override public void deleteWhere(Filter[] filters) { - canDeleteWhere(filters); + Preconditions.checkArgument( + snapshotId == null, + "Cannot delete from table at a specific snapshot: %s", snapshotId); + Expression deleteExpr = SparkFilters.convert(filters); if (deleteExpr == Expressions.alwaysFalse()) {