From 2ecff3384cf6d3a1d42cdd55bd82253c3a7a7632 Mon Sep 17 00:00:00 2001 From: Rajarshi Sarkar Date: Thu, 2 Jun 2022 15:15:33 +0530 Subject: [PATCH 1/3] Use Table Partitioning sort with manual Sort Order in Rewrite Data Files --- .../src/main/java/org/apache/iceberg/actions/SortStrategy.java | 3 ++- .../test/java/org/apache/iceberg/actions/TestSortStrategy.java | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/actions/SortStrategy.java b/core/src/main/java/org/apache/iceberg/actions/SortStrategy.java index 4a650e5bfcf7..0680765418d4 100644 --- a/core/src/main/java/org/apache/iceberg/actions/SortStrategy.java +++ b/core/src/main/java/org/apache/iceberg/actions/SortStrategy.java @@ -24,6 +24,7 @@ import org.apache.iceberg.SortOrder; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.util.SortOrderUtil; /** * A rewrite strategy for data files which aims to reorder data with data files to optimally lay them out @@ -48,7 +49,7 @@ public abstract class SortStrategy extends BinPackStrategy { * @return this for method chaining */ public SortStrategy sortOrder(SortOrder order) { - this.sortOrder = order; + this.sortOrder = SortOrderUtil.buildSortOrder(table(), order); return this; } diff --git a/core/src/test/java/org/apache/iceberg/actions/TestSortStrategy.java b/core/src/test/java/org/apache/iceberg/actions/TestSortStrategy.java index 373728fcb3b1..63861f099607 100644 --- a/core/src/test/java/org/apache/iceberg/actions/TestSortStrategy.java +++ b/core/src/test/java/org/apache/iceberg/actions/TestSortStrategy.java @@ -86,9 +86,6 @@ private List tasksForSortOrder(int sortOrderId, int... fileSizesMB @Test public void testInvalidSortOrder() { - AssertHelpers.assertThrows("Should not allow an unsorted Sort order", IllegalArgumentException.class, - () -> defaultSort().sortOrder(SortOrder.unsorted()).options(Collections.emptyMap())); - AssertHelpers.assertThrows("Should not allow a Sort order with bad columns", ValidationException.class, () -> { Schema badSchema = new Schema( From 0fa6ac3ebe51bb811cdff77a362bd314fbccb4fd Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Sun, 3 Jul 2022 16:27:31 -0700 Subject: [PATCH 2/3] Add argument check for unsorted order. --- .../src/main/java/org/apache/iceberg/actions/SortStrategy.java | 1 + .../test/java/org/apache/iceberg/actions/TestSortStrategy.java | 3 +++ 2 files changed, 4 insertions(+) diff --git a/core/src/main/java/org/apache/iceberg/actions/SortStrategy.java b/core/src/main/java/org/apache/iceberg/actions/SortStrategy.java index 0680765418d4..0fe957692a04 100644 --- a/core/src/main/java/org/apache/iceberg/actions/SortStrategy.java +++ b/core/src/main/java/org/apache/iceberg/actions/SortStrategy.java @@ -49,6 +49,7 @@ public abstract class SortStrategy extends BinPackStrategy { * @return this for method chaining */ public SortStrategy sortOrder(SortOrder order) { + Preconditions.checkArgument(!order.isUnsorted(), "Invalid sort order for sort strategy: unsorted"); this.sortOrder = SortOrderUtil.buildSortOrder(table(), order); return this; } diff --git a/core/src/test/java/org/apache/iceberg/actions/TestSortStrategy.java b/core/src/test/java/org/apache/iceberg/actions/TestSortStrategy.java index 63861f099607..373728fcb3b1 100644 --- a/core/src/test/java/org/apache/iceberg/actions/TestSortStrategy.java +++ b/core/src/test/java/org/apache/iceberg/actions/TestSortStrategy.java @@ -86,6 +86,9 @@ private List tasksForSortOrder(int sortOrderId, int... fileSizesMB @Test public void testInvalidSortOrder() { + AssertHelpers.assertThrows("Should not allow an unsorted Sort order", IllegalArgumentException.class, + () -> defaultSort().sortOrder(SortOrder.unsorted()).options(Collections.emptyMap())); + AssertHelpers.assertThrows("Should not allow a Sort order with bad columns", ValidationException.class, () -> { Schema badSchema = new Schema( From f942e659bf096e714c98cecfd199b1c1b42e3078 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Mon, 4 Jul 2022 11:41:52 -0700 Subject: [PATCH 3/3] Fix error message. --- core/src/main/java/org/apache/iceberg/actions/SortStrategy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/iceberg/actions/SortStrategy.java b/core/src/main/java/org/apache/iceberg/actions/SortStrategy.java index 0fe957692a04..96914de09525 100644 --- a/core/src/main/java/org/apache/iceberg/actions/SortStrategy.java +++ b/core/src/main/java/org/apache/iceberg/actions/SortStrategy.java @@ -49,7 +49,7 @@ public abstract class SortStrategy extends BinPackStrategy { * @return this for method chaining */ public SortStrategy sortOrder(SortOrder order) { - Preconditions.checkArgument(!order.isUnsorted(), "Invalid sort order for sort strategy: unsorted"); + Preconditions.checkArgument(!order.isUnsorted(), "Cannot set strategy sort order: unsorted"); this.sortOrder = SortOrderUtil.buildSortOrder(table(), order); return this; }