From e28f1f7cc461c327254bbf7c78e4e01985abcf11 Mon Sep 17 00:00:00 2001 From: Alexey Kudinkin Date: Wed, 1 Dec 2021 19:06:39 -0800 Subject: [PATCH] Fixed partitions produced by layout optimization in case order-by key is composed of a single column --- .../src/main/java/org/apache/spark/OrderingIndexHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/spark/OrderingIndexHelper.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/spark/OrderingIndexHelper.java index 67b1c672ec86b..3d0138e9b3810 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/spark/OrderingIndexHelper.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/spark/OrderingIndexHelper.java @@ -100,7 +100,7 @@ public static Dataset createOptimizedDataFrameByMapValue(Dataset df, L } // only one col to sort, no need to use z-order if (sortCols.size() == 1) { - return df.repartitionByRange(fieldNum, org.apache.spark.sql.functions.col(sortCols.get(0))); + return df.repartitionByRange(fileNum, org.apache.spark.sql.functions.col(sortCols.get(0))); } Map fieldMap = sortCols .stream().collect(Collectors.toMap(e -> Arrays.asList(df.schema().fields()).indexOf(columnsMap.get(e)), e -> columnsMap.get(e)));