From e2cd3fc8abd7209c2dfc7379c551aeb969034358 Mon Sep 17 00:00:00 2001 From: xiaoxingstack Date: Mon, 19 Sep 2022 18:43:52 +0800 Subject: [PATCH 1/2] [HUDI-4877] Fix org.apache.hudi.index.bucket.TestHoodieSimpleBucketIndex#testTagLocation always passed whether org.apache.hudi.index.bucket.HoodieSimpleBucketIndex#loadPartitionBucketIdFileIdMapping is by fileSlice or baseFile --- .../hudi/index/bucket/TestHoodieSimpleBucketIndex.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bucket/TestHoodieSimpleBucketIndex.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bucket/TestHoodieSimpleBucketIndex.java index ea6418696c694..ebe330858f1e6 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bucket/TestHoodieSimpleBucketIndex.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bucket/TestHoodieSimpleBucketIndex.java @@ -127,10 +127,12 @@ public void testTagLocation(boolean isInsert) throws Exception { testTable.addCommit("001").withInserts("2016/01/31", getRecordFileId(record1), record1); testTable.addCommit("002").withInserts("2016/01/31", getRecordFileId(record2), record2); testTable.addCommit("003").withInserts("2016/01/31", getRecordFileId(record3), record3); + testTable.addCommit("004").withInserts("2015/01/31", getRecordFileId(record4), record4); } else { testTable.addCommit("001").withLogAppends("2016/01/31", getRecordFileId(record1), record1); testTable.addCommit("002").withLogAppends("2016/01/31", getRecordFileId(record2), record2); testTable.addCommit("003").withLogAppends("2016/01/31", getRecordFileId(record3), record3); + testTable.addCommit("004").withLogAppends("2015/01/31", getRecordFileId(record4), record4); } taggedRecordRDD = bucketIndex.tagLocation(HoodieJavaRDD.of(recordRDD), context, @@ -138,8 +140,7 @@ public void testTagLocation(boolean isInsert) throws Exception { assertFalse(taggedRecordRDD.collectAsList().stream().filter(r -> r.isCurrentLocationKnown()) .filter(r -> BucketIdentifier.bucketIdFromFileId(r.getCurrentLocation().getFileId()) != getRecordBucketId(r)).findAny().isPresent()); - assertTrue(taggedRecordRDD.collectAsList().stream().filter(r -> r.getPartitionPath().equals("2015/01/31") - && !r.isCurrentLocationKnown()).count() == 1L); + assertTrue(taggedRecordRDD.collectAsList().stream().allMatch(r -> r.isCurrentLocationKnown())); } private HoodieWriteConfig makeConfig() { From 668b55c8b04a755f1a7b3d135bc8acea8c04e844 Mon Sep 17 00:00:00 2001 From: xiaoxingstack Date: Mon, 19 Sep 2022 19:01:11 +0800 Subject: [PATCH 2/2] correct --- .../hudi/index/bucket/TestHoodieSimpleBucketIndex.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bucket/TestHoodieSimpleBucketIndex.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bucket/TestHoodieSimpleBucketIndex.java index ebe330858f1e6..34728c6cf368b 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bucket/TestHoodieSimpleBucketIndex.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bucket/TestHoodieSimpleBucketIndex.java @@ -127,12 +127,10 @@ public void testTagLocation(boolean isInsert) throws Exception { testTable.addCommit("001").withInserts("2016/01/31", getRecordFileId(record1), record1); testTable.addCommit("002").withInserts("2016/01/31", getRecordFileId(record2), record2); testTable.addCommit("003").withInserts("2016/01/31", getRecordFileId(record3), record3); - testTable.addCommit("004").withInserts("2015/01/31", getRecordFileId(record4), record4); } else { testTable.addCommit("001").withLogAppends("2016/01/31", getRecordFileId(record1), record1); testTable.addCommit("002").withLogAppends("2016/01/31", getRecordFileId(record2), record2); testTable.addCommit("003").withLogAppends("2016/01/31", getRecordFileId(record3), record3); - testTable.addCommit("004").withLogAppends("2015/01/31", getRecordFileId(record4), record4); } taggedRecordRDD = bucketIndex.tagLocation(HoodieJavaRDD.of(recordRDD), context, @@ -140,7 +138,10 @@ public void testTagLocation(boolean isInsert) throws Exception { assertFalse(taggedRecordRDD.collectAsList().stream().filter(r -> r.isCurrentLocationKnown()) .filter(r -> BucketIdentifier.bucketIdFromFileId(r.getCurrentLocation().getFileId()) != getRecordBucketId(r)).findAny().isPresent()); - assertTrue(taggedRecordRDD.collectAsList().stream().allMatch(r -> r.isCurrentLocationKnown())); + assertTrue(taggedRecordRDD.collectAsList().stream().filter(r -> r.getPartitionPath().equals("2015/01/31") + && !r.isCurrentLocationKnown()).count() == 1L); + assertTrue(taggedRecordRDD.collectAsList().stream().filter(r -> r.getPartitionPath().equals("2016/01/31") + && r.isCurrentLocationKnown()).count() == 3L); } private HoodieWriteConfig makeConfig() {