diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java index 87dd26f0c6428..69cc25feb5590 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java @@ -1252,7 +1252,7 @@ private Map> generateBootstrapIndexAndSourceD assertTrue(new File(sourcePath.toString()).exists()); // recreate metaClient with Bootstrap base path - metaClient = HoodieTestUtils.init(basePath, getTableType(), sourcePath.toString()); + metaClient = HoodieTestUtils.init(basePath, getTableType(), sourcePath.toString(), true); // generate bootstrap index Map> bootstrapMapping = TestBootstrapIndex.generateBootstrapIndex(metaClient, sourcePath.toString(), diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java index 6d3a7d0015057..3d6290a085fc3 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java @@ -136,10 +136,10 @@ public class HoodieTableConfig extends HoodieConfig implements Serializable { .defaultValue("archived") .withDocumentation("path under the meta folder, to store archived timeline instants at."); - public static final ConfigProperty BOOTSTRAP_INDEX_ENABLE = ConfigProperty + public static final ConfigProperty BOOTSTRAP_INDEX_ENABLE = ConfigProperty .key("hoodie.bootstrap.index.enable") - .noDefaultValue() - .withDocumentation("Whether or not, this is a bootstrapped table, with bootstrap base data and an mapping index defined."); + .defaultValue(true) + .withDocumentation("Whether or not, this is a bootstrapped table, with bootstrap base data and an mapping index defined, default true."); public static final ConfigProperty BOOTSTRAP_INDEX_CLASS_NAME = ConfigProperty .key("hoodie.bootstrap.index.class") @@ -298,8 +298,9 @@ public String getBootstrapIndexClass() { } public static String getDefaultBootstrapIndexClass(Properties props) { + HoodieConfig hoodieConfig = new HoodieConfig(props); String defaultClass = BOOTSTRAP_INDEX_CLASS_NAME.defaultValue(); - if ("false".equalsIgnoreCase(props.getProperty(BOOTSTRAP_INDEX_ENABLE.key()))) { + if (!hoodieConfig.getBooleanOrDefault(BOOTSTRAP_INDEX_ENABLE)) { defaultClass = NO_OP_BOOTSTRAP_INDEX_CLASS; } return defaultClass; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java index 97464b1b28d1c..d5a3988fe983a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java @@ -627,6 +627,7 @@ public static class PropertyBuilder { private String partitionFields; private String bootstrapIndexClass; private String bootstrapBasePath; + private Boolean bootstrapIndexEnable; private Boolean populateMetaFields; private String keyGeneratorClassProp; @@ -702,6 +703,11 @@ public PropertyBuilder setBootstrapBasePath(String bootstrapBasePath) { return this; } + public PropertyBuilder setBootstrapIndexEnable(Boolean bootstrapIndexEnable) { + this.bootstrapIndexEnable = bootstrapIndexEnable; + return this; + } + public PropertyBuilder setPopulateMetaFields(boolean populateMetaFields) { this.populateMetaFields = populateMetaFields; return this; @@ -749,6 +755,11 @@ public PropertyBuilder fromProperties(Properties properties) { if (hoodieConfig.contains(HoodieTableConfig.BOOTSTRAP_BASE_PATH)) { setBootstrapBasePath(hoodieConfig.getString(HoodieTableConfig.BOOTSTRAP_BASE_PATH)); } + + if (hoodieConfig.contains(HoodieTableConfig.BOOTSTRAP_INDEX_ENABLE)) { + setBootstrapIndexEnable(hoodieConfig.getBoolean(HoodieTableConfig.BOOTSTRAP_INDEX_ENABLE)); + } + if (hoodieConfig.contains(HoodieTableConfig.PRECOMBINE_FIELD)) { setPreCombineField(hoodieConfig.getString(HoodieTableConfig.PRECOMBINE_FIELD)); } @@ -807,6 +818,10 @@ public Properties build() { tableConfig.setValue(HoodieTableConfig.BOOTSTRAP_INDEX_CLASS_NAME, bootstrapIndexClass); } + if (null != bootstrapIndexEnable) { + tableConfig.setValue(HoodieTableConfig.BOOTSTRAP_INDEX_ENABLE, Boolean.toString(bootstrapIndexEnable)); + } + if (null != bootstrapBasePath) { tableConfig.setValue(HoodieTableConfig.BOOTSTRAP_BASE_PATH, bootstrapBasePath); } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java index 37a7c91c811bd..45bf5a2e42ddc 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java @@ -107,7 +107,7 @@ public static Stream configParams() { @BeforeEach public void setup() throws IOException { - metaClient = HoodieTestUtils.init(tempDir.toAbsolutePath().toString(), getTableType(), BOOTSTRAP_SOURCE_PATH); + metaClient = HoodieTestUtils.init(tempDir.toAbsolutePath().toString(), getTableType(), BOOTSTRAP_SOURCE_PATH, false); basePath = metaClient.getBasePath(); refreshFsView(); } @@ -303,6 +303,11 @@ private void checkExternalFile(HoodieFileStatus srcFileStatus, Option protected void testViewForFileSlicesWithAsyncCompaction(boolean skipCreatingDataFile, boolean isCompactionInFlight, int expTotalFileSlices, int expTotalDataFiles, boolean includeInvalidAndInflight, boolean testBootstrap) throws Exception { + + if (testBootstrap) { + metaClient = HoodieTestUtils.init(tempDir.toAbsolutePath().toString(), getTableType(), BOOTSTRAP_SOURCE_PATH, testBootstrap); + } + String partitionPath = "2016/05/01"; new File(basePath + "/" + partitionPath).mkdirs(); String fileId = UUID.randomUUID().toString(); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java index bc1c18a7913be..259d0c2503f46 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java @@ -63,9 +63,10 @@ public static HoodieTableMetaClient init(String basePath, HoodieTableType tableT return init(getDefaultHadoopConf(), basePath, tableType); } - public static HoodieTableMetaClient init(String basePath, HoodieTableType tableType, String bootstrapBasePath) throws IOException { + public static HoodieTableMetaClient init(String basePath, HoodieTableType tableType, String bootstrapBasePath, boolean bootstrapIndexEnable) throws IOException { Properties props = new Properties(); props.setProperty(HoodieTableConfig.BOOTSTRAP_BASE_PATH.key(), bootstrapBasePath); + props.put(HoodieTableConfig.BOOTSTRAP_INDEX_ENABLE.key(), bootstrapIndexEnable); return init(getDefaultHadoopConf(), basePath, tableType, props); } diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java index 5cbcf6cf87279..59671322b0652 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java @@ -195,9 +195,9 @@ private enum EffectiveMode { private void testBootstrapCommon(boolean partitioned, boolean deltaCommit, EffectiveMode mode) throws Exception { if (deltaCommit) { - metaClient = HoodieTestUtils.init(basePath, HoodieTableType.MERGE_ON_READ, bootstrapBasePath); + metaClient = HoodieTestUtils.init(basePath, HoodieTableType.MERGE_ON_READ, bootstrapBasePath, true); } else { - metaClient = HoodieTestUtils.init(basePath, HoodieTableType.COPY_ON_WRITE, bootstrapBasePath); + metaClient = HoodieTestUtils.init(basePath, HoodieTableType.COPY_ON_WRITE, bootstrapBasePath, true); } int totalRecords = 100;