diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index eee676822a8aa..987583f3a708a 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -18,7 +18,6 @@ package org.apache.hudi.metadata; -import org.apache.avro.specific.SpecificRecordBase; import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.avro.model.HoodieInstantInfo; import org.apache.hudi.avro.model.HoodieMetadataRecord; @@ -57,13 +56,14 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.config.HoodieCompactionConfig; -import org.apache.hudi.config.metrics.HoodieMetricsConfig; import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.config.metrics.HoodieMetricsConfig; import org.apache.hudi.config.metrics.HoodieMetricsGraphiteConfig; import org.apache.hudi.config.metrics.HoodieMetricsJmxConfig; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieMetadataException; +import org.apache.avro.specific.SpecificRecordBase; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -367,8 +367,14 @@ protected void bootstrapIfNeeded(HoodieEngineCont // If the un-synced instants have been archived, then // the metadata table will need to be bootstrapped again. if (exists) { - final HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setConf(hadoopConf.get()) + HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setConf(hadoopConf.get()) .setBasePath(metadataWriteConfig.getBasePath()).build(); + + if (dataWriteConfig.getMetadataConfig().populateMetaFields() != metadataMetaClient.getTableConfig().populateMetaFields()) { + LOG.info("Re-initiating metadata table properties since populate meta fields have changed"); + metadataMetaClient = initializeMetaClient(dataWriteConfig.getMetadataConfig().populateMetaFields()); + } + final Option latestMetadataInstant = metadataMetaClient.getActiveTimeline().filterCompletedInstants().lastInstant(); @@ -504,17 +510,7 @@ private boolean bootstrapFromFilesystem(HoodieEngineContext engineContext, Hoodi .getReverseOrderedInstants().findFirst().map(HoodieInstant::getTimestamp).orElse(SOLO_COMMIT_TIMESTAMP); LOG.info("Creating a new metadata table in " + metadataWriteConfig.getBasePath() + " at instant " + createInstantTime); - HoodieTableMetaClient.withPropertyBuilder() - .setTableType(HoodieTableType.MERGE_ON_READ) - .setTableName(tableName) - .setArchiveLogFolder(ARCHIVELOG_FOLDER.defaultValue()) - .setPayloadClassName(HoodieMetadataPayload.class.getName()) - .setBaseFileFormat(HoodieFileFormat.HFILE.toString()) - .setRecordKeyFields(RECORD_KEY_FIELD_NAME) - .setPopulateMetaFields(dataWriteConfig.getMetadataConfig().populateMetaFields()) - .setKeyGeneratorClassProp(HoodieTableMetadataKeyGenerator.class.getCanonicalName()) - .initTable(hadoopConf.get(), metadataWriteConfig.getBasePath()); - + initializeMetaClient(dataWriteConfig.getMetadataConfig().populateMetaFields()); initTableMetadata(); initializeEnabledFileGroups(dataMetaClient, createInstantTime); @@ -530,6 +526,19 @@ private boolean bootstrapFromFilesystem(HoodieEngineContext engineContext, Hoodi return true; } + private HoodieTableMetaClient initializeMetaClient(boolean populatMetaFields) throws IOException { + return HoodieTableMetaClient.withPropertyBuilder() + .setTableType(HoodieTableType.MERGE_ON_READ) + .setTableName(tableName) + .setArchiveLogFolder(ARCHIVELOG_FOLDER.defaultValue()) + .setPayloadClassName(HoodieMetadataPayload.class.getName()) + .setBaseFileFormat(HoodieFileFormat.HFILE.toString()) + .setRecordKeyFields(RECORD_KEY_FIELD_NAME) + .setPopulateMetaFields(populatMetaFields) + .setKeyGeneratorClassProp(HoodieTableMetadataKeyGenerator.class.getCanonicalName()) + .initTable(hadoopConf.get(), metadataWriteConfig.getBasePath()); + } + /** * Function to find hoodie partitions and list files in them in parallel. * diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java index 223625fe7e469..1c40bc808c0be 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java @@ -328,6 +328,36 @@ public void testMetadataInsertUpsertClean(HoodieTableType tableType) throws Exce validateMetadata(testTable, emptyList(), true); } + @Test + public void testUpdationOfPopulateMetaFieldsForMetadataTable() throws Exception { + tableType = COPY_ON_WRITE; + init(tableType, false); + + writeConfig = getWriteConfigBuilder(true, true, false) + .withMetadataConfig(HoodieMetadataConfig.newBuilder() + .enable(true) + .withPopulateMetaFields(true) + .build()) + .build(); + initWriteConfigAndMetatableWriter(writeConfig, true); + doWriteOperation(testTable, "0000001", INSERT); + + HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setBasePath(writeConfig.getBasePath() + "/.hoodie/metadata").setConf(hadoopConf).build(); + assertTrue(metaClient.getTableConfig().populateMetaFields()); + + // update populateMeta fields to false. + writeConfig = getWriteConfigBuilder(true, true, false) + .withMetadataConfig(HoodieMetadataConfig.newBuilder() + .enable(true) + .withPopulateMetaFields(false) + .build()) + .build(); + initWriteConfigAndMetatableWriter(writeConfig, true); + doWriteOperation(testTable, "0000002", INSERT); + metaClient = HoodieTableMetaClient.builder().setBasePath(writeConfig.getBasePath() + "/.hoodie/metadata").setConf(hadoopConf).build(); + assertFalse(metaClient.getTableConfig().populateMetaFields()); + } + @Test public void testMetadataInsertUpsertCleanNonPartitioned() throws Exception { HoodieTableType tableType = COPY_ON_WRITE;