diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index 4faac22a841fe..bb65375834cd4 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -774,11 +774,15 @@ private Set getMetadataPartitionsToUpdate() { // fetch partitions to update from table config Set partitionsToUpdate = getCompletedMetadataPartitions(dataMetaClient.getTableConfig()); // add inflight indexes as well because the file groups have already been initialized, so writers can log updates + // NOTE: Async HoodieIndexer can move some partition to inflight. While that partition is still being built, + // the regular ingestion writers should not be blocked. They can go ahead and log updates to the metadata partition. + // Instead of depending on enabledPartitionTypes, the table config becomes the source of truth for which partitions to update. partitionsToUpdate.addAll(getInflightMetadataPartitions(dataMetaClient.getTableConfig())); if (!partitionsToUpdate.isEmpty()) { return partitionsToUpdate; } // fallback to all enabled partitions if table config returned no partitions + LOG.warn("There are no partitions to update according to table config. Falling back to enabled partition types in the write config."); return getEnabledPartitionTypes().stream().map(MetadataPartitionType::getPartitionPath).collect(Collectors.toSet()); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java index 38b5509cd577f..b2a201eb72496 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java @@ -701,6 +701,8 @@ public static class PropertyBuilder { private HoodieTimelineTimeZone commitTimeZone; private Boolean partitionMetafileUseBaseFormat; private Boolean dropPartitionColumnsWhenWrite; + private String metadataPartitions; + private String inflightMetadataPartitions; /** * Persist the configs that is written at the first time, and should not be changed. @@ -825,6 +827,16 @@ public PropertyBuilder setDropPartitionColumnsWhenWrite(Boolean dropPartitionCol return this; } + public PropertyBuilder setMetadataPartitions(String partitions) { + this.metadataPartitions = partitions; + return this; + } + + public PropertyBuilder setInflightMetadataPartitions(String partitions) { + this.inflightMetadataPartitions = partitions; + return this; + } + public PropertyBuilder set(String key, Object value) { if (HoodieTableConfig.PERSISTED_CONFIG_LIST.contains(key)) { this.others.put(key, value); @@ -927,6 +939,14 @@ public PropertyBuilder fromProperties(Properties properties) { if (hoodieConfig.contains(HoodieTableConfig.DROP_PARTITION_COLUMNS)) { setDropPartitionColumnsWhenWrite(hoodieConfig.getBoolean(HoodieTableConfig.DROP_PARTITION_COLUMNS)); } + + if (hoodieConfig.contains(HoodieTableConfig.TABLE_METADATA_PARTITIONS)) { + setMetadataPartitions(hoodieConfig.getString(HoodieTableConfig.TABLE_METADATA_PARTITIONS)); + } + + if (hoodieConfig.contains(HoodieTableConfig.TABLE_METADATA_PARTITIONS_INFLIGHT)) { + setInflightMetadataPartitions(hoodieConfig.getString(HoodieTableConfig.TABLE_METADATA_PARTITIONS_INFLIGHT)); + } return this; } @@ -1012,6 +1032,14 @@ public Properties build() { if (null != dropPartitionColumnsWhenWrite) { tableConfig.setValue(HoodieTableConfig.DROP_PARTITION_COLUMNS, Boolean.toString(dropPartitionColumnsWhenWrite)); } + + if (null != metadataPartitions) { + tableConfig.setValue(HoodieTableConfig.TABLE_METADATA_PARTITIONS, metadataPartitions); + } + + if (null != inflightMetadataPartitions) { + tableConfig.setValue(HoodieTableConfig.TABLE_METADATA_PARTITIONS_INFLIGHT, inflightMetadataPartitions); + } return tableConfig.getProps(); }