From 518cf04e19bc2bb99f68f0a3dd34168a4bb892bc Mon Sep 17 00:00:00 2001 From: peterxcli Date: Mon, 12 May 2025 16:24:54 +0800 Subject: [PATCH 01/31] Support cacheIterator method with startKey for full table cache --- .../main/java/org/apache/hadoop/hdds/utils/db/Table.java | 8 ++++++++ .../java/org/apache/hadoop/hdds/utils/db/TypedTable.java | 5 +++++ .../apache/hadoop/hdds/utils/db/cache/FullTableCache.java | 8 +++++++- .../hadoop/hdds/utils/db/cache/PartialTableCache.java | 6 ++++++ .../org/apache/hadoop/hdds/utils/db/cache/TableCache.java | 2 ++ .../apache/hadoop/hdds/utils/db/cache/TableNoCache.java | 5 +++++ .../org/apache/hadoop/ozone/om/OMMetadataManager.java | 3 +++ .../org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 6 ++++++ 8 files changed, 42 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java index 9552f327e2e8..dc15c950f501 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java @@ -231,6 +231,14 @@ default void cleanupCache(List epochs) { throw new NotImplementedException("cacheIterator is not implemented"); } + /** + * Return cache iterator maintained for this table. + */ + default Iterator, CacheValue>> + cacheIterator(KEY startKey) { + throw new NotImplementedException("cacheIterator with startKey is not implemented"); + } + /** * Create the metrics datasource that emits table cache metrics. */ diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java index dcf482aad2ad..41e2c2330ee2 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java @@ -459,6 +459,11 @@ public Iterator, CacheValue>> cacheIterator() { return cache.iterator(); } + @Override + public Iterator, CacheValue>> cacheIterator(KEY startKey) { + return cache.iterator(startKey); + } + @Override public TableCacheMetrics createCacheMetrics() { return TableCacheMetrics.create(cache, getName()); diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/FullTableCache.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/FullTableCache.java index 2f60d0cf4d24..e0f19b4d62d3 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/FullTableCache.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/FullTableCache.java @@ -53,7 +53,7 @@ public class FullTableCache implements TableCache { private static final Logger LOG = LoggerFactory.getLogger(FullTableCache.class); - private final Map, CacheValue> cache; + private final NavigableMap, CacheValue> cache; private final NavigableMap>> epochEntries; private final ScheduledExecutorService executorService; private final Queue epochCleanupQueue = new ConcurrentLinkedQueue<>(); @@ -150,6 +150,12 @@ public Iterator, CacheValue>> iterator() { return cache.entrySet().iterator(); } + @Override + public Iterator, CacheValue>> iterator(KEY startKey) { + statsRecorder.recordIteration(); + return cache.tailMap(new CacheKey<>(startKey)).entrySet().iterator(); + } + @VisibleForTesting @Override public void evictCache(List epochs) { diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/PartialTableCache.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/PartialTableCache.java index 9b7240364947..0c9fc528504e 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/PartialTableCache.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/PartialTableCache.java @@ -19,6 +19,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -117,6 +118,11 @@ public Iterator, CacheValue>> iterator() { return cache.entrySet().iterator(); } + @Override + public Iterator, CacheValue>> iterator(KEY startKey) { + return Collections.emptyIterator(); + } + @VisibleForTesting @Override public void evictCache(List epochs) { diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/TableCache.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/TableCache.java index 5d3782f76e16..24bdfefef7ae 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/TableCache.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/TableCache.java @@ -78,6 +78,8 @@ public interface TableCache { */ Iterator, CacheValue>> iterator(); + Iterator, CacheValue>> iterator(KEY startKey); + /** * Check key exist in cache or not. * diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/TableNoCache.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/TableNoCache.java index b3ec98619ce6..90adf09e3946 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/TableNoCache.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/TableNoCache.java @@ -73,6 +73,11 @@ public Iterator, CacheValue>> iterator() { return Collections.emptyIterator(); } + @Override + public Iterator, CacheValue>> iterator(KEY startKey) { + return Collections.emptyIterator(); + } + @VisibleForTesting @Override public void evictCache(List epochs) { diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index ec9e34cec720..b43392af5c12 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -533,6 +533,9 @@ List getMultipartUploadKeys(String volumeName, Iterator, CacheValue>> getBucketIterator(); + Iterator, CacheValue>> + getBucketIterator(String startKey); + TableIterator> getKeyIterator() throws IOException; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index e1f50b1922f1..4a82046e96c5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -915,6 +915,12 @@ public List listBuckets(final String volumeName, return bucketTable.cacheIterator(); } + @Override + public Iterator, CacheValue>> + getBucketIterator(String startKey) { + return bucketTable.cacheIterator(startKey); + } + @Override public TableIterator> getKeyIterator() throws IOException { From 3c987ed6acb0a3e93962b0abb6f806028b4714f8 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Mon, 12 May 2025 16:26:01 +0800 Subject: [PATCH 02/31] Support CF key range compaction and SST properties retrieval for rocksDB --- .../apache/hadoop/hdds/utils/db/DBStore.java | 30 ++++++++++ .../apache/hadoop/hdds/utils/db/KeyRange.java | 57 +++++++++++++++++++ .../apache/hadoop/hdds/utils/db/RDBStore.java | 52 ++++++++++++++++- .../hadoop/hdds/utils/db/RocksDatabase.java | 7 +++ .../hdds/utils/db/managed/ManagedRange.java | 36 ++++++++++++ .../hdds/utils/db/managed/ManagedRocksDB.java | 12 ++++ 6 files changed, 193 insertions(+), 1 deletion(-) create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java create mode 100644 hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRange.java diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java index e33ef15898ff..82c96bf19fff 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java @@ -21,12 +21,14 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.List; import java.util.Map; import org.apache.hadoop.hdds.annotation.InterfaceStability; import org.apache.hadoop.hdds.utils.db.cache.TableCache; import org.apache.hadoop.hdds.utils.db.cache.TableCache.CacheType; import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions; import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer; +import org.rocksdb.TableProperties; /** * The DBStore interface provides the ability to create Tables, which store @@ -116,6 +118,34 @@ TypedTable getTable( */ void compactTable(String tableName, ManagedCompactRangeOptions options) throws IOException; + /** + * Compact a range of keys in the database. + * + * @param tableName - The table name to compact. + * @param startKey - The starting key of the range to compact. + * @param endKey - The ending key of the range to compact. + * @throws IOException on Failure + */ + void compactTable(String tableName, String startKey, String endKey) throws IOException; + + void compactTable(String tableName, String startKey, String endKey, + ManagedCompactRangeOptions options) throws IOException; + + /** + * Get the properties of a column family in a range. + * @param columnFamily - The column family to get the properties of. + * @param startKey - The starting key of the range. + * @param endKey - The ending key of the range. + * @return - The properties of the column family in the range. + * @throws IOException on Failure + */ + Map getPropertiesOfTableInRange(String tableName, String startKey, + String endKey) throws IOException; + + Map getPropertiesOfTableInRange(String tableName, List ranges) + throws IOException; + + /** * Moves a key from the Source Table to the destination Table. * diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java new file mode 100644 index 000000000000..6bcc2872a970 --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.utils.db; + +import java.io.IOException; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRange; +import org.apache.hadoop.hdds.utils.db.managed.ManagedSlice; + +/** + * Represents a range of keys for compaction. + */ +public class KeyRange { + private final String startKey; + private final String endKey; + + public KeyRange(String startKey, String endKey) { + this.startKey = startKey; + this.endKey = endKey; + } + + public String getStartKey() { + return startKey; + } + + public String getEndKey() { + return endKey; + } + + public ManagedRange toRocksRange() throws IOException { + return new ManagedRange( + new ManagedSlice(StringCodec.get().toPersistedFormat(startKey)), + new ManagedSlice(StringCodec.get().toPersistedFormat(endKey))); + } + + @Override + public String toString() { + return "KeyRange{" + + "startKey='" + startKey + '\'' + + ", endKey='" + endKey + '\'' + + '}'; + } +} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index 3539a906117d..10a9422d8fd9 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -34,8 +34,12 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.RocksDBStoreMetrics; @@ -43,12 +47,14 @@ import org.apache.hadoop.hdds.utils.db.cache.TableCache; import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRange; import org.apache.hadoop.hdds.utils.db.managed.ManagedStatistics; import org.apache.hadoop.hdds.utils.db.managed.ManagedTransactionLogIterator; import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteOptions; import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer; import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.RocksDBCheckpointDifferHolder; import org.rocksdb.RocksDBException; +import org.rocksdb.TableProperties; import org.rocksdb.TransactionLogIterator.BatchResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -226,19 +232,63 @@ public void compactDB() throws IOException { public void compactTable(String tableName) throws IOException { try (ManagedCompactRangeOptions options = new ManagedCompactRangeOptions()) { compactTable(tableName, options); + public void compactTable(String tableName, String startKey, String endKey) throws IOException { + try (ManagedCompactRangeOptions options = new ManagedCompactRangeOptions()) { + compactTable(tableName, startKey, endKey, options); } } @Override + public void compactTable(String tableName, ManagedCompactRangeOptions options) throws IOException { RocksDatabase.ColumnFamily columnFamily = db.getColumnFamily(tableName); - if (columnFamily == null) { throw new IOException("Table not found: " + tableName); } db.compactRange(columnFamily, null, null, options); } + public void compactTable(String tableName, String startKey, String endKey, + ManagedCompactRangeOptions options) throws IOException { + if (columnFamily == null) { + throw new IOException("No such table in this DB. TableName : " + tableName); + } + db.compactRange(columnFamily, getPersistedKey(startKey), + getPersistedKey(endKey), options); + } + @Override + public Map getPropertiesOfTableInRange(String tableName, String startKey, + String endKey) throws IOException { + return getPropertiesOfTableInRange(tableName, + Collections.singletonList(new KeyRange(startKey, endKey))); + } + + @Override + public Map getPropertiesOfTableInRange(String tableName, + List ranges) throws IOException { + ColumnFamily columnFamily = db.getColumnFamily(tableName); + if (columnFamily == null) { + throw new IOException("No such table in this DB. TableName : " + tableName); + } + + List rocksRanges = ranges.stream() + .map(t -> { + try { + return t.toRocksRange(); + } catch (IOException e) { + LOG.error("Failed to convert {} to RocksDB Range", t, e); + return null; + } + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + return db.getPropertiesOfColumnFamilyInRange(columnFamily, rocksRanges); + } + + private byte[] getPersistedKey(String key) throws IOException { + return StringCodec.get().toPersistedFormat(key); + } + public void close() throws IOException { if (metrics != null) { metrics.unregister(); diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java index cf0c84f375e3..65e89cd7cc1a 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedFlushOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedIngestExternalFileOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRange; import org.apache.hadoop.hdds.utils.db.managed.ManagedReadOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator; @@ -64,6 +65,7 @@ import org.rocksdb.KeyMayExist; import org.rocksdb.LiveFileMetaData; import org.rocksdb.RocksDBException; +import org.rocksdb.TableProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -881,6 +883,11 @@ public void deleteFilesNotMatchingPrefix(Map prefixPairs) throws } } + public Map getPropertiesOfColumnFamilyInRange(ColumnFamily columnFamily, + List ranges) throws RocksDatabaseException { + return db.getPropertiesOfColumnFamilyInRange(columnFamily.getHandle(), ranges); + } + @Override protected void finalize() throws Throwable { if (!isClosed()) { diff --git a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRange.java b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRange.java new file mode 100644 index 000000000000..ad573dc44319 --- /dev/null +++ b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRange.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.utils.db.managed; + +import org.rocksdb.Range; + +/** + * Managed {@link Range}. + */ +public class ManagedRange { + + private final Range original; + + public ManagedRange(final ManagedSlice start, final ManagedSlice limit) { + this.original = new Range(start, limit); + } + + public Range getOriginal() { + return original; + } +} diff --git a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java index 0fff6098bff4..5855e998768d 100644 --- a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java +++ b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java @@ -30,6 +30,7 @@ import org.rocksdb.LiveFileMetaData; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; +import org.rocksdb.TableProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -123,4 +124,15 @@ public static Map getLiveMetadataForSSTFiles(RocksDB d public Map getLiveMetadataForSSTFiles() { return getLiveMetadataForSSTFiles(this.get()); } + + public Map getPropertiesOfColumnFamilyInRange(ColumnFamilyHandle columnFamilyHandle, + List ranges) throws RocksDatabaseException { + try { + return get().getPropertiesOfTablesInRange(ranges.stream() + .map(ManagedRange::getOriginal) + .collect(Collectors.toList())); + } catch (RocksDBException e) { + throw new RocksDatabaseException("Failed to get properties of column family in range", e); + } + } } From 565b258a34a54071c35999676ba4836568c81d87 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Mon, 12 May 2025 16:27:07 +0800 Subject: [PATCH 03/31] Add range compaction service skelton --- .../apache/hadoop/ozone/om/OMConfigKeys.java | 20 +++ .../hadoop/ozone/om/KeyManagerImpl.java | 29 ++++ .../hadoop/ozone/om/codec/OMDBDefinition.java | 33 ++++ .../om/compaction/AbstractCompactor.java | 144 +++++++++++++++++ .../ozone/om/compaction/BucketCompactor.java | 111 +++++++++++++ .../hadoop/ozone/om/compaction/Compactor.java | 51 ++++++ .../ozone/om/compaction/CompactorBuilder.java | 122 ++++++++++++++ .../om/compaction/FSOBucketCompactor.java | 134 +++++++++++++++ .../ozone/om/compaction/KeyRangeStats.java | 68 ++++++++ .../ozone/om/compaction/package-info.java | 22 +++ .../om/service/RangeCompactionService.java | 152 ++++++++++++++++++ 11 files changed, 886 insertions(+) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/AbstractCompactor.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/BucketCompactor.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/Compactor.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompactorBuilder.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOBucketCompactor.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/KeyRangeStats.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/package-info.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 1a7cafd8ee27..20d04a374a07 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -657,6 +657,26 @@ public final class OMConfigKeys { "ozone.om.snapshot.compact.non.snapshot.diff.tables"; public static final boolean OZONE_OM_SNAPSHOT_COMPACT_NON_SNAPSHOT_DIFF_TABLES_DEFAULT = false; + /** + * Configuration keys for RocksDB range compaction service. + */ + public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_ENABLED = "ozone.om.range.compaction.service.enabled"; + public static final boolean OZONE_OM_RANGE_COMPACTION_SERVICE_ENABLED_DEFAULT = true; + public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_INTERVAL = "ozone.om.range.compaction.service.interval"; + public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_INTERVAL_DEFAULT = "1s"; + public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_TIMEOUT = "ozone.om.range.compaction.service.timeout"; + public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_TIMEOUT_DEFAULT = "10m"; + + public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_MAX_ENTRIES_SUM + = "ozone.om.range.compaction.service.max.entries.sum"; + public static final int OZONE_OM_RANGE_COMPACTION_SERVICE_MAX_ENTRIES_SUM_DEFAULT = 1000000; + public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_TOMBSTONE_RATIO + = "ozone.om.range.compaction.service.tombstone.ratio"; + public static final double OZONE_OM_RANGE_COMPACTION_SERVICE_TOMBSTONE_RATIO_DEFAULT = 0.3; + public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_MIN_TOMBSTONES + = "ozone.om.range.compaction.service.min.tombstones"; + public static final int OZONE_OM_RANGE_COMPACTION_SERVICE_MIN_TOMBSTONES_DEFAULT = 10000; + /** * Never constructed. */ diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 578afc630a1c..9676f2c26b83 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -56,6 +56,12 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_TIMEOUT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_TIMEOUT_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RANGE_COMPACTION_SERVICE_ENABLED; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RANGE_COMPACTION_SERVICE_ENABLED_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RANGE_COMPACTION_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RANGE_COMPACTION_SERVICE_INTERVAL_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RANGE_COMPACTION_SERVICE_TIMEOUT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RANGE_COMPACTION_SERVICE_TIMEOUT_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_DEEP_CLEANING_ENABLED; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_DEEP_CLEANING_ENABLED_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL; @@ -166,6 +172,7 @@ import org.apache.hadoop.ozone.om.service.KeyDeletingService; import org.apache.hadoop.ozone.om.service.MultipartUploadCleanupService; import org.apache.hadoop.ozone.om.service.OpenKeyCleanupService; +import org.apache.hadoop.ozone.om.service.RangeCompactionService; import org.apache.hadoop.ozone.om.service.SnapshotDeletingService; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ExpiredMultipartUploadsBucket; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo; @@ -211,6 +218,7 @@ public class KeyManagerImpl implements KeyManager { private BackgroundService multipartUploadCleanupService; private DNSToSwitchMapping dnsToSwitchMapping; private CompactionService compactionService; + private RangeCompactionService rangeCompactionService; public KeyManagerImpl(OzoneManager om, ScmClient scmClient, OzoneConfiguration conf, OMPerformanceMetrics metrics) { @@ -244,6 +252,10 @@ public void start(OzoneConfiguration configuration) { OZONE_OM_COMPACTION_SERVICE_ENABLED_DEFAULT); startCompactionService(configuration, isCompactionEnabled); + boolean isRangeCompactionEnabled = configuration.getBoolean(OZONE_OM_RANGE_COMPACTION_SERVICE_ENABLED, + OZONE_OM_RANGE_COMPACTION_SERVICE_ENABLED_DEFAULT); + startRangeCompactionService(configuration, isRangeCompactionEnabled); + boolean isSnapshotDeepCleaningEnabled = configuration.getBoolean(OZONE_SNAPSHOT_DEEP_CLEANING_ENABLED, OZONE_SNAPSHOT_DEEP_CLEANING_ENABLED_DEFAULT); if (keyDeletingService == null) { @@ -391,6 +403,23 @@ private void startCompactionService(OzoneConfiguration configuration, } } + private void startRangeCompactionService(OzoneConfiguration configuration, + boolean isRangeCompactionEnabled) { + if (rangeCompactionService == null && isRangeCompactionEnabled) { + long serviceIntervalMs = configuration.getTimeDuration( + OZONE_OM_RANGE_COMPACTION_SERVICE_INTERVAL, + OZONE_OM_RANGE_COMPACTION_SERVICE_INTERVAL_DEFAULT, + TimeUnit.MILLISECONDS); + long serviceTimeoutMs = configuration.getTimeDuration( + OZONE_OM_RANGE_COMPACTION_SERVICE_TIMEOUT, + OZONE_OM_RANGE_COMPACTION_SERVICE_TIMEOUT_DEFAULT, + TimeUnit.MILLISECONDS); + rangeCompactionService = new RangeCompactionService(configuration, ozoneManager, + serviceIntervalMs, serviceTimeoutMs); + rangeCompactionService.start(); + } + } + KeyProviderCryptoExtension getKMSProvider() { return kmsProvider; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java index 6d053e1e5e05..bd6c238c352d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java @@ -17,6 +17,8 @@ package org.apache.hadoop.ozone.om.codec; +import java.util.Collections; +import java.util.HashMap; import java.util.Map; import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition; @@ -26,6 +28,7 @@ import org.apache.hadoop.hdds.utils.db.StringCodec; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDBAccessIdInfo; import org.apache.hadoop.ozone.om.helpers.OmDBTenantState; @@ -339,6 +342,27 @@ public final class OMDBDefinition extends DBDefinition.WithMap { USER_TABLE_DEF, VOLUME_TABLE_DEF); + /** + * Mapping from table names to their corresponding bucket layouts. + * This is used to determine which type of compactor to use for each table. + */ + private static final Map TABLE_BUCKET_LAYOUT_MAP; + + static { + Map map = new HashMap<>(); + // FSO tables + map.put(FILE_TABLE_DEF.getName(), BucketLayout.FILE_SYSTEM_OPTIMIZED); + map.put(OPEN_FILE_TABLE_DEF.getName(), BucketLayout.FILE_SYSTEM_OPTIMIZED); + map.put(DIRECTORY_TABLE_DEF.getName(), BucketLayout.FILE_SYSTEM_OPTIMIZED); + map.put(DELETED_DIR_TABLE_DEF.getName(), BucketLayout.FILE_SYSTEM_OPTIMIZED); + // OBS/Legacy tables + map.put(KEY_TABLE_DEF.getName(), BucketLayout.OBJECT_STORE); + map.put(DELETED_TABLE_DEF.getName(), BucketLayout.OBJECT_STORE); + map.put(OPEN_KEY_TABLE_DEF.getName(), BucketLayout.OBJECT_STORE); + map.put(MULTIPART_INFO_TABLE_DEF.getName(), BucketLayout.OBJECT_STORE); + TABLE_BUCKET_LAYOUT_MAP = Collections.unmodifiableMap(map); + } + private static final OMDBDefinition INSTANCE = new OMDBDefinition(); public static OMDBDefinition get() { @@ -358,5 +382,14 @@ public String getName() { public String getLocationConfigKey() { return OMConfigKeys.OZONE_OM_DB_DIRS; } + + /** + * Get the bucket layout for a given table name. + * @param tableName The name of the table + * @return The bucket layout for the table, or null if not found + */ + public static BucketLayout getBucketLayoutForTable(String tableName) { + return TABLE_BUCKET_LAYOUT_MAP.get(tableName); + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/AbstractCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/AbstractCompactor.java new file mode 100644 index 000000000000..6aafa232de9e --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/AbstractCompactor.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.compaction; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.hdds.utils.BackgroundTask; +import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; +import org.apache.hadoop.hdds.utils.BackgroundTaskResult; +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.hdds.utils.db.KeyRange; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.rocksdb.TableProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Abstract base class for compactors that provides common functionality. + */ +public abstract class AbstractCompactor implements Compactor { + private static final Logger LOG = LoggerFactory.getLogger(AbstractCompactor.class); + + private OMMetadataManager metadataManager; + private DBStore dbStore; + private String tableName; + private BackgroundTaskQueue compactRangeQueue; + + private int maxEntriesSum; + private int minTombstones; + private double tombstoneRatio; + + protected AbstractCompactor(CompactorBuilder builder) { + this.tableName = builder.getTableName(); + this.maxEntriesSum = builder.getMaxEntriesSum(); + this.minTombstones = builder.getMinTombstones(); + this.tombstoneRatio = builder.getTombstoneRatio(); + this.metadataManager = builder.getMetadataManager(); + this.dbStore = builder.getDBStore(); + this.compactRangeQueue = builder.getCompactRangeQueue(); + } + + @Override + public String getTableName() { + return tableName; + } + + @Override + public void compactRange(KeyRange range) { + try { + LOG.info("Compacting range {} for table {}", range, tableName); + dbStore.compactTable(tableName, range.getStartKey(), range.getEndKey()); + } catch (Exception e) { + LOG.error("Failed to compact range {} for table {}", range, tableName, e); + } + } + + protected KeyRangeStats getRangeStats(KeyRange range) { + try { + Map properties = dbStore.getPropertiesOfTableInRange( + tableName, Collections.singletonList(range)); + + KeyRangeStats stats = new KeyRangeStats(0, 0); + for (TableProperties prop : properties.values()) { + stats.add(KeyRangeStats.fromTableProperties(prop)); + } + return stats; + } catch (Exception e) { + LOG.error("Failed to get range stats for range {} in table {}", range, tableName, e); + return new KeyRangeStats(0, 0); + } + } + + @Override + public List getRangesNeedingCompaction() { + List ranges = new ArrayList<>(); + collectRangesNeedingCompaction(ranges); + return ranges; + } + + protected abstract void collectRangesNeedingCompaction(List ranges); + + protected OMMetadataManager getMetadataManager() { + return metadataManager; + } + + protected int getMaxEntriesSum() { + return maxEntriesSum; + } + + protected int getMinTombstones() { + return minTombstones; + } + + protected double getTombstoneRatio() { + return tombstoneRatio; + } + + protected DBStore getDBStore() { + return dbStore; + } + + protected void addRangeCompactionTask(KeyRange range) { + compactRangeQueue.add(new CompactionTask(range)); + } + + private class CompactionTask implements BackgroundTask { + private final KeyRange range; + + CompactionTask(KeyRange range) { + this.range = range; + } + + @Override + public int getPriority() { + // TODO: Implement priority based on some criteria + return 0; + } + + @Override + public BackgroundTaskResult call() throws Exception { + LOG.debug("Running compaction for range {} in table {}", range, tableName); + + dbStore.compactTable(tableName, range.getStartKey(), range.getEndKey()); + return () -> 1; + } + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/BucketCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/BucketCompactor.java new file mode 100644 index 000000000000..2d818aeeff38 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/BucketCompactor.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.compaction; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.hdds.utils.db.KeyRange; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Compactor for OBS and legacy layout that compacts based on bucket ranges. + */ +public class BucketCompactor extends AbstractCompactor { + private static final Logger LOG = LoggerFactory.getLogger(BucketCompactor.class); + + private String nextBucket; + private String nextKey; + + public BucketCompactor(CompactorBuilder builder) { + super(builder); + } + + @Override + public void run() { + + } + + @Override + protected void collectRangesNeedingCompaction(List ranges) { + Iterator, CacheValue>> bucketIterator = getBucketIterator(); + while (bucketIterator.hasNext()) { + Map.Entry, CacheValue> entry = bucketIterator.next(); + String bucketKey = entry.getKey().getCacheKey(); + OmBucketInfo bucketInfo = entry.getValue().getCacheValue(); + + if (nextBucket != null && !nextBucket.equals(bucketKey)) { + continue; + } + + KeyRange bucketRange = new KeyRange(bucketKey, getNextBucketKey(bucketKey)); + KeyRangeStats stats = getRangeStats(bucketRange); + + if (stats.getNumEntries() <= getMaxEntriesSum()) { + // If the entire bucket fits within maxEntriesSum, check if it needs compaction + if (needsCompaction(stats, getMinTombstones(), getTombstoneRatio())) { + ranges.add(bucketRange); + } + nextBucket = null; + nextKey = null; + } else { + // Need to split the bucket range + String splitKey = findSplitKey(bucketRange); + if (splitKey != null) { + KeyRange splitRange = new KeyRange(bucketKey, splitKey); + KeyRangeStats splitStats = getRangeStats(splitRange); + if (needsCompaction(splitStats, getMinTombstones(), getTombstoneRatio())) { + ranges.add(splitRange); + } + nextBucket = bucketKey; + nextKey = splitKey; + break; + } + } + } + } + + private Iterator, CacheValue>> getBucketIterator() { + if (nextBucket != null) { + return getMetadataManager().getBucketIterator(nextBucket); + } + return getMetadataManager().getBucketIterator(); + } + + private String getNextBucketKey(String currentBucketKey) { + // For OBS and legacy layout, the next bucket key is the current bucket key + 1 + // This is a simplified version - in reality, you'd need to handle the key + // format + // based on your specific key structure + return currentBucketKey + "\0"; + } + + private String findSplitKey(KeyRange range) { + // Binary search to find a split key that keeps the range under maxEntriesSum + String startKey = range.getStartKey(); + String endKey = range.getEndKey(); + + // This is a simplified version - in reality, you'd need to implement + // a proper binary search based on your key format + return startKey + "\0"; + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/Compactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/Compactor.java new file mode 100644 index 000000000000..c91b9b7fbe75 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/Compactor.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.compaction; + +import java.util.List; +import org.apache.hadoop.hdds.utils.db.KeyRange; + +/** + * Interface for compactors that handle RocksDB compaction. + */ +public interface Compactor extends Runnable { + /** + * Get the table this compactor is responsible for. + */ + String getTableName(); + + /** + * Check if a range needs compaction based on its statistics. + */ + default boolean needsCompaction(KeyRangeStats stats, int minTombstones, double tombstonePercentage) { + if (stats.getNumDeletion() < minTombstones) { + return false; + } + return stats.getTombstonePercentage() >= tombstonePercentage; + } + + /** + * Compact a specific range. + */ + void compactRange(KeyRange range); + + /** + * Get the ranges that need compaction. + */ + List getRangesNeedingCompaction(); +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompactorBuilder.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompactorBuilder.java new file mode 100644 index 000000000000..fa77279e4986 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompactorBuilder.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.compaction; + +import java.io.IOException; +import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.codec.OMDBDefinition; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; + +/** + * Builder class for creating compactors based on table names. + */ +public class CompactorBuilder { + private DBStore db; + private BackgroundTaskQueue compactRangeQueue; + private String tableName; + private int maxEntriesSum; + private int minTombstones; + private double tombstoneRatio; + + private OMMetadataManager metadataManager; + + public CompactorBuilder setCompactRangeQueue(BackgroundTaskQueue compactRangeQueue) { + this.compactRangeQueue = compactRangeQueue; + return this; + } + + public CompactorBuilder setMetadataManager(OMMetadataManager metadataManager) { + this.metadataManager = metadataManager; + return this; + } + + public CompactorBuilder setDBStore(DBStore dbStore) { + this.db = dbStore; + return this; + } + + public CompactorBuilder setTableName(String tableName) { + this.tableName = tableName; + return this; + } + + public CompactorBuilder setMaxEntriesSum(int maxEntriesSum) { + this.maxEntriesSum = maxEntriesSum; + return this; + } + + public CompactorBuilder setMinTombstones(int minTombstones) { + this.minTombstones = minTombstones; + return this; + } + + public CompactorBuilder setTombstoneRatio(double tombstoneRatio) { + this.tombstoneRatio = tombstoneRatio; + return this; + } + + public Compactor build() throws IOException { + if (tableName == null) { + throw new IllegalArgumentException("Name and table must be set"); + } + + BucketLayout layout = OMDBDefinition.getBucketLayoutForTable(tableName); + if (layout == null) { + throw new IllegalArgumentException("No bucket layout mapping found for table: " + tableName); + } + + switch (layout) { + case FILE_SYSTEM_OPTIMIZED: + return new FSOBucketCompactor(this); + case OBJECT_STORE: + return new BucketCompactor(this); + default: + throw new IllegalArgumentException("Unsupported bucket layout: " + layout); + } + } + + public String getTableName() { + return tableName; + } + + public int getMaxEntriesSum() { + return maxEntriesSum; + } + + public int getMinTombstones() { + return minTombstones; + } + + public double getTombstoneRatio() { + return tombstoneRatio; + } + + public DBStore getDBStore() { + return db; + } + + public OMMetadataManager getMetadataManager() { + return metadataManager; + } + + public BackgroundTaskQueue getCompactRangeQueue() { + return compactRangeQueue; + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOBucketCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOBucketCompactor.java new file mode 100644 index 000000000000..5caad098f23c --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOBucketCompactor.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.compaction; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.hdds.utils.db.KeyRange; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Compactor for FSO layout that compacts based on bucket and parent ID ranges. + */ +public class FSOBucketCompactor extends AbstractCompactor { + private static final Logger LOG = LoggerFactory.getLogger(FSOBucketCompactor.class); + + private String nextBucket; + private String nextParentId; + private String nextKey; + + public FSOBucketCompactor(CompactorBuilder builder) { + super(builder); + } + + @Override + public void run() { + } + + @Override + protected void collectRangesNeedingCompaction(List ranges) { + Iterator, CacheValue>> bucketIterator = getBucketIterator(); + while (bucketIterator.hasNext()) { + Map.Entry, CacheValue> entry = bucketIterator.next(); + String bucketKey = entry.getKey().getCacheKey(); + OmBucketInfo bucketInfo = entry.getValue().getCacheValue(); + + if (nextBucket != null && !nextBucket.equals(bucketKey)) { + continue; + } + + // For FSO, we need to handle parent IDs + if (nextParentId != null) { + // Continue with the same bucket but different parent ID + KeyRange parentRange = new KeyRange( + getParentKey(bucketKey, nextParentId), + getNextParentKey(bucketKey, nextParentId)); + processRange(parentRange, ranges); + } else { + // Start with the first parent ID for this bucket + String firstParentId = getFirstParentId(bucketKey); + if (firstParentId != null) { + KeyRange parentRange = new KeyRange( + getParentKey(bucketKey, firstParentId), + getNextParentKey(bucketKey, firstParentId)); + processRange(parentRange, ranges); + } + } + } + } + + private void processRange(KeyRange range, List ranges) { + KeyRangeStats stats = getRangeStats(range); + + if (stats.getNumEntries() <= getMaxEntriesSum()) { + if (needsCompaction(stats, getMinTombstones(), getTombstoneRatio())) { + ranges.add(range); + } + nextParentId = null; + nextKey = null; + } else { + String splitKey = findSplitKey(range); + if (splitKey != null) { + KeyRange splitRange = new KeyRange(range.getStartKey(), splitKey); + KeyRangeStats splitStats = getRangeStats(splitRange); + if (needsCompaction(splitStats, getMinTombstones(), getTombstoneRatio())) { + ranges.add(splitRange); + } + nextKey = splitKey; + } + } + } + + private Iterator, CacheValue>> getBucketIterator() { + if (nextBucket != null) { + return getMetadataManager().getBucketIterator(nextBucket); + } + return getMetadataManager().getBucketIterator(); + } + + private String getParentKey(String bucketKey, String parentId) { + // Format: /volumeId/bucketId/parentId/ + return bucketKey + "/" + parentId + "/"; + } + + private String getNextParentKey(String bucketKey, String parentId) { + // Format: /volumeId/bucketId/parentId/ + 1 + return bucketKey + "/" + parentId + "/\0"; + } + + private String getFirstParentId(String bucketKey) { + // This is a simplified version - in reality, you'd need to implement + // a proper way to get the first parent ID for a bucket + return "0"; + } + + private String findSplitKey(KeyRange range) { + // Binary search to find a split key that keeps the range under maxEntriesSum + String startKey = range.getStartKey(); + String endKey = range.getEndKey(); + + // This is a simplified version - in reality, you'd need to implement + // a proper binary search based on your key format + return startKey + "\0"; + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/KeyRangeStats.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/KeyRangeStats.java new file mode 100644 index 000000000000..878939435e15 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/KeyRangeStats.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.compaction; + +import org.rocksdb.TableProperties; + +/** + * Statistics for a key range. + */ +public class KeyRangeStats { + private int numEntries; + private int numDeletion; + + public KeyRangeStats(int numEntries, int numDeletion) { + this.numEntries = numEntries; + this.numDeletion = numDeletion; + } + + public static KeyRangeStats fromTableProperties(TableProperties properties) { + return new KeyRangeStats( + (int) properties.getNumEntries(), + (int) properties.getNumDeletions()); + } + + public void add(KeyRangeStats other) { + this.numEntries += other.numEntries; + this.numDeletion += other.numDeletion; + } + + public int getNumEntries() { + return numEntries; + } + + public int getNumDeletion() { + return numDeletion; + } + + public double getTombstonePercentage() { + if (numEntries == 0) { + return 0.0; + } + return ((double) numDeletion / numEntries) * 100; + } + + @Override + public String toString() { + return "KeyRangeStats{" + + "numEntries=" + numEntries + + ", numDeletion=" + numDeletion + + ", tombstonePercentage=" + getTombstonePercentage() + + '}'; + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/package-info.java new file mode 100644 index 000000000000..51dc70428b12 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Utility classes to encode/decode DTO objects to/from byte array. + * OM DB definitions. + */ +package org.apache.hadoop.ozone.om.compaction; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java new file mode 100644 index 000000000000..776d2f9b0471 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.service; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.BackgroundService; +import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.compaction.Compactor; +import org.apache.hadoop.ozone.om.compaction.CompactorBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manages all compactors and schedules their execution. + */ +public class RangeCompactionService extends BackgroundService { + private static final Logger LOG = LoggerFactory.getLogger(RangeCompactionService.class); + private static final int THREAD_POOL_SIZE = 1; + + private final OMMetadataManager metadataManager; + private final List compactors; + private final Map compactorExecutors; + private final long checkIntervalMs; + private final int maxEntriesSum; + private final int minTombstones; + private final double tombstoneRatio; + + private final BackgroundTaskQueue tasks; + + public RangeCompactionService(OzoneConfiguration conf, OzoneManager ozoneManager, long intervalMs, + long timeoutMs) { + super("RangeCompactionService", intervalMs, TimeUnit.MILLISECONDS, THREAD_POOL_SIZE, timeoutMs, + ozoneManager.getThreadNamePrefix()); + this.metadataManager = ozoneManager.getMetadataManager(); + this.checkIntervalMs = intervalMs; + this.maxEntriesSum = conf.getInt(OMConfigKeys.OZONE_OM_RANGE_COMPACTION_SERVICE_MAX_ENTRIES_SUM, + OMConfigKeys.OZONE_OM_RANGE_COMPACTION_SERVICE_MAX_ENTRIES_SUM_DEFAULT); + this.minTombstones = conf.getInt(OMConfigKeys.OZONE_OM_RANGE_COMPACTION_SERVICE_MIN_TOMBSTONES, + OMConfigKeys.OZONE_OM_RANGE_COMPACTION_SERVICE_MIN_TOMBSTONES_DEFAULT); + this.tombstoneRatio = conf.getDouble(OMConfigKeys.OZONE_OM_RANGE_COMPACTION_SERVICE_TOMBSTONE_RATIO, + OMConfigKeys.OZONE_OM_RANGE_COMPACTION_SERVICE_TOMBSTONE_RATIO_DEFAULT); + this.compactors = new ArrayList<>(); + this.compactorExecutors = new ConcurrentHashMap<>(); + + this.tasks = new BackgroundTaskQueue(); + + start(); + } + + @Override + public BackgroundTaskQueue getTasks() { + return tasks; + } + + @Override + public void start() { + initializeCompactors(); + scheduleCompactionChecks(); + } + + public void stop() { + for (Map.Entry entry : compactorExecutors.entrySet()) { + ScheduledExecutorService executor = entry.getValue(); + executor.shutdown(); + try { + if (!executor.awaitTermination(60, TimeUnit.SECONDS)) { + executor.shutdownNow(); + } + } catch (InterruptedException e) { + executor.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + compactorExecutors.clear(); + } + + private void initializeCompactors() { + // Initialize compactors using the builder pattern + CompactorBuilder builder = new CompactorBuilder() + .setCompactRangeQueue(tasks) + .setMetadataManager(metadataManager) + .setDBStore(metadataManager.getStore()) + .setMaxEntriesSum(maxEntriesSum) + .setMinTombstones(minTombstones) + .setTombstoneRatio(tombstoneRatio); + + // Initialize compactors for OBS and legacy layout + try { + compactors.add(builder.setTableName("keyTable") + .build()); + compactors.add(builder.setTableName("deletedTable") + .build()); + + // Initialize compactors for FSO layout + compactors.add(builder.setTableName("directoryTable") + .build()); + compactors.add(builder.setTableName("fileTable") + .build()); + compactors.add(builder.setTableName("deletedDirTable") + .build()); + } catch (IOException e) { + LOG.error("Failed to initialize compactors", e); + throw new RuntimeException("Failed to initialize compactors", e); + } + + // Initialize all compactors + // for (Compactor compactor : compactors) { + // compactor.init(metadataManager, dbStore); + // } + } + + private void scheduleCompactionChecks() { + for (Compactor compactor : compactors) { + String tableName = compactor.getTableName(); + ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + compactorExecutors.put(tableName, executor); + + executor.scheduleWithFixedDelay( + compactor::run, + checkIntervalMs, + checkIntervalMs, + TimeUnit.MILLISECONDS); + } + } +} + From ef1b22d2de7857e65a15f92d41d4f605a500837c Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sun, 18 May 2025 22:36:20 +0800 Subject: [PATCH 04/31] Refactor compaction logic to utilize StringUtils for key conversion and enhance KeyRangeStats to use long for entry counts. Introduce FSOTableCompactor and OBSTableCompactor classes for improved compaction strategies based on bucket layouts. --- .../apache/hadoop/hdds/utils/db/KeyRange.java | 5 +++-- .../apache/hadoop/hdds/utils/db/RDBStore.java | 10 ++++------ .../hadoop/ozone/om/compaction/Compactor.java | 4 ++-- .../ozone/om/compaction/CompactorBuilder.java | 4 ++-- ...tCompactor.java => FSOTableCompactor.java} | 6 +++--- .../ozone/om/compaction/KeyRangeStats.java | 20 +++++++++---------- ...tCompactor.java => OBSTableCompactor.java} | 6 +++--- 7 files changed, 27 insertions(+), 28 deletions(-) rename hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/{FSOBucketCompactor.java => FSOTableCompactor.java} (95%) rename hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/{BucketCompactor.java => OBSTableCompactor.java} (95%) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java index 6bcc2872a970..8f159dfd9fa9 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdds.utils.db; import java.io.IOException; +import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.utils.db.managed.ManagedRange; import org.apache.hadoop.hdds.utils.db.managed.ManagedSlice; @@ -43,8 +44,8 @@ public String getEndKey() { public ManagedRange toRocksRange() throws IOException { return new ManagedRange( - new ManagedSlice(StringCodec.get().toPersistedFormat(startKey)), - new ManagedSlice(StringCodec.get().toPersistedFormat(endKey))); + new ManagedSlice(StringUtils.string2Bytes(startKey)), + new ManagedSlice(StringUtils.string2Bytes(endKey))); } @Override diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index 10a9422d8fd9..e67c9c6d1a4a 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -40,6 +40,7 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; +import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.RocksDBStoreMetrics; @@ -252,8 +253,8 @@ public void compactTable(String tableName, String startKey, String endKey, if (columnFamily == null) { throw new IOException("No such table in this DB. TableName : " + tableName); } - db.compactRange(columnFamily, getPersistedKey(startKey), - getPersistedKey(endKey), options); + db.compactRange(columnFamily, StringUtils.string2Bytes(startKey), + StringUtils.string2Bytes(endKey), options); } @Override @@ -285,10 +286,7 @@ public Map getPropertiesOfTableInRange(String tableName return db.getPropertiesOfColumnFamilyInRange(columnFamily, rocksRanges); } - private byte[] getPersistedKey(String key) throws IOException { - return StringCodec.get().toPersistedFormat(key); - } - + @Override public void close() throws IOException { if (metrics != null) { metrics.unregister(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/Compactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/Compactor.java index c91b9b7fbe75..fba9d0106e06 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/Compactor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/Compactor.java @@ -32,11 +32,11 @@ public interface Compactor extends Runnable { /** * Check if a range needs compaction based on its statistics. */ - default boolean needsCompaction(KeyRangeStats stats, int minTombstones, double tombstonePercentage) { + default boolean needsCompaction(KeyRangeStats stats, int minTombstones, double tombstoneRatio) { if (stats.getNumDeletion() < minTombstones) { return false; } - return stats.getTombstonePercentage() >= tombstonePercentage; + return stats.getTombstoneRatio() >= tombstoneRatio; } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompactorBuilder.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompactorBuilder.java index fa77279e4986..e2a7821cf996 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompactorBuilder.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompactorBuilder.java @@ -84,9 +84,9 @@ public Compactor build() throws IOException { switch (layout) { case FILE_SYSTEM_OPTIMIZED: - return new FSOBucketCompactor(this); + return new FSOTableCompactor(this); case OBJECT_STORE: - return new BucketCompactor(this); + return new OBSTableCompactor(this); default: throw new IllegalArgumentException("Unsupported bucket layout: " + layout); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOBucketCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOTableCompactor.java similarity index 95% rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOBucketCompactor.java rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOTableCompactor.java index 5caad098f23c..e659fd951d57 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOBucketCompactor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOTableCompactor.java @@ -30,14 +30,14 @@ /** * Compactor for FSO layout that compacts based on bucket and parent ID ranges. */ -public class FSOBucketCompactor extends AbstractCompactor { - private static final Logger LOG = LoggerFactory.getLogger(FSOBucketCompactor.class); +public class FSOTableCompactor extends AbstractCompactor { + private static final Logger LOG = LoggerFactory.getLogger(FSOTableCompactor.class); private String nextBucket; private String nextParentId; private String nextKey; - public FSOBucketCompactor(CompactorBuilder builder) { + public FSOTableCompactor(CompactorBuilder builder) { super(builder); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/KeyRangeStats.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/KeyRangeStats.java index 878939435e15..a7de0c02b44c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/KeyRangeStats.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/KeyRangeStats.java @@ -23,18 +23,18 @@ * Statistics for a key range. */ public class KeyRangeStats { - private int numEntries; - private int numDeletion; + private long numEntries; + private long numDeletion; - public KeyRangeStats(int numEntries, int numDeletion) { + public KeyRangeStats(long numEntries, long numDeletion) { this.numEntries = numEntries; this.numDeletion = numDeletion; } public static KeyRangeStats fromTableProperties(TableProperties properties) { return new KeyRangeStats( - (int) properties.getNumEntries(), - (int) properties.getNumDeletions()); + properties.getNumEntries(), + properties.getNumDeletions()); } public void add(KeyRangeStats other) { @@ -42,19 +42,19 @@ public void add(KeyRangeStats other) { this.numDeletion += other.numDeletion; } - public int getNumEntries() { + public long getNumEntries() { return numEntries; } - public int getNumDeletion() { + public long getNumDeletion() { return numDeletion; } - public double getTombstonePercentage() { + public double getTombstoneRatio() { if (numEntries == 0) { return 0.0; } - return ((double) numDeletion / numEntries) * 100; + return ((double) numDeletion / numEntries); } @Override @@ -62,7 +62,7 @@ public String toString() { return "KeyRangeStats{" + "numEntries=" + numEntries + ", numDeletion=" + numDeletion + - ", tombstonePercentage=" + getTombstonePercentage() + + ", tombstoneRatio=" + getTombstoneRatio() + '}'; } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/BucketCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java similarity index 95% rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/BucketCompactor.java rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java index 2d818aeeff38..bebb31ffc6de 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/BucketCompactor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java @@ -30,13 +30,13 @@ /** * Compactor for OBS and legacy layout that compacts based on bucket ranges. */ -public class BucketCompactor extends AbstractCompactor { - private static final Logger LOG = LoggerFactory.getLogger(BucketCompactor.class); +public class OBSTableCompactor extends AbstractCompactor { + private static final Logger LOG = LoggerFactory.getLogger(OBSTableCompactor.class); private String nextBucket; private String nextKey; - public BucketCompactor(CompactorBuilder builder) { + public OBSTableCompactor(CompactorBuilder builder) { super(builder); } From dd3aa2093710eb222368586b6cd180ea323d3d7a Mon Sep 17 00:00:00 2001 From: peterxcli Date: Mon, 19 May 2025 01:11:13 +0800 Subject: [PATCH 05/31] Enhance FullTableCache iterator to handle null startKey and add getKeyPrefixUpperBound method in AbstractCompactor for improved key range handling. Update FSOTableCompactor and OBSTableCompactor to utilize new methods and streamline compaction logic. --- .../hdds/utils/db/cache/FullTableCache.java | 3 + .../om/compaction/AbstractCompactor.java | 6 + .../om/compaction/FSOTableCompactor.java | 16 +-- .../om/compaction/OBSTableCompactor.java | 111 ++++++++++-------- 4 files changed, 82 insertions(+), 54 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/FullTableCache.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/FullTableCache.java index e0f19b4d62d3..3badd4d7b0ec 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/FullTableCache.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/FullTableCache.java @@ -153,6 +153,9 @@ public Iterator, CacheValue>> iterator() { @Override public Iterator, CacheValue>> iterator(KEY startKey) { statsRecorder.recordIteration(); + if (startKey == null) { + return cache.entrySet().iterator(); + } return cache.tailMap(new CacheKey<>(startKey)).entrySet().iterator(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/AbstractCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/AbstractCompactor.java index 6aafa232de9e..9299be2dd265 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/AbstractCompactor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/AbstractCompactor.java @@ -120,6 +120,12 @@ protected void addRangeCompactionTask(KeyRange range) { compactRangeQueue.add(new CompactionTask(range)); } + protected String getKeyPrefixUpperBound(String keyPrefix) { + char[] upperBoundCharArray = keyPrefix.toCharArray(); + upperBoundCharArray[upperBoundCharArray.length - 1] += 1; + return String.valueOf(upperBoundCharArray); + } + private class CompactionTask implements BackgroundTask { private final KeyRange range; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOTableCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOTableCompactor.java index e659fd951d57..7186cb22057a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOTableCompactor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOTableCompactor.java @@ -24,18 +24,18 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +// import org.slf4j.Logger; +// import org.slf4j.LoggerFactory; /** * Compactor for FSO layout that compacts based on bucket and parent ID ranges. */ public class FSOTableCompactor extends AbstractCompactor { - private static final Logger LOG = LoggerFactory.getLogger(FSOTableCompactor.class); + // private static final Logger LOG = LoggerFactory.getLogger(FSOTableCompactor.class); - private String nextBucket; - private String nextParentId; - private String nextKey; + private String nextBucket = null; + private String nextParentId = null; + // private String nextKey; public FSOTableCompactor(CompactorBuilder builder) { super(builder); @@ -85,7 +85,7 @@ private void processRange(KeyRange range, List ranges) { ranges.add(range); } nextParentId = null; - nextKey = null; + // nextKey = null; } else { String splitKey = findSplitKey(range); if (splitKey != null) { @@ -94,7 +94,7 @@ private void processRange(KeyRange range, List ranges) { if (needsCompaction(splitStats, getMinTombstones(), getTombstoneRatio())) { ranges.add(splitRange); } - nextKey = splitKey; + // nextKey = splitKey; } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java index bebb31ffc6de..80eff55ae78f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java @@ -33,7 +33,7 @@ public class OBSTableCompactor extends AbstractCompactor { private static final Logger LOG = LoggerFactory.getLogger(OBSTableCompactor.class); - private String nextBucket; + private String currentBucketUpperBound; private String nextKey; public OBSTableCompactor(CompactorBuilder builder) { @@ -42,70 +42,89 @@ public OBSTableCompactor(CompactorBuilder builder) { @Override public void run() { - + List ranges = getRangesNeedingCompaction(); + for (KeyRange range : ranges) { + addRangeCompactionTask(range); + } } @Override protected void collectRangesNeedingCompaction(List ranges) { - Iterator, CacheValue>> bucketIterator = getBucketIterator(); - while (bucketIterator.hasNext()) { - Map.Entry, CacheValue> entry = bucketIterator.next(); - String bucketKey = entry.getKey().getCacheKey(); - OmBucketInfo bucketInfo = entry.getValue().getCacheValue(); + long accumulatedEntries = 0; + if (nextKey != null) { + LOG.info("Last range has been splited, start from the next key {}", nextKey); + // TODO: handle pagination + } + + while (accumulatedEntries < getMaxEntriesSum()) { + KeyRange bucketBound = getNextBucketBound(); + if (bucketBound == null && accumulatedEntries == 0) { + // the last run may reach the end of the iterator, reset the iterator + currentBucketUpperBound = null; + bucketBound = getNextBucketBound(); + } - if (nextBucket != null && !nextBucket.equals(bucketKey)) { - continue; + if (bucketBound == null) { + currentBucketUpperBound = null; + break; } - KeyRange bucketRange = new KeyRange(bucketKey, getNextBucketKey(bucketKey)); - KeyRangeStats stats = getRangeStats(bucketRange); + currentBucketUpperBound = bucketBound.getEndKey(); + KeyRangeStats stats = getRangeStats(bucketBound); - if (stats.getNumEntries() <= getMaxEntriesSum()) { + if (accumulatedEntries + stats.getNumEntries() <= getMaxEntriesSum()) { // If the entire bucket fits within maxEntriesSum, check if it needs compaction if (needsCompaction(stats, getMinTombstones(), getTombstoneRatio())) { - ranges.add(bucketRange); + // TODO: merge consecutive ranges into one + ranges.add(bucketBound); + accumulatedEntries += stats.getNumEntries(); } - nextBucket = null; - nextKey = null; } else { - // Need to split the bucket range - String splitKey = findSplitKey(bucketRange); - if (splitKey != null) { - KeyRange splitRange = new KeyRange(bucketKey, splitKey); - KeyRangeStats splitStats = getRangeStats(splitRange); - if (needsCompaction(splitStats, getMinTombstones(), getTombstoneRatio())) { - ranges.add(splitRange); - } - nextBucket = bucketKey; - nextKey = splitKey; - break; - } + LOG.info("Bucket {} is too large, need to split", bucketBound); + // TODO:Need to split the bucket range + // String splitKey = findSplitKey(bucketBound); + // if (splitKey != null) { + // KeyRange splitRange = new KeyRange(bucketKey, splitKey); + // KeyRangeStats splitStats = getRangeStats(splitRange); + // if (needsCompaction(splitStats, getMinTombstones(), getTombstoneRatio())) { + // ranges.add(splitRange); + // } + // nextBucket = bucketKey; + // nextKey = splitKey; + // break; + // } } } } - private Iterator, CacheValue>> getBucketIterator() { - if (nextBucket != null) { - return getMetadataManager().getBucketIterator(nextBucket); + /** + * Get the next bucket bound. + * + * @return the next bucket bound, or null if reach the end of the iterator + */ + private KeyRange getNextBucketBound() { + Iterator, CacheValue>> iterator = getMetadataManager() + .getBucketIterator(currentBucketUpperBound); + if (!iterator.hasNext()) { + return null; } - return getMetadataManager().getBucketIterator(); - } - private String getNextBucketKey(String currentBucketKey) { - // For OBS and legacy layout, the next bucket key is the current bucket key + 1 - // This is a simplified version - in reality, you'd need to handle the key - // format - // based on your specific key structure - return currentBucketKey + "\0"; + String bucketStartKey = iterator.next().getKey().getCacheKey(), bucketEndKey; + if (iterator.hasNext()) { + bucketEndKey = iterator.next().getKey().getCacheKey(); + } else { + bucketEndKey = getKeyPrefixUpperBound(bucketStartKey); + } + return new KeyRange(bucketStartKey, bucketEndKey); } - private String findSplitKey(KeyRange range) { - // Binary search to find a split key that keeps the range under maxEntriesSum - String startKey = range.getStartKey(); - String endKey = range.getEndKey(); + // private String findSplitKey(KeyRange range) { + // // Binary search to find a split key that keeps the range under maxEntriesSum + // String startKey = range.getStartKey(); + // String endKey = range.getEndKey(); - // This is a simplified version - in reality, you'd need to implement - // a proper binary search based on your key format - return startKey + "\0"; - } + // // This is a simplified version - in reality, you'd need to implement + // // a proper binary search based on your key format + // return startKey + "\0"; + // } } From 8ee058ff73a39571b4c5fdbf04058904a3102424 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Mon, 19 May 2025 15:47:54 +0800 Subject: [PATCH 06/31] comment out FSO table compactor --- .../om/compaction/FSOTableCompactor.java | 178 +++++++++--------- 1 file changed, 89 insertions(+), 89 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOTableCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOTableCompactor.java index 7186cb22057a..9161ae7e1417 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOTableCompactor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOTableCompactor.java @@ -17,13 +17,13 @@ package org.apache.hadoop.ozone.om.compaction; -import java.util.Iterator; +// import java.util.Iterator; import java.util.List; -import java.util.Map; +// import java.util.Map; import org.apache.hadoop.hdds.utils.db.KeyRange; -import org.apache.hadoop.hdds.utils.db.cache.CacheKey; -import org.apache.hadoop.hdds.utils.db.cache.CacheValue; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +// import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +// import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +// import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; // import org.slf4j.Logger; // import org.slf4j.LoggerFactory; @@ -33,8 +33,8 @@ public class FSOTableCompactor extends AbstractCompactor { // private static final Logger LOG = LoggerFactory.getLogger(FSOTableCompactor.class); - private String nextBucket = null; - private String nextParentId = null; + // private String nextBucket = null; + // private String nextParentId = null; // private String nextKey; public FSOTableCompactor(CompactorBuilder builder) { @@ -47,88 +47,88 @@ public void run() { @Override protected void collectRangesNeedingCompaction(List ranges) { - Iterator, CacheValue>> bucketIterator = getBucketIterator(); - while (bucketIterator.hasNext()) { - Map.Entry, CacheValue> entry = bucketIterator.next(); - String bucketKey = entry.getKey().getCacheKey(); - OmBucketInfo bucketInfo = entry.getValue().getCacheValue(); - - if (nextBucket != null && !nextBucket.equals(bucketKey)) { - continue; - } - - // For FSO, we need to handle parent IDs - if (nextParentId != null) { - // Continue with the same bucket but different parent ID - KeyRange parentRange = new KeyRange( - getParentKey(bucketKey, nextParentId), - getNextParentKey(bucketKey, nextParentId)); - processRange(parentRange, ranges); - } else { - // Start with the first parent ID for this bucket - String firstParentId = getFirstParentId(bucketKey); - if (firstParentId != null) { - KeyRange parentRange = new KeyRange( - getParentKey(bucketKey, firstParentId), - getNextParentKey(bucketKey, firstParentId)); - processRange(parentRange, ranges); - } - } - } + // Iterator, CacheValue>> bucketIterator = getBucketIterator(); + // while (bucketIterator.hasNext()) { + // Map.Entry, CacheValue> entry = bucketIterator.next(); + // String bucketKey = entry.getKey().getCacheKey(); + // OmBucketInfo bucketInfo = entry.getValue().getCacheValue(); + + // if (nextBucket != null && !nextBucket.equals(bucketKey)) { + // continue; + // } + + // // For FSO, we need to handle parent IDs + // if (nextParentId != null) { + // // Continue with the same bucket but different parent ID + // KeyRange parentRange = new KeyRange( + // getParentKey(bucketKey, nextParentId), + // getNextParentKey(bucketKey, nextParentId)); + // processRange(parentRange, ranges); + // } else { + // // Start with the first parent ID for this bucket + // String firstParentId = getFirstParentId(bucketKey); + // if (firstParentId != null) { + // KeyRange parentRange = new KeyRange( + // getParentKey(bucketKey, firstParentId), + // getNextParentKey(bucketKey, firstParentId)); + // processRange(parentRange, ranges); + // } + // } + // } } - private void processRange(KeyRange range, List ranges) { - KeyRangeStats stats = getRangeStats(range); - - if (stats.getNumEntries() <= getMaxEntriesSum()) { - if (needsCompaction(stats, getMinTombstones(), getTombstoneRatio())) { - ranges.add(range); - } - nextParentId = null; - // nextKey = null; - } else { - String splitKey = findSplitKey(range); - if (splitKey != null) { - KeyRange splitRange = new KeyRange(range.getStartKey(), splitKey); - KeyRangeStats splitStats = getRangeStats(splitRange); - if (needsCompaction(splitStats, getMinTombstones(), getTombstoneRatio())) { - ranges.add(splitRange); - } - // nextKey = splitKey; - } - } - } - - private Iterator, CacheValue>> getBucketIterator() { - if (nextBucket != null) { - return getMetadataManager().getBucketIterator(nextBucket); - } - return getMetadataManager().getBucketIterator(); - } - - private String getParentKey(String bucketKey, String parentId) { - // Format: /volumeId/bucketId/parentId/ - return bucketKey + "/" + parentId + "/"; - } - - private String getNextParentKey(String bucketKey, String parentId) { - // Format: /volumeId/bucketId/parentId/ + 1 - return bucketKey + "/" + parentId + "/\0"; - } - - private String getFirstParentId(String bucketKey) { - // This is a simplified version - in reality, you'd need to implement - // a proper way to get the first parent ID for a bucket - return "0"; - } - - private String findSplitKey(KeyRange range) { - // Binary search to find a split key that keeps the range under maxEntriesSum - String startKey = range.getStartKey(); - String endKey = range.getEndKey(); - - // This is a simplified version - in reality, you'd need to implement - // a proper binary search based on your key format - return startKey + "\0"; - } + // private void processRange(KeyRange range, List ranges) { + // KeyRangeStats stats = getRangeStats(range); + + // if (stats.getNumEntries() <= getMaxEntriesSum()) { + // if (needsCompaction(stats, getMinTombstones(), getTombstoneRatio())) { + // ranges.add(range); + // } + // nextParentId = null; + // // nextKey = null; + // } else { + // String splitKey = findSplitKey(range); + // if (splitKey != null) { + // KeyRange splitRange = new KeyRange(range.getStartKey(), splitKey); + // KeyRangeStats splitStats = getRangeStats(splitRange); + // if (needsCompaction(splitStats, getMinTombstones(), getTombstoneRatio())) { + // ranges.add(splitRange); + // } + // // nextKey = splitKey; + // } + // } + // } + + // private Iterator, CacheValue>> getBucketIterator() { + // if (nextBucket != null) { + // return getMetadataManager().getBucketIterator(nextBucket); + // } + // return getMetadataManager().getBucketIterator(); + // } + + // private String getParentKey(String bucketKey, String parentId) { + // // Format: /volumeId/bucketId/parentId/ + // return bucketKey + "/" + parentId + "/"; + // } + + // private String getNextParentKey(String bucketKey, String parentId) { + // // Format: /volumeId/bucketId/parentId/ + 1 + // return bucketKey + "/" + parentId + "/\0"; + // } + + // private String getFirstParentId(String bucketKey) { + // // This is a simplified version - in reality, you'd need to implement + // // a proper way to get the first parent ID for a bucket + // return "0"; + // } + + // private String findSplitKey(KeyRange range) { + // // Binary search to find a split key that keeps the range under maxEntriesSum + // String startKey = range.getStartKey(); + // String endKey = range.getEndKey(); + + // // This is a simplified version - in reality, you'd need to implement + // // a proper binary search based on your key format + // return startKey + "\0"; + // } } From dbe59d30bf297ac407a5d05443aa712887962f01 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sat, 24 May 2025 15:15:53 +0800 Subject: [PATCH 07/31] Add CompoundKeyRangeStats class for managing key range statistics and enhance KeyRange constructor to support byte array inputs. Update OBSTableCompactor to utilize new CompoundKeyRangeStats for improved compaction logic. --- .../apache/hadoop/hdds/utils/db/KeyRange.java | 5 ++ .../om/compaction/CompoundKeyRangeStats.java | 55 ++++++++++++++++ .../om/compaction/OBSTableCompactor.java | 63 +++++++++++++++++++ 3 files changed, 123 insertions(+) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompoundKeyRangeStats.java diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java index 8f159dfd9fa9..c26958f2039b 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java @@ -34,6 +34,11 @@ public KeyRange(String startKey, String endKey) { this.endKey = endKey; } + public KeyRange(byte[] startKey, byte[] endKey) { + this.startKey = StringUtils.bytes2String(startKey); + this.endKey = StringUtils.bytes2String(endKey); + } + public String getStartKey() { return startKey; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompoundKeyRangeStats.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompoundKeyRangeStats.java new file mode 100644 index 000000000000..1b5326821988 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompoundKeyRangeStats.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.compaction; + +import java.util.List; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.utils.db.KeyRange; + +/** + * CompoundKeyRangeStats is a class that contains a list of KeyRangeStats and a + * compound KeyRangeStats. + */ +public class CompoundKeyRangeStats { + private KeyRangeStats compoundStats; + private List> keyRangeStatsList; + + public CompoundKeyRangeStats(List> keyRangeStatsList) { + this.keyRangeStatsList = keyRangeStatsList; + for (Pair entry : keyRangeStatsList) { + if (compoundStats == null) { + compoundStats = entry.getRight(); + } else { + compoundStats.add(entry.getRight()); + } + } + } + + public void index() { + // TODO: use some interval tree or segment tree to index the key range stats as + // the range may overlap with each other. + } + + public KeyRangeStats getCompoundStats() { + return compoundStats; + } + + public List> getKeyRangeStatsList() { + return keyRangeStatsList; + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java index 80eff55ae78f..a496f129ccf3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java @@ -17,16 +17,29 @@ package org.apache.hadoop.ozone.om.compaction; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; + +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.utils.db.KeyRange; +import org.apache.hadoop.hdds.utils.db.RDBStore; +import org.apache.hadoop.hdds.utils.db.RocksDatabaseException; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.rocksdb.LiveFileMetaData; +import org.rocksdb.TableProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.api.Distribution.Range; + /** * Compactor for OBS and legacy layout that compacts based on bucket ranges. */ @@ -118,6 +131,56 @@ private KeyRange getNextBucketBound() { return new KeyRange(bucketStartKey, bucketEndKey); } + private List prepareRanges() throws IOException { + // dd a function to prepare ranges first, start from `nextKey` or + // `currentBucketBoundary`. + // This can be done by getting the live file metadata list, then use + // `getPropsOfTableInRange` to retrieve SSTs in range `[nextKey, + // currentBucketBoundary]` or `[currentBucket, currentBucketBoundry]`, + // and if there are too many entries in these ranges, we need to further spilt + // them into smaller one, them save spilt point in `nextKey` for next round to + // use. + // In `collectRangesNeedingCompaction`, it call the previous + List ranges; + long accumulatedEntries = 0; + + while (accumulatedEntries <= getMaxEntriesSum()) { + if (nextKey != null) { + // handle pagination + } + } + String startKey, endKey; + + } + + private CompoundKeyRangeStats getCompoundKeyRangeStatsFromRange(KeyRange range) throws IOException { + List> keyRangeStatsList = new ArrayList<>(); + List liveFileMetaDataList = ((RDBStore)getDBStore()).getDb().getLiveFilesMetaData(); + Map fileMap = new HashMap<>(); + // https://github.com/facebook/rocksdb/blob/09cd25f76305f2110131f51068656ab392dc2bf5/db/version_set.cc#L1744-L1768 + // https://github.com/facebook/rocksdb/blob/a00391c72996a5dbdd93a621dbc53719c13b05c4/file/filename.cc#L140-L150 + for (LiveFileMetaData metaData : liveFileMetaDataList) { + fileMap.put(metaData.fileName(), metaData); + } + + Map propsMap = getDBStore().getPropertiesOfTableInRange(getTableName(), + Collections.singletonList(range)); + + for (Map.Entry entry : propsMap.entrySet()) { + String fullPath = entry.getKey(); + String fileName = fullPath.substring(fullPath.lastIndexOf('/') + 1); + LiveFileMetaData meta = fileMap.get(fileName); + if (meta == null) { + LOG.warn("LiveFileMetaData not found for file {}", fullPath); + continue; + } + KeyRange keyRange = new KeyRange(meta.smallestKey(), meta.largestKey()); + KeyRangeStats stats = KeyRangeStats.fromTableProperties(entry.getValue()); + keyRangeStatsList.add(Pair.of(keyRange, stats)); + } + return new CompoundKeyRangeStats(keyRangeStatsList); + } + // private String findSplitKey(KeyRange range) { // // Binary search to find a split key that keeps the range under maxEntriesSum // String startKey = range.getStartKey(); From 290219031bad371e1027ac7c83501cbfd052ce4e Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sun, 25 May 2025 15:01:10 +0800 Subject: [PATCH 08/31] Refactor range compaction configuration by renaming maxEntriesSum to maxCompactionEntries and adding rangesPerRun parameter. Update related classes to accommodate new configuration keys and improve compaction logic. --- .../apache/hadoop/ozone/om/OMConfigKeys.java | 11 ++++++---- .../om/compaction/AbstractCompactor.java | 14 +++++++++---- .../ozone/om/compaction/CompactorBuilder.java | 20 ++++++++++++++----- .../om/service/RangeCompactionService.java | 14 ++++++++----- 4 files changed, 41 insertions(+), 18 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 20d04a374a07..6de0c5ee2c75 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -667,15 +667,18 @@ public final class OMConfigKeys { public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_TIMEOUT = "ozone.om.range.compaction.service.timeout"; public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_TIMEOUT_DEFAULT = "10m"; - public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_MAX_ENTRIES_SUM - = "ozone.om.range.compaction.service.max.entries.sum"; - public static final int OZONE_OM_RANGE_COMPACTION_SERVICE_MAX_ENTRIES_SUM_DEFAULT = 1000000; + public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_MAX_COMPACTION_ENTRIES + = "ozone.om.range.compaction.service.max.compaction.entries"; + public static final long OZONE_OM_RANGE_COMPACTION_SERVICE_MAX_COMPACTION_ENTRIES_DEFAULT = 1_000_000; + public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_RANGES_PER_RUN + = "ozone.om.range.compaction.service.ranges.per.run"; + public static final int OZONE_OM_RANGE_COMPACTION_SERVICE_RANGES_PER_RUN_DEFAULT = 20; public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_TOMBSTONE_RATIO = "ozone.om.range.compaction.service.tombstone.ratio"; public static final double OZONE_OM_RANGE_COMPACTION_SERVICE_TOMBSTONE_RATIO_DEFAULT = 0.3; public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_MIN_TOMBSTONES = "ozone.om.range.compaction.service.min.tombstones"; - public static final int OZONE_OM_RANGE_COMPACTION_SERVICE_MIN_TOMBSTONES_DEFAULT = 10000; + public static final int OZONE_OM_RANGE_COMPACTION_SERVICE_MIN_TOMBSTONES_DEFAULT = 10_000; /** * Never constructed. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/AbstractCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/AbstractCompactor.java index 9299be2dd265..610b74f853d7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/AbstractCompactor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/AbstractCompactor.java @@ -42,14 +42,16 @@ public abstract class AbstractCompactor implements Compactor { private String tableName; private BackgroundTaskQueue compactRangeQueue; - private int maxEntriesSum; + private long maxCompactionEntries; private int minTombstones; private double tombstoneRatio; + private int rangesPerRun; protected AbstractCompactor(CompactorBuilder builder) { this.tableName = builder.getTableName(); - this.maxEntriesSum = builder.getMaxEntriesSum(); + this.maxCompactionEntries = builder.getMaxCompactionEntries(); this.minTombstones = builder.getMinTombstones(); + this.rangesPerRun = builder.getRangesPerRun(); this.tombstoneRatio = builder.getTombstoneRatio(); this.metadataManager = builder.getMetadataManager(); this.dbStore = builder.getDBStore(); @@ -100,8 +102,12 @@ protected OMMetadataManager getMetadataManager() { return metadataManager; } - protected int getMaxEntriesSum() { - return maxEntriesSum; + protected long getMaxCompactionEntries() { + return maxCompactionEntries; + } + + protected int getRangesPerRun() { + return rangesPerRun; } protected int getMinTombstones() { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompactorBuilder.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompactorBuilder.java index e2a7821cf996..5e674de13e85 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompactorBuilder.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompactorBuilder.java @@ -31,9 +31,10 @@ public class CompactorBuilder { private DBStore db; private BackgroundTaskQueue compactRangeQueue; private String tableName; - private int maxEntriesSum; + private long maxCompactionEntries; private int minTombstones; private double tombstoneRatio; + private int rangesPerRun; private OMMetadataManager metadataManager; @@ -57,8 +58,8 @@ public CompactorBuilder setTableName(String tableName) { return this; } - public CompactorBuilder setMaxEntriesSum(int maxEntriesSum) { - this.maxEntriesSum = maxEntriesSum; + public CompactorBuilder setMaxCompactionEntries(long maxCompactionEntries) { + this.maxCompactionEntries = maxCompactionEntries; return this; } @@ -72,6 +73,11 @@ public CompactorBuilder setTombstoneRatio(double tombstoneRatio) { return this; } + public CompactorBuilder setRangesPerRun(int rangesPerRun) { + this.rangesPerRun = rangesPerRun; + return this; + } + public Compactor build() throws IOException { if (tableName == null) { throw new IllegalArgumentException("Name and table must be set"); @@ -96,8 +102,8 @@ public String getTableName() { return tableName; } - public int getMaxEntriesSum() { - return maxEntriesSum; + public long getMaxCompactionEntries() { + return maxCompactionEntries; } public int getMinTombstones() { @@ -119,4 +125,8 @@ public OMMetadataManager getMetadataManager() { public BackgroundTaskQueue getCompactRangeQueue() { return compactRangeQueue; } + + public int getRangesPerRun() { + return rangesPerRun; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java index 776d2f9b0471..3069bdcd84fc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java @@ -47,7 +47,8 @@ public class RangeCompactionService extends BackgroundService { private final List compactors; private final Map compactorExecutors; private final long checkIntervalMs; - private final int maxEntriesSum; + private final long maxCompactionEntries; + private final int rangesPerRun; private final int minTombstones; private final double tombstoneRatio; @@ -59,8 +60,10 @@ public RangeCompactionService(OzoneConfiguration conf, OzoneManager ozoneManager ozoneManager.getThreadNamePrefix()); this.metadataManager = ozoneManager.getMetadataManager(); this.checkIntervalMs = intervalMs; - this.maxEntriesSum = conf.getInt(OMConfigKeys.OZONE_OM_RANGE_COMPACTION_SERVICE_MAX_ENTRIES_SUM, - OMConfigKeys.OZONE_OM_RANGE_COMPACTION_SERVICE_MAX_ENTRIES_SUM_DEFAULT); + this.maxCompactionEntries = conf.getLong(OMConfigKeys.OZONE_OM_RANGE_COMPACTION_SERVICE_MAX_COMPACTION_ENTRIES, + OMConfigKeys.OZONE_OM_RANGE_COMPACTION_SERVICE_MAX_COMPACTION_ENTRIES_DEFAULT); + this.rangesPerRun = conf.getInt(OMConfigKeys.OZONE_OM_RANGE_COMPACTION_SERVICE_RANGES_PER_RUN, + OMConfigKeys.OZONE_OM_RANGE_COMPACTION_SERVICE_RANGES_PER_RUN_DEFAULT); this.minTombstones = conf.getInt(OMConfigKeys.OZONE_OM_RANGE_COMPACTION_SERVICE_MIN_TOMBSTONES, OMConfigKeys.OZONE_OM_RANGE_COMPACTION_SERVICE_MIN_TOMBSTONES_DEFAULT); this.tombstoneRatio = conf.getDouble(OMConfigKeys.OZONE_OM_RANGE_COMPACTION_SERVICE_TOMBSTONE_RATIO, @@ -106,9 +109,10 @@ private void initializeCompactors() { .setCompactRangeQueue(tasks) .setMetadataManager(metadataManager) .setDBStore(metadataManager.getStore()) - .setMaxEntriesSum(maxEntriesSum) + .setMaxCompactionEntries(maxCompactionEntries) .setMinTombstones(minTombstones) - .setTombstoneRatio(tombstoneRatio); + .setTombstoneRatio(tombstoneRatio) + .setRangesPerRun(rangesPerRun); // Initialize compactors for OBS and legacy layout try { From dcb840fdfefadc6daa1aed083996f5537faf538f Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sun, 25 May 2025 15:46:13 +0800 Subject: [PATCH 09/31] Enhance StringUtils with new methods for key prefix upper bound, max, and min string comparisons. Refactor KeyRange and RDBStore to use updated method names. Update OBSTableCompactor to utilize StringUtils for key prefix handling and improve compaction logic. Add new methods to CompoundKeyRangeStats for better statistics management. --- .../org/apache/hadoop/hdds/StringUtils.java | 15 ++ .../apache/hadoop/hdds/utils/db/KeyRange.java | 2 +- .../apache/hadoop/hdds/utils/db/RDBStore.java | 2 +- .../om/compaction/AbstractCompactor.java | 6 - .../om/compaction/CompoundKeyRangeStats.java | 34 ++- .../ozone/om/compaction/KeyRangeStats.java | 5 + .../om/compaction/OBSTableCompactor.java | 215 ++++++++++++------ 7 files changed, 192 insertions(+), 87 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java index e2c50a2da740..e41489002b79 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java @@ -97,4 +97,19 @@ public static String bytes2String(byte[] bytes) { public static byte[] string2Bytes(String str) { return str.getBytes(UTF8); } + + public static String getKeyPrefixUpperBound(String key) { + return key.substring(0, key.length() - 1) + (char)(key.charAt(key.length() - 1) + 1); + } + + /** + * Return the max string between str1 and str2, if both are null, return null. + */ + public static String max(String str1, String str2) { + return str1.compareTo(str2) > 0 ? str1 : str2; + } + + public static String min(String str1, String str2) { + return str1.compareTo(str2) < 0 ? str1 : str2; + } } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java index c26958f2039b..6d97746034e4 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java @@ -47,7 +47,7 @@ public String getEndKey() { return endKey; } - public ManagedRange toRocksRange() throws IOException { + public ManagedRange toManagedRange() throws IOException { return new ManagedRange( new ManagedSlice(StringUtils.string2Bytes(startKey)), new ManagedSlice(StringUtils.string2Bytes(endKey))); diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index e67c9c6d1a4a..6d1c027625cd 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -275,7 +275,7 @@ public Map getPropertiesOfTableInRange(String tableName List rocksRanges = ranges.stream() .map(t -> { try { - return t.toRocksRange(); + return t.toManagedRange(); } catch (IOException e) { LOG.error("Failed to convert {} to RocksDB Range", t, e); return null; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/AbstractCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/AbstractCompactor.java index 610b74f853d7..1b552d57efdb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/AbstractCompactor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/AbstractCompactor.java @@ -126,12 +126,6 @@ protected void addRangeCompactionTask(KeyRange range) { compactRangeQueue.add(new CompactionTask(range)); } - protected String getKeyPrefixUpperBound(String keyPrefix) { - char[] upperBoundCharArray = keyPrefix.toCharArray(); - upperBoundCharArray[upperBoundCharArray.length - 1] += 1; - return String.valueOf(upperBoundCharArray); - } - private class CompactionTask implements BackgroundTask { private final KeyRange range; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompoundKeyRangeStats.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompoundKeyRangeStats.java index 1b5326821988..c421ac07fdca 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompoundKeyRangeStats.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompoundKeyRangeStats.java @@ -40,11 +40,6 @@ public CompoundKeyRangeStats(List> keyRangeStatsLi } } - public void index() { - // TODO: use some interval tree or segment tree to index the key range stats as - // the range may overlap with each other. - } - public KeyRangeStats getCompoundStats() { return compoundStats; } @@ -52,4 +47,33 @@ public KeyRangeStats getCompoundStats() { public List> getKeyRangeStatsList() { return keyRangeStatsList; } + + public int size() { + return keyRangeStatsList.size(); + } + + public long getNumEntries() { + return compoundStats.getNumEntries(); + } + + public long getNumDeletion() { + return compoundStats.getNumDeletion(); + } + + public double getTombstoneRatio() { + return compoundStats.getTombstoneRatio(); + } + + public void add(CompoundKeyRangeStats other) { + compoundStats.add(other.getCompoundStats()); + keyRangeStatsList.addAll(other.getKeyRangeStatsList()); + } + + @Override + public String toString() { + return "CompoundKeyRangeStats{" + + "compoundStats=" + compoundStats + + ", keyRangeStatsList=" + keyRangeStatsList + + '}'; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/KeyRangeStats.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/KeyRangeStats.java index a7de0c02b44c..8da09ccff4a1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/KeyRangeStats.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/KeyRangeStats.java @@ -26,6 +26,11 @@ public class KeyRangeStats { private long numEntries; private long numDeletion; + public KeyRangeStats() { + this.numEntries = 0; + this.numDeletion = 0; + } + public KeyRangeStats(long numEntries, long numDeletion) { this.numEntries = numEntries; this.numDeletion = numDeletion; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java index a496f129ccf3..51d786515169 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.om.compaction; +import com.google.common.base.Preconditions; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -24,12 +25,10 @@ import java.util.Iterator; import java.util.List; import java.util.Map; - import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.utils.db.KeyRange; import org.apache.hadoop.hdds.utils.db.RDBStore; -import org.apache.hadoop.hdds.utils.db.RocksDatabaseException; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -38,8 +37,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.api.Distribution.Range; - /** * Compactor for OBS and legacy layout that compacts based on bucket ranges. */ @@ -63,49 +60,39 @@ public void run() { @Override protected void collectRangesNeedingCompaction(List ranges) { - long accumulatedEntries = 0; - if (nextKey != null) { - LOG.info("Last range has been splited, start from the next key {}", nextKey); - // TODO: handle pagination - } - - while (accumulatedEntries < getMaxEntriesSum()) { - KeyRange bucketBound = getNextBucketBound(); - if (bucketBound == null && accumulatedEntries == 0) { - // the last run may reach the end of the iterator, reset the iterator - currentBucketUpperBound = null; - bucketBound = getNextBucketBound(); - } - - if (bucketBound == null) { - currentBucketUpperBound = null; - break; - } - - currentBucketUpperBound = bucketBound.getEndKey(); - KeyRangeStats stats = getRangeStats(bucketBound); - - if (accumulatedEntries + stats.getNumEntries() <= getMaxEntriesSum()) { - // If the entire bucket fits within maxEntriesSum, check if it needs compaction - if (needsCompaction(stats, getMinTombstones(), getTombstoneRatio())) { - // TODO: merge consecutive ranges into one - ranges.add(bucketBound); - accumulatedEntries += stats.getNumEntries(); + for (int i = 0; i < getRangesPerRun(); i++) { + try { + Pair preparedRange = prepareRanges(); + if (preparedRange == null) { + continue; } - } else { - LOG.info("Bucket {} is too large, need to split", bucketBound); - // TODO:Need to split the bucket range - // String splitKey = findSplitKey(bucketBound); - // if (splitKey != null) { - // KeyRange splitRange = new KeyRange(bucketKey, splitKey); - // KeyRangeStats splitStats = getRangeStats(splitRange); - // if (needsCompaction(splitStats, getMinTombstones(), getTombstoneRatio())) { - // ranges.add(splitRange); - // } - // nextBucket = bucketKey; - // nextKey = splitKey; - // break; - // } + // TODO: merge consecutive ranges if they aren't too big, notice the situation that iterator has been reset + // to be more specific, + // constain 1: temp start key and end key are either null or not null at the same time, called them temp range + // constain 2: if temp range is submitted, clean temp range + // if range is null: + // 1. if temp range is not null, then submit and reset the temp range + // otherwise, do nothing + // if range is a spiltted range: + // 1. if temp range is null, submit the range directly + // 2. if temp range is not null and temp accumlated numEntries plus the current range's numEntries + // is greater than maxEntries, then submit **temp range and the current range** + // 3. if temp range is not null, and temp accumlated numEntries plus the current range's numEntries + // is less than maxEntries, then submit **a range that composed of temp range and the current range** + // (notice the difference between point 2, 3) + // if range is not a splitted range: + // 1. if temp range is null, update it to the current range's start key, and temp end key as well + // 2. if temp range is not null and temp accumlated numEntries plus the current range's numEntries + // is greater than maxEntries, then submit temp range and set temp range to the current range + // 3. if temp range is not null and temp accumlated numEntries plus the current range's numEntries + // is less than maxEntries, + // then **set temp range to a range that composed of temp range and the current range** + // (notice the difference between point 2, 3) + if (needsCompaction(preparedRange.getRight(), getMinTombstones(), getTombstoneRatio())) { + addRangeCompactionTask(preparedRange.getLeft()); + } + } catch (IOException e) { + LOG.error("Failed to prepare ranges for compaction", e); } } } @@ -126,31 +113,121 @@ private KeyRange getNextBucketBound() { if (iterator.hasNext()) { bucketEndKey = iterator.next().getKey().getCacheKey(); } else { - bucketEndKey = getKeyPrefixUpperBound(bucketStartKey); + bucketEndKey = StringUtils.getKeyPrefixUpperBound(bucketStartKey); } return new KeyRange(bucketStartKey, bucketEndKey); } - private List prepareRanges() throws IOException { - // dd a function to prepare ranges first, start from `nextKey` or - // `currentBucketBoundary`. - // This can be done by getting the live file metadata list, then use - // `getPropsOfTableInRange` to retrieve SSTs in range `[nextKey, - // currentBucketBoundary]` or `[currentBucket, currentBucketBoundry]`, - // and if there are too many entries in these ranges, we need to further spilt - // them into smaller one, them save spilt point in `nextKey` for next round to - // use. - // In `collectRangesNeedingCompaction`, it call the previous - List ranges; - long accumulatedEntries = 0; - - while (accumulatedEntries <= getMaxEntriesSum()) { - if (nextKey != null) { - // handle pagination + /** + * Prepare ranges for compaction. + * + * @return the prepared ranges, the range and its stats in pair may be **combined** by multiple ranges + * @throws IOException if an error occurs + */ + private Pair prepareRanges() throws IOException { + + // Get the initial range to process + KeyRange currentRange; + if (nextKey != null) { + // Continue from last split point + currentRange = new KeyRange(nextKey, currentBucketUpperBound); + // clean up the nextKey + nextKey = null; + } else { + // Start from next bucket boundary + KeyRange bucketBound = getNextBucketBound(); + if (bucketBound == null) { + return null; + } + currentRange = new KeyRange(bucketBound.getStartKey(), bucketBound.getEndKey()); + } + + // Get compound stats for the range + CompoundKeyRangeStats compoundStats = getCompoundKeyRangeStatsFromRange(currentRange); + + if (compoundStats.getNumEntries() <= getMaxCompactionEntries()) { + return Pair.of(currentRange, compoundStats.getCompoundStats()); + } else { + LOG.info("max compaction entries is {}, but range {} has {} entries, splitting", + getMaxCompactionEntries(), currentRange, compoundStats.getNumEntries()); + List> splittedRanges = findFitRanges( + compoundStats.getKeyRangeStatsList(), getMaxCompactionEntries()); + if (splittedRanges == null || splittedRanges.size() == 0) { + LOG.warn("splitted ranges is null or empty, return null, current range: {}", currentRange); + return null; + } + Pair squashedRange = squashRanges(splittedRanges); + String squashedRangeEndKeyUpperBound = StringUtils.getKeyPrefixUpperBound(squashedRange.getLeft().getEndKey()); + if (currentRange.getEndKey().compareTo(squashedRangeEndKeyUpperBound) > 0) { + // if the squashed range is not the last range, then we need to split the range + nextKey = squashedRangeEndKeyUpperBound; + LOG.info("squashed range [{}] is not the last range, need to split, next key: {}", + squashedRange, nextKey); + } else { + LOG.info("squashed range [{}] is the last range, no need to split, " + + "probably means there are some SSTs that are ignored", squashedRange); + } + return squashedRange; + } + } + + /** + * return a list of ranges that each range's numEntries ≤ maxEntries, + * and their sum of numEntries are not greater than maxEntries, too. + */ + private List> findFitRanges( + List> ranges, long maxEntries) { + + ranges.sort((a, b) -> a.getLeft().getStartKey().compareTo(b.getLeft().getStartKey())); + + List> out = new ArrayList<>(); + + KeyRangeStats chunkStats = new KeyRangeStats(); + + for (int i = 0; i < ranges.size(); i++) { + Pair range = ranges.get(i); + long n = range.getRight().getNumEntries(); + + // this single SST already exceeds the threshold + if (n > maxEntries) { + LOG.warn("Single SST [{}] holds {} entries (> maxEntries {}), it will be ignored.", + range.getLeft(), n, maxEntries); + continue; + } + + // adding this SST would overflow the threshold + if (chunkStats.getNumEntries() > 0 && chunkStats.getNumEntries() + n > maxEntries) { + LOG.info("Chunk stats [{}] + range [{}]'s numEntries [{}] > maxEntries [{}], fit ranges found", + chunkStats, range, n, maxEntries); + return out; } + + /* add current SST into (possibly new) chunk */ + out.add(range); + chunkStats.add(range.getRight()); } - String startKey, endKey; + return out; + } + + /** + * Build a KeyRange (start inclusive, end exclusive) that encloses the + * SSTs between firstIdx and lastIdx (inclusive). Uses + * getKeyPrefixUpperBound() to make the end key exclusive. + */ + private Pair squashRanges( + List> list) { + Preconditions.checkNotNull(list, "list is null"); + Preconditions.checkArgument(list.size() > 0, "list is empty"); + + String minKey = list.get(0).getLeft().getStartKey(), maxKey = list.get(0).getLeft().getEndKey(); + KeyRangeStats stats = new KeyRangeStats(); + for (Pair range : list) { + minKey = StringUtils.min(minKey, range.getLeft().getStartKey()); + maxKey = StringUtils.max(maxKey, range.getLeft().getEndKey()); + stats.add(range.getRight()); + } + return Pair.of(new KeyRange(minKey, maxKey), stats); } private CompoundKeyRangeStats getCompoundKeyRangeStatsFromRange(KeyRange range) throws IOException { @@ -180,14 +257,4 @@ private CompoundKeyRangeStats getCompoundKeyRangeStatsFromRange(KeyRange range) } return new CompoundKeyRangeStats(keyRangeStatsList); } - - // private String findSplitKey(KeyRange range) { - // // Binary search to find a split key that keeps the range under maxEntriesSum - // String startKey = range.getStartKey(); - // String endKey = range.getEndKey(); - - // // This is a simplified version - in reality, you'd need to implement - // // a proper binary search based on your key format - // return startKey + "\0"; - // } } From 63d65cb7a3d0f90ab3d98517f819fb50db2d9e96 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sun, 25 May 2025 15:59:24 +0800 Subject: [PATCH 10/31] fix pmd --- .../hadoop/ozone/om/compaction/OBSTableCompactor.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java index 51d786515169..035d578c3531 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java @@ -152,7 +152,7 @@ private Pair prepareRanges() throws IOException { getMaxCompactionEntries(), currentRange, compoundStats.getNumEntries()); List> splittedRanges = findFitRanges( compoundStats.getKeyRangeStatsList(), getMaxCompactionEntries()); - if (splittedRanges == null || splittedRanges.size() == 0) { + if (splittedRanges == null || splittedRanges.isEmpty()) { LOG.warn("splitted ranges is null or empty, return null, current range: {}", currentRange); return null; } @@ -184,8 +184,7 @@ private List> findFitRanges( KeyRangeStats chunkStats = new KeyRangeStats(); - for (int i = 0; i < ranges.size(); i++) { - Pair range = ranges.get(i); + for (Pair range : ranges) { long n = range.getRight().getNumEntries(); // this single SST already exceeds the threshold @@ -218,7 +217,7 @@ private List> findFitRanges( private Pair squashRanges( List> list) { Preconditions.checkNotNull(list, "list is null"); - Preconditions.checkArgument(list.size() > 0, "list is empty"); + Preconditions.checkArgument(!list.isEmpty(), "list is empty"); String minKey = list.get(0).getLeft().getStartKey(), maxKey = list.get(0).getLeft().getEndKey(); KeyRangeStats stats = new KeyRangeStats(); From 9ac937b3e7a9abea1c606e9708bec28a9878f7c0 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sun, 25 May 2025 16:36:01 +0800 Subject: [PATCH 11/31] Rename "deletedDirTable" to "deletedDirectoryTable" in RangeCompactionService for improved clarity. --- .../apache/hadoop/ozone/om/service/RangeCompactionService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java index 3069bdcd84fc..376a81051c7b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java @@ -126,7 +126,7 @@ private void initializeCompactors() { .build()); compactors.add(builder.setTableName("fileTable") .build()); - compactors.add(builder.setTableName("deletedDirTable") + compactors.add(builder.setTableName("deletedDirectoryTable") .build()); } catch (IOException e) { LOG.error("Failed to initialize compactors", e); From 0de2e265c901b210111af737b88eca1d0648a3fe Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sun, 25 May 2025 17:33:24 +0800 Subject: [PATCH 12/31] Refactor ManagedRange to implement AutoCloseable, ensuring proper resource management by closing ManagedSlices. Update KeyRange and RocksDatabase to handle exceptions and close resources appropriately during range operations. --- .../org/apache/hadoop/hdds/utils/db/KeyRange.java | 12 +++++++++--- .../apache/hadoop/hdds/utils/db/RocksDatabase.java | 6 +++++- .../hadoop/hdds/utils/db/managed/ManagedRange.java | 12 +++++++++++- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java index 6d97746034e4..9a7eb809f6bb 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java @@ -48,9 +48,15 @@ public String getEndKey() { } public ManagedRange toManagedRange() throws IOException { - return new ManagedRange( - new ManagedSlice(StringUtils.string2Bytes(startKey)), - new ManagedSlice(StringUtils.string2Bytes(endKey))); + ManagedSlice start = new ManagedSlice(StringUtils.string2Bytes(startKey)); + ManagedSlice end = new ManagedSlice(StringUtils.string2Bytes(endKey)); + try { + return new ManagedRange(start, end); + } catch (Exception e) { + start.close(); + end.close(); + throw e; + } } @Override diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java index 65e89cd7cc1a..54377c5f47f4 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java @@ -885,7 +885,11 @@ public void deleteFilesNotMatchingPrefix(Map prefixPairs) throws public Map getPropertiesOfColumnFamilyInRange(ColumnFamily columnFamily, List ranges) throws RocksDatabaseException { - return db.getPropertiesOfColumnFamilyInRange(columnFamily.getHandle(), ranges); + try { + return db.getPropertiesOfColumnFamilyInRange(columnFamily.getHandle(), ranges); + } finally { + ranges.forEach(ManagedRange::close); + } } @Override diff --git a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRange.java b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRange.java index ad573dc44319..04666d5a31e5 100644 --- a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRange.java +++ b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRange.java @@ -22,15 +22,25 @@ /** * Managed {@link Range}. */ -public class ManagedRange { +public class ManagedRange implements AutoCloseable { private final Range original; + private final ManagedSlice start; + private final ManagedSlice limit; public ManagedRange(final ManagedSlice start, final ManagedSlice limit) { + this.start = start; + this.limit = limit; this.original = new Range(start, limit); } public Range getOriginal() { return original; } + + @Override + public void close() { + start.close(); + limit.close(); + } } From c6dc4938d5523b7a5f2c7af615c13afa6859f3b5 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sun, 25 May 2025 17:56:57 +0800 Subject: [PATCH 13/31] Update Javadoc in DBStore to clarify parameter naming from 'columnFamily' to 'tableName' for improved documentation accuracy. --- .../src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java index 82c96bf19fff..85fdccd807a7 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java @@ -133,7 +133,7 @@ void compactTable(String tableName, String startKey, String endKey, /** * Get the properties of a column family in a range. - * @param columnFamily - The column family to get the properties of. + * @param tableName - The table name to get the properties of. * @param startKey - The starting key of the range. * @param endKey - The ending key of the range. * @return - The properties of the column family in the range. From 8ece80872cf3147853a875f41c767cd37bdbc587 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sun, 25 May 2025 21:03:35 +0800 Subject: [PATCH 14/31] Add range compaction service configuration to ozone-default.xml Introduce new properties for the range compaction service, including enabling the service, setting compaction intervals, and defining thresholds for tombstones and entries. These enhancements aim to improve performance and storage efficiency by managing data compaction based on tombstone density. --- .../src/main/resources/ozone-default.xml | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 1544492cd72b..cb851ee07f79 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -4782,4 +4782,54 @@ warm up edek cache if none of key successful on OM start up. + + + + ozone.om.range.compaction.service.enabled + true + OZONE, OM, PERFORMANCE + Enable or disable the range compaction service that compacts specific ranges of data based on tombstone ratio. This service helps improve performance and reduce storage overhead by compacting ranges with high tombstone density. + + + + ozone.om.range.compaction.service.interval + 1s + OZONE, OM, PERFORMANCE + The interval at which the range compaction service checks for ranges that need compaction. The service will scan for ranges with high tombstone density at this interval. + + + + ozone.om.range.compaction.service.max.compaction.entries + 1000000 + OZONE, OM, PERFORMANCE + The maximum number of entries that can be compacted in a single range compaction operation. This limit helps prevent long-running compaction operations that could impact system performance. + + + + ozone.om.range.compaction.service.min.tombstones + 10000 + OZONE, OM, PERFORMANCE + The minimum number of tombstones required in a range before it is considered for compaction. This threshold helps ensure that only ranges with significant tombstone density are compacted. + + + + ozone.om.range.compaction.service.ranges.per.run + 20 + OZONE, OM, PERFORMANCE + The number of ranges to process in each run of the range compaction service. This controls the granularity of compaction operations and helps balance system load. + + + + ozone.om.range.compaction.service.timeout + 10m + OZONE, OM, PERFORMANCE + The timeout for range compaction operations. If a compaction operation takes longer than this, it will be cancelled to prevent long-running operations from impacting system performance. + + + + ozone.om.range.compaction.service.tombstone.ratio + 0.3 + OZONE, OM, PERFORMANCE + The ratio of tombstones to total entries that triggers compaction of a range. When the ratio exceeds this threshold, the range will be considered for compaction to improve storage efficiency. + From 86a9a693fb1a2161b2aef2dd135fc8b328ec96fd Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 27 May 2025 11:32:55 +0800 Subject: [PATCH 15/31] Minor adjustment and fixes --- .../main/java/org/apache/hadoop/hdds/StringUtils.java | 3 --- .../common/src/main/resources/ozone-default.xml | 2 +- .../java/org/apache/hadoop/ozone/om/OMConfigKeys.java | 2 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 4 ++++ .../hadoop/ozone/om/compaction/OBSTableCompactor.java | 2 +- .../ozone/om/service/RangeCompactionService.java | 11 ++--------- 6 files changed, 9 insertions(+), 15 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java index e41489002b79..eb85d8e7dc11 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java @@ -102,9 +102,6 @@ public static String getKeyPrefixUpperBound(String key) { return key.substring(0, key.length() - 1) + (char)(key.charAt(key.length() - 1) + 1); } - /** - * Return the max string between str1 and str2, if both are null, return null. - */ public static String max(String str1, String str2) { return str1.compareTo(str2) > 0 ? str1 : str2; } diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index cb851ee07f79..616dd4247d3a 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -4793,7 +4793,7 @@ ozone.om.range.compaction.service.interval - 1s + 30s OZONE, OM, PERFORMANCE The interval at which the range compaction service checks for ranges that need compaction. The service will scan for ranges with high tombstone density at this interval. diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 6de0c5ee2c75..d2fa3cbd6b48 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -663,7 +663,7 @@ public final class OMConfigKeys { public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_ENABLED = "ozone.om.range.compaction.service.enabled"; public static final boolean OZONE_OM_RANGE_COMPACTION_SERVICE_ENABLED_DEFAULT = true; public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_INTERVAL = "ozone.om.range.compaction.service.interval"; - public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_INTERVAL_DEFAULT = "1s"; + public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_INTERVAL_DEFAULT = "30s"; public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_TIMEOUT = "ozone.om.range.compaction.service.timeout"; public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_TIMEOUT_DEFAULT = "10m"; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 9676f2c26b83..f9eac7018dc7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -454,6 +454,10 @@ public void stop() throws IOException { compactionService.shutdown(); compactionService = null; } + if (rangeCompactionService != null) { + rangeCompactionService.shutdown(); + rangeCompactionService = null; + } } private OmBucketInfo getBucketInfo(String volumeName, String bucketName) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java index 035d578c3531..185b195120b0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java @@ -89,7 +89,7 @@ protected void collectRangesNeedingCompaction(List ranges) { // then **set temp range to a range that composed of temp range and the current range** // (notice the difference between point 2, 3) if (needsCompaction(preparedRange.getRight(), getMinTombstones(), getTombstoneRatio())) { - addRangeCompactionTask(preparedRange.getLeft()); + ranges.add(preparedRange.getLeft()); } } catch (IOException e) { LOG.error("Failed to prepare ranges for compaction", e); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java index 376a81051c7b..79925abbc415 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java @@ -72,8 +72,7 @@ public RangeCompactionService(OzoneConfiguration conf, OzoneManager ozoneManager this.compactorExecutors = new ConcurrentHashMap<>(); this.tasks = new BackgroundTaskQueue(); - - start(); + initializeCompactors(); } @Override @@ -83,7 +82,6 @@ public BackgroundTaskQueue getTasks() { @Override public void start() { - initializeCompactors(); scheduleCompactionChecks(); } @@ -132,11 +130,6 @@ private void initializeCompactors() { LOG.error("Failed to initialize compactors", e); throw new RuntimeException("Failed to initialize compactors", e); } - - // Initialize all compactors - // for (Compactor compactor : compactors) { - // compactor.init(metadataManager, dbStore); - // } } private void scheduleCompactionChecks() { @@ -147,7 +140,7 @@ private void scheduleCompactionChecks() { executor.scheduleWithFixedDelay( compactor::run, - checkIntervalMs, + checkIntervalMs, // TODO: randomize the start time checkIntervalMs, TimeUnit.MILLISECONDS); } From 2ef8b4553bd231b62f6b2d615fa34314a1e26e34 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 27 May 2025 14:30:44 +0800 Subject: [PATCH 16/31] Refactor key range handling by removing ManagedRange and replacing it with RocksDB's Range. Update RDBStore and RocksDatabase to utilize the new Range implementation, ensuring proper resource management and exception handling during range operations. Additionally, import the new Range class in pom.xml. --- .../apache/hadoop/hdds/utils/db/KeyRange.java | 15 ------ .../apache/hadoop/hdds/utils/db/RDBStore.java | 20 ++++---- .../hadoop/hdds/utils/db/RocksDatabase.java | 12 ++--- .../hdds/utils/db/managed/ManagedRange.java | 46 ------------------- .../hdds/utils/db/managed/ManagedRocksDB.java | 12 ----- pom.xml | 1 + 6 files changed, 18 insertions(+), 88 deletions(-) delete mode 100644 hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRange.java diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java index 9a7eb809f6bb..103707e29e14 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/KeyRange.java @@ -17,10 +17,7 @@ package org.apache.hadoop.hdds.utils.db; -import java.io.IOException; import org.apache.hadoop.hdds.StringUtils; -import org.apache.hadoop.hdds.utils.db.managed.ManagedRange; -import org.apache.hadoop.hdds.utils.db.managed.ManagedSlice; /** * Represents a range of keys for compaction. @@ -47,18 +44,6 @@ public String getEndKey() { return endKey; } - public ManagedRange toManagedRange() throws IOException { - ManagedSlice start = new ManagedSlice(StringUtils.string2Bytes(startKey)); - ManagedSlice end = new ManagedSlice(StringUtils.string2Bytes(endKey)); - try { - return new ManagedRange(start, end); - } catch (Exception e) { - start.close(); - end.close(); - throw e; - } - } - @Override public String toString() { return "KeyRange{" + diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index 6d1c027625cd..faa88177bd0c 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -48,12 +48,13 @@ import org.apache.hadoop.hdds.utils.db.cache.TableCache; import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions; -import org.apache.hadoop.hdds.utils.db.managed.ManagedRange; +import org.apache.hadoop.hdds.utils.db.managed.ManagedSlice; import org.apache.hadoop.hdds.utils.db.managed.ManagedStatistics; import org.apache.hadoop.hdds.utils.db.managed.ManagedTransactionLogIterator; import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteOptions; import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer; import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.RocksDBCheckpointDifferHolder; +import org.rocksdb.Range; import org.rocksdb.RocksDBException; import org.rocksdb.TableProperties; import org.rocksdb.TransactionLogIterator.BatchResult; @@ -272,18 +273,19 @@ public Map getPropertiesOfTableInRange(String tableName throw new IOException("No such table in this DB. TableName : " + tableName); } - List rocksRanges = ranges.stream() + List rocksRanges = ranges.stream() .map(t -> { - try { - return t.toManagedRange(); - } catch (IOException e) { - LOG.error("Failed to convert {} to RocksDB Range", t, e); - return null; - } + ManagedSlice start = new ManagedSlice(StringUtils.string2Bytes(t.getStartKey())); + ManagedSlice end = new ManagedSlice(StringUtils.string2Bytes(t.getEndKey())); + return new Range(start, end); }) .filter(Objects::nonNull) .collect(Collectors.toList()); - return db.getPropertiesOfColumnFamilyInRange(columnFamily, rocksRanges); + try { + return db.getPropertiesOfColumnFamilyInRange(columnFamily, rocksRanges); + } catch (RocksDatabaseException e) { + throw new IOException("Failed to get properties of table in range", e); + } } @Override diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java index 54377c5f47f4..d61a3e4d130b 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java @@ -48,7 +48,6 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedFlushOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedIngestExternalFileOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions; -import org.apache.hadoop.hdds.utils.db.managed.ManagedRange; import org.apache.hadoop.hdds.utils.db.managed.ManagedReadOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator; @@ -64,6 +63,7 @@ import org.rocksdb.Holder; import org.rocksdb.KeyMayExist; import org.rocksdb.LiveFileMetaData; +import org.rocksdb.Range; import org.rocksdb.RocksDBException; import org.rocksdb.TableProperties; import org.slf4j.Logger; @@ -884,11 +884,11 @@ public void deleteFilesNotMatchingPrefix(Map prefixPairs) throws } public Map getPropertiesOfColumnFamilyInRange(ColumnFamily columnFamily, - List ranges) throws RocksDatabaseException { - try { - return db.getPropertiesOfColumnFamilyInRange(columnFamily.getHandle(), ranges); - } finally { - ranges.forEach(ManagedRange::close); + List ranges) throws RocksDatabaseException { + try (UncheckedAutoCloseable ignored = acquire()) { + return db.get().getPropertiesOfTablesInRange(columnFamily.getHandle(), ranges); + } catch (RocksDBException e) { + throw new RocksDatabaseException("Failed to get properties of column family in range", e); } } diff --git a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRange.java b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRange.java deleted file mode 100644 index 04666d5a31e5..000000000000 --- a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRange.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdds.utils.db.managed; - -import org.rocksdb.Range; - -/** - * Managed {@link Range}. - */ -public class ManagedRange implements AutoCloseable { - - private final Range original; - private final ManagedSlice start; - private final ManagedSlice limit; - - public ManagedRange(final ManagedSlice start, final ManagedSlice limit) { - this.start = start; - this.limit = limit; - this.original = new Range(start, limit); - } - - public Range getOriginal() { - return original; - } - - @Override - public void close() { - start.close(); - limit.close(); - } -} diff --git a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java index 5855e998768d..0fff6098bff4 100644 --- a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java +++ b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java @@ -30,7 +30,6 @@ import org.rocksdb.LiveFileMetaData; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; -import org.rocksdb.TableProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -124,15 +123,4 @@ public static Map getLiveMetadataForSSTFiles(RocksDB d public Map getLiveMetadataForSSTFiles() { return getLiveMetadataForSSTFiles(this.get()); } - - public Map getPropertiesOfColumnFamilyInRange(ColumnFamilyHandle columnFamilyHandle, - List ranges) throws RocksDatabaseException { - try { - return get().getPropertiesOfTablesInRange(ranges.stream() - .map(ManagedRange::getOriginal) - .collect(Collectors.toList())); - } catch (RocksDBException e) { - throw new RocksDatabaseException("Failed to get properties of column family in range", e); - } - } } diff --git a/pom.xml b/pom.xml index 4923db920498..8c15c3edcdd2 100644 --- a/pom.xml +++ b/pom.xml @@ -1950,6 +1950,7 @@ org.rocksdb.InfoLogLevel org.rocksdb.OptionsUtil org.rocksdb.RocksDBException + org.rocksdb.Range org.rocksdb.StatsLevel org.rocksdb.TableProperties org.rocksdb.TransactionLogIterator.BatchResult From 27b3900caaa40308f5bcd86898a9bbebd1bfac39 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 29 May 2025 11:10:03 +0800 Subject: [PATCH 17/31] Update rocksJava JNI patch to fix GetPropertiesOfTableInRange --- .../hadoop/hdds/utils/db/TestRDBStore.java | 40 ++++++++++++ hadoop-hdds/rocks-native/pom.xml | 33 +++------- .../src/main/patches/rocks-native.patch | 63 ++++++++++++++++++- 3 files changed, 109 insertions(+), 27 deletions(-) diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStore.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStore.java index 01be3a73f8e2..21d83b0c2c31 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStore.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStore.java @@ -51,6 +51,7 @@ import org.rocksdb.RocksDB; import org.rocksdb.Statistics; import org.rocksdb.StatsLevel; +import org.rocksdb.TableProperties; /** * RDBStore Tests. @@ -467,4 +468,43 @@ private void compareSstWithSameName(File checkpoint1, File checkpoint2) } } } + + @Test + public void testGetPropertiesOfTableInRange() throws Exception { + // Insert 100 keys with predictable names into the "First" column family + String tableName = families.get(1); + try (Table table = rdbStore.getTable(tableName)) { + for (int i = 0; i < 100; i++) { + String keyStr = String.format("key%03d", i); + byte[] key = keyStr.getBytes(StandardCharsets.UTF_8); + byte[] value = ("value" + i).getBytes(StandardCharsets.UTF_8); + table.put(key, value); + // Flush periodically to make sure keys are written to SST files so that + // RocksDB can report properties. + if (i % 10 == 9) { + rdbStore.flushDB(); + } + } + } + // Ensure any remaining data is flushed. + rdbStore.flushDB(); + + // Fetch table properties for the full key range we just inserted. + Map propsMap = rdbStore.getPropertiesOfTableInRange( + tableName, "key000", "key099"); + + assertNotNull(propsMap, "Properties map should not be null"); + assertFalse(propsMap.isEmpty(), "Properties map should not be empty"); + + long totalEntries = 0; + long totalDeletions = 0; + for (TableProperties props : propsMap.values()) { + totalEntries += props.getNumEntries(); + totalDeletions += props.getNumDeletions(); + } + + // We inserted exactly 100 keys and did not delete any. + assertEquals(100, totalEntries, "Total number of entries reported should match the inserted keys"); + assertEquals(0, totalDeletions, "There should be no deletion entries reported"); + } } diff --git a/hadoop-hdds/rocks-native/pom.xml b/hadoop-hdds/rocks-native/pom.xml index 0c7e8fa7e2da..17c86ec4da74 100644 --- a/hadoop-hdds/rocks-native/pom.xml +++ b/hadoop-hdds/rocks-native/pom.xml @@ -159,30 +159,6 @@ - - org.apache.maven.plugins - maven-dependency-plugin - - - unpack-dependency - - unpack - - initialize - - - - org.rocksdb - rocksdbjni - jar - false - ${project.build.directory}/rocksdbjni - - - - - - com.googlecode.maven-download-plugin download-maven-plugin @@ -255,6 +231,13 @@ + + + + + + + @@ -267,7 +250,7 @@ - + diff --git a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch index b2627fbbb3ef..b4af899aa7b6 100644 --- a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch +++ b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch @@ -115,13 +115,72 @@ index 000000000..09e748208 +} // namespace ROCKSDB_NAMESPACE + +#endif // ROCKSDB_LITE +diff --git a/java/rocksjni/rocksjni.cc b/java/rocksjni/rocksjni.cc +index fca7074ed..a1acac48c 100644 +--- a/java/rocksjni/rocksjni.cc ++++ b/java/rocksjni/rocksjni.cc +@@ -3690,7 +3690,53 @@ jobject Java_org_rocksdb_RocksDB_getPropertiesOfTablesInRange( + // cleanup + env->ReleaseLongArrayElements(jrange_slice_handles, jrange_slice_handle, JNI_ABORT); + +- return jrange_slice_handles; ++ // convert to Java type (HashMap) ++ jobject jhash_map = ROCKSDB_NAMESPACE::HashMapJni::construct( ++ env, static_cast(table_properties_collection.size())); ++ if (jhash_map == nullptr) { ++ // exception occurred ++ return nullptr; ++ } ++ ++ const ROCKSDB_NAMESPACE::HashMapJni::FnMapKV< ++ const std::string, ++ const std::shared_ptr, jobject, ++ jobject> ++ fn_map_kv = ++ [env](const std::pair>& ++ kv) { ++ jstring jkey = ROCKSDB_NAMESPACE::JniUtil::toJavaString( ++ env, &(kv.first), false); ++ if (env->ExceptionCheck()) { ++ // an error occurred ++ return std::unique_ptr>(nullptr); ++ } ++ ++ jobject jtable_properties = ++ ROCKSDB_NAMESPACE::TablePropertiesJni::fromCppTableProperties( ++ env, *(kv.second.get())); ++ if (jtable_properties == nullptr) { ++ // an error occurred ++ env->DeleteLocalRef(jkey); ++ return std::unique_ptr>(nullptr); ++ } ++ ++ return std::unique_ptr>( ++ new std::pair( ++ static_cast(jkey), ++ static_cast(jtable_properties))); ++ }; ++ ++ if (!ROCKSDB_NAMESPACE::HashMapJni::putAll( ++ env, jhash_map, table_properties_collection.begin(), ++ table_properties_collection.end(), fn_map_kv)) { ++ // exception occurred ++ return nullptr; ++ } ++ ++ return jhash_map; + } + + /* diff --git a/src.mk b/src.mk index b94bc43ca..c13e5cde6 100644 --- a/src.mk +++ b/src.mk @@ -338,11 +338,8 @@ RANGE_TREE_SOURCES =\ utilities/transactions/lock/range/range_tree/range_tree_lock_tracker.cc - + TOOL_LIB_SOURCES = \ - tools/io_tracer_parser_tool.cc \ - tools/ldb_cmd.cc \ @@ -130,7 +189,7 @@ index b94bc43ca..c13e5cde6 100644 - utilities/blob_db/blob_dump_tool.cc \ + tools/raw_sst_file_reader.cc \ + tools/raw_sst_file_iterator.cc \ - + ANALYZER_LIB_SOURCES = \ tools/block_cache_analyzer/block_cache_trace_analyzer.cc \ diff --git a/tools/raw_sst_file_iterator.cc b/tools/raw_sst_file_iterator.cc From 191c19bc5e739469e111fccebff67bd47127835d Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 29 May 2025 15:41:26 +0800 Subject: [PATCH 18/31] try to fix ci --- .github/workflows/check.yml | 10 ++++++++++ hadoop-hdds/rocks-native/pom.xml | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 6d06484d9444..5db364e1900d 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -220,6 +220,13 @@ jobs: distribution: 'temurin' java-version: ${{ inputs.java-version }} + - name: Install GCC-10 + run: | + sudo apt-get update + sudo apt-get install -y gcc-10 g++-10 + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100 + - name: Execute pre-test steps if: ${{ inputs.pre-script }} run: | @@ -231,6 +238,9 @@ jobs: env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} OZONE_WITH_COVERAGE: ${{ inputs.with-coverage }} + CC: 'gcc-10' + CXX: 'g++-10' + ROCKSDB_CXX_STANDARD: 'c++20' - name: Execute post-failure steps if: ${{ failure() && inputs.post-failure }} diff --git a/hadoop-hdds/rocks-native/pom.xml b/hadoop-hdds/rocks-native/pom.xml index 17c86ec4da74..1e124569dd34 100644 --- a/hadoop-hdds/rocks-native/pom.xml +++ b/hadoop-hdds/rocks-native/pom.xml @@ -234,7 +234,7 @@ - + From 38987600689904be98f2bb8a7700df3cd45d4c45 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 29 May 2025 18:09:09 +0800 Subject: [PATCH 19/31] Add disk space cleanup script for ci --- .github/workflows/check.yml | 4 ++ dev-support/ci/cleanup-disk-space.sh | 62 ++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100755 dev-support/ci/cleanup-disk-space.sh diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 5db364e1900d..cda31c3fdedb 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -150,6 +150,10 @@ jobs: ref: ${{ inputs.sha }} fetch-depth: ${{ inputs.checkout-fetch-depth }} + - name: Free up disk space + run: | + dev-support/ci/cleanup-disk-space.sh + - name: Download Ozone source tarball if: ${{ inputs.needs-ozone-source-tarball }} uses: actions/download-artifact@v4 diff --git a/dev-support/ci/cleanup-disk-space.sh b/dev-support/ci/cleanup-disk-space.sh new file mode 100755 index 000000000000..c4c6d4ab39f3 --- /dev/null +++ b/dev-support/ci/cleanup-disk-space.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# +# This script frees up disk space on CI systems by removing unneeded packages +# and large directories. This is particularly useful for builds that require +# significant disk space, such as RocksDB native builds. +# + +echo "==============================================================================" +echo "Freeing up disk space on CI system" +echo "==============================================================================" + +echo "Listing 100 largest packages" +dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100 +df -h + +echo "Removing large packages" +sudo apt-get remove -y '^ghc-8.*' +sudo apt-get remove -y 'php.*' +sudo apt-get remove -y azure-cli google-cloud-sdk hhvm google-chrome-stable firefox powershell mono-devel +sudo apt-get autoremove -y +sudo apt-get clean + +df -h + +echo "Removing large directories" +# Remove .NET SDK (typically 15GB+) +rm -rf /usr/share/dotnet/ + +# Remove Maven repository cache (if needed) +if [ "${CLEAN_MAVEN_CACHE}" = "true" ]; then + echo "Cleaning Maven repository cache" + rm -rf ~/.m2/repository +fi + +# Remove RocksDB build artifacts (if needed) +if [ "${CLEAN_ROCKSDB_BUILD}" = "true" ]; then + echo "Cleaning RocksDB build artifacts" + find . -name "rocksdb-*" -type d -exec rm -rf {} + +fi + +df -h + +echo "==============================================================================" +echo "Disk space cleanup completed" +echo "==============================================================================" \ No newline at end of file From 835a39ce83107f4cec60a8ae9f2f19a96899b437 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sat, 31 May 2025 14:11:39 +0800 Subject: [PATCH 20/31] fix checkstyle and pmd --- .../apache/hadoop/hdds/utils/db/RDBStore.java | 39 +++++++++++++------ .../hadoop/hdds/utils/db/RocksDatabase.java | 3 +- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index faa88177bd0c..7d9ea37b53d3 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -37,9 +37,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.utils.IOUtils; @@ -234,6 +232,10 @@ public void compactDB() throws IOException { public void compactTable(String tableName) throws IOException { try (ManagedCompactRangeOptions options = new ManagedCompactRangeOptions()) { compactTable(tableName, options); + } + } + + @Override public void compactTable(String tableName, String startKey, String endKey) throws IOException { try (ManagedCompactRangeOptions options = new ManagedCompactRangeOptions()) { compactTable(tableName, startKey, endKey, options); @@ -241,16 +243,18 @@ public void compactTable(String tableName, String startKey, String endKey) throw } @Override - public void compactTable(String tableName, ManagedCompactRangeOptions options) throws IOException { RocksDatabase.ColumnFamily columnFamily = db.getColumnFamily(tableName); + if (columnFamily == null) { throw new IOException("Table not found: " + tableName); } db.compactRange(columnFamily, null, null, options); } + @Override public void compactTable(String tableName, String startKey, String endKey, ManagedCompactRangeOptions options) throws IOException { + ColumnFamily columnFamily = db.getColumnFamily(tableName); if (columnFamily == null) { throw new IOException("No such table in this DB. TableName : " + tableName); } @@ -261,8 +265,9 @@ public void compactTable(String tableName, String startKey, String endKey, @Override public Map getPropertiesOfTableInRange(String tableName, String startKey, String endKey) throws IOException { - return getPropertiesOfTableInRange(tableName, + Map result = getPropertiesOfTableInRange(tableName, Collections.singletonList(new KeyRange(startKey, endKey))); + return result; } @Override @@ -273,18 +278,28 @@ public Map getPropertiesOfTableInRange(String tableName throw new IOException("No such table in this DB. TableName : " + tableName); } - List rocksRanges = ranges.stream() - .map(t -> { - ManagedSlice start = new ManagedSlice(StringUtils.string2Bytes(t.getStartKey())); - ManagedSlice end = new ManagedSlice(StringUtils.string2Bytes(t.getEndKey())); - return new Range(start, end); - }) - .filter(Objects::nonNull) - .collect(Collectors.toList()); + List rocksRanges = new ArrayList<>(); + List managedSlices = new ArrayList<>(); try { + for (KeyRange t : ranges) { + ManagedSlice start = new ManagedSlice(StringUtils.string2Bytes(t.getStartKey())); + ManagedSlice end = new ManagedSlice(StringUtils.string2Bytes(t.getEndKey())); + managedSlices.add(start); + managedSlices.add(end); + rocksRanges.add(new Range(start, end)); + } return db.getPropertiesOfColumnFamilyInRange(columnFamily, rocksRanges); } catch (RocksDatabaseException e) { throw new IOException("Failed to get properties of table in range", e); + } finally { + // Close all ManagedSlice objects + for (ManagedSlice slice : managedSlices) { + try { + slice.close(); + } catch (Exception e) { + LOG.warn("Failed to close ManagedSlice", e); + } + } } } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java index d61a3e4d130b..d1afd360b57e 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java @@ -888,7 +888,8 @@ public Map getPropertiesOfColumnFamilyInRange(ColumnFam try (UncheckedAutoCloseable ignored = acquire()) { return db.get().getPropertiesOfTablesInRange(columnFamily.getHandle(), ranges); } catch (RocksDBException e) { - throw new RocksDatabaseException("Failed to get properties of column family in range", e); + closeOnError(e); + throw toRocksDatabaseException(this, "getPropertiesOfColumnFamilyInRange", e); } } From d39fd565747d4726b96b586c38d4447d77f9218c Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 4 Jun 2025 11:39:37 +0800 Subject: [PATCH 21/31] Update pom.xml for rocksdb version and add GitHub repository configuration; enhance CI workflows with GitHub actor and token; implement Maven dependency plugin for rocksdbjni unpacking; improve logging in OBSTableCompactor for better compaction tracking. --- .github/workflows/check.yml | 9 +-- .github/workflows/ci.yml | 3 + dev-support/ci/maven-settings.xml | 8 +++ hadoop-hdds/rocks-native/pom.xml | 41 ++++++++++-- .../src/main/patches/rocks-native.patch | 63 +------------------ .../om/compaction/CompoundKeyRangeStats.java | 11 +++- .../om/compaction/OBSTableCompactor.java | 17 +++-- pom.xml | 6 +- 8 files changed, 78 insertions(+), 80 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index cda31c3fdedb..cc09241e26b8 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -136,6 +136,8 @@ env: OZONE_IMAGE: ghcr.io/apache/ozone OZONE_RUNNER_IMAGE: ghcr.io/apache/ozone-runner OZONE_VOLUME_OWNER: 1000 + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: check: @@ -224,13 +226,6 @@ jobs: distribution: 'temurin' java-version: ${{ inputs.java-version }} - - name: Install GCC-10 - run: | - sudo apt-get update - sudo apt-get install -y gcc-10 g++-10 - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100 - - name: Execute pre-test steps if: ${{ inputs.pre-script }} run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04c944c9b4a5..9447bbb674ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,9 @@ env: MAVEN_ARGS: --batch-mode --settings ${{ github.workspace }}/dev-support/ci/maven-settings.xml MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 OZONE_WITH_COVERAGE: ${{ github.event_name == 'push' }} + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + jobs: build-info: diff --git a/dev-support/ci/maven-settings.xml b/dev-support/ci/maven-settings.xml index 43fa07bb52ba..2cecade99f49 100644 --- a/dev-support/ci/maven-settings.xml +++ b/dev-support/ci/maven-settings.xml @@ -32,4 +32,12 @@ true + + + + github + ${env.GITHUB_ACTOR} + ${env.GITHUB_TOKEN} + + diff --git a/hadoop-hdds/rocks-native/pom.xml b/hadoop-hdds/rocks-native/pom.xml index 1e124569dd34..d3d028c6fd91 100644 --- a/hadoop-hdds/rocks-native/pom.xml +++ b/hadoop-hdds/rocks-native/pom.xml @@ -159,6 +159,30 @@ + + org.apache.maven.plugins + maven-dependency-plugin + + + unpack-dependency + + unpack + + initialize + + + + org.rocksdb + rocksdbjni + jar + false + ${project.build.directory}/rocksdbjni + + + + + + com.googlecode.maven-download-plugin download-maven-plugin @@ -170,7 +194,7 @@ generate-sources - https://github.com/facebook/rocksdb/archive/refs/tags/v${rocksdb.version}.tar.gz + https://github.com/peterxcli/rocksdb/archive/refs/tags/v${rocksdb.version}.tar.gz rocksdb-v${rocksdb.version}.tar.gz ${project.build.directory}/rocksdb @@ -231,13 +255,22 @@ - + + + @@ -250,7 +283,7 @@ - + diff --git a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch index b4af899aa7b6..b2627fbbb3ef 100644 --- a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch +++ b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch @@ -115,72 +115,13 @@ index 000000000..09e748208 +} // namespace ROCKSDB_NAMESPACE + +#endif // ROCKSDB_LITE -diff --git a/java/rocksjni/rocksjni.cc b/java/rocksjni/rocksjni.cc -index fca7074ed..a1acac48c 100644 ---- a/java/rocksjni/rocksjni.cc -+++ b/java/rocksjni/rocksjni.cc -@@ -3690,7 +3690,53 @@ jobject Java_org_rocksdb_RocksDB_getPropertiesOfTablesInRange( - // cleanup - env->ReleaseLongArrayElements(jrange_slice_handles, jrange_slice_handle, JNI_ABORT); - -- return jrange_slice_handles; -+ // convert to Java type (HashMap) -+ jobject jhash_map = ROCKSDB_NAMESPACE::HashMapJni::construct( -+ env, static_cast(table_properties_collection.size())); -+ if (jhash_map == nullptr) { -+ // exception occurred -+ return nullptr; -+ } -+ -+ const ROCKSDB_NAMESPACE::HashMapJni::FnMapKV< -+ const std::string, -+ const std::shared_ptr, jobject, -+ jobject> -+ fn_map_kv = -+ [env](const std::pair>& -+ kv) { -+ jstring jkey = ROCKSDB_NAMESPACE::JniUtil::toJavaString( -+ env, &(kv.first), false); -+ if (env->ExceptionCheck()) { -+ // an error occurred -+ return std::unique_ptr>(nullptr); -+ } -+ -+ jobject jtable_properties = -+ ROCKSDB_NAMESPACE::TablePropertiesJni::fromCppTableProperties( -+ env, *(kv.second.get())); -+ if (jtable_properties == nullptr) { -+ // an error occurred -+ env->DeleteLocalRef(jkey); -+ return std::unique_ptr>(nullptr); -+ } -+ -+ return std::unique_ptr>( -+ new std::pair( -+ static_cast(jkey), -+ static_cast(jtable_properties))); -+ }; -+ -+ if (!ROCKSDB_NAMESPACE::HashMapJni::putAll( -+ env, jhash_map, table_properties_collection.begin(), -+ table_properties_collection.end(), fn_map_kv)) { -+ // exception occurred -+ return nullptr; -+ } -+ -+ return jhash_map; - } - - /* diff --git a/src.mk b/src.mk index b94bc43ca..c13e5cde6 100644 --- a/src.mk +++ b/src.mk @@ -338,11 +338,8 @@ RANGE_TREE_SOURCES =\ utilities/transactions/lock/range/range_tree/range_tree_lock_tracker.cc - + TOOL_LIB_SOURCES = \ - tools/io_tracer_parser_tool.cc \ - tools/ldb_cmd.cc \ @@ -189,7 +130,7 @@ index b94bc43ca..c13e5cde6 100644 - utilities/blob_db/blob_dump_tool.cc \ + tools/raw_sst_file_reader.cc \ + tools/raw_sst_file_iterator.cc \ - + ANALYZER_LIB_SOURCES = \ tools/block_cache_analyzer/block_cache_trace_analyzer.cc \ diff --git a/tools/raw_sst_file_iterator.cc b/tools/raw_sst_file_iterator.cc diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompoundKeyRangeStats.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompoundKeyRangeStats.java index c421ac07fdca..eb0eba4d075f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompoundKeyRangeStats.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/CompoundKeyRangeStats.java @@ -26,10 +26,13 @@ * compound KeyRangeStats. */ public class CompoundKeyRangeStats { - private KeyRangeStats compoundStats; - private List> keyRangeStatsList; + private KeyRangeStats compoundStats = null; + private List> keyRangeStatsList = null; public CompoundKeyRangeStats(List> keyRangeStatsList) { + if (keyRangeStatsList == null || keyRangeStatsList.isEmpty()) { + return; + } this.keyRangeStatsList = keyRangeStatsList; for (Pair entry : keyRangeStatsList) { if (compoundStats == null) { @@ -69,6 +72,10 @@ public void add(CompoundKeyRangeStats other) { keyRangeStatsList.addAll(other.getKeyRangeStatsList()); } + public boolean isEmpty() { + return keyRangeStatsList == null || keyRangeStatsList.isEmpty() || compoundStats == null; + } + @Override public String toString() { return "CompoundKeyRangeStats{" + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java index 185b195120b0..836f4ddbd32f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java @@ -52,9 +52,13 @@ public OBSTableCompactor(CompactorBuilder builder) { @Override public void run() { - List ranges = getRangesNeedingCompaction(); - for (KeyRange range : ranges) { - addRangeCompactionTask(range); + try { + List ranges = getRangesNeedingCompaction(); + for (KeyRange range : ranges) { + addRangeCompactionTask(range); + } + } catch (Exception e) { + LOG.error("Failed to run range compaction for table {}", getTableName(), e); } } @@ -144,6 +148,9 @@ private Pair prepareRanges() throws IOException { // Get compound stats for the range CompoundKeyRangeStats compoundStats = getCompoundKeyRangeStatsFromRange(currentRange); + if (compoundStats.isEmpty()) { + return null; + } if (compoundStats.getNumEntries() <= getMaxCompactionEntries()) { return Pair.of(currentRange, compoundStats.getCompoundStats()); @@ -253,7 +260,7 @@ private CompoundKeyRangeStats getCompoundKeyRangeStatsFromRange(KeyRange range) KeyRange keyRange = new KeyRange(meta.smallestKey(), meta.largestKey()); KeyRangeStats stats = KeyRangeStats.fromTableProperties(entry.getValue()); keyRangeStatsList.add(Pair.of(keyRange, stats)); - } - return new CompoundKeyRangeStats(keyRangeStatsList); + CompoundKeyRangeStats result = new CompoundKeyRangeStats(keyRangeStatsList); + return result; } } diff --git a/pom.xml b/pom.xml index 8c15c3edcdd2..e38a0ec34d91 100644 --- a/pom.xml +++ b/pom.xml @@ -206,7 +206,7 @@ 0.10.2 1.2.26 2.6.1 - 7.7.3 + 7.7.3.2 3.1.0 bash 2.0.17 @@ -2489,6 +2489,10 @@ apache.snapshots.https https://repository.apache.org/content/repositories/snapshots + + github + https://maven.pkg.github.com/peterxcli/rocksdb + From aa83ae194efb0de264ad78a480c23b1687655ba3 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 4 Jun 2025 11:58:06 +0800 Subject: [PATCH 22/31] Remove disk space cleanup script and related workflow steps from CI configuration --- .github/workflows/check.yml | 7 ---- dev-support/ci/cleanup-disk-space.sh | 62 ---------------------------- 2 files changed, 69 deletions(-) delete mode 100755 dev-support/ci/cleanup-disk-space.sh diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index cc09241e26b8..0891c0775197 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -152,10 +152,6 @@ jobs: ref: ${{ inputs.sha }} fetch-depth: ${{ inputs.checkout-fetch-depth }} - - name: Free up disk space - run: | - dev-support/ci/cleanup-disk-space.sh - - name: Download Ozone source tarball if: ${{ inputs.needs-ozone-source-tarball }} uses: actions/download-artifact@v4 @@ -237,9 +233,6 @@ jobs: env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} OZONE_WITH_COVERAGE: ${{ inputs.with-coverage }} - CC: 'gcc-10' - CXX: 'g++-10' - ROCKSDB_CXX_STANDARD: 'c++20' - name: Execute post-failure steps if: ${{ failure() && inputs.post-failure }} diff --git a/dev-support/ci/cleanup-disk-space.sh b/dev-support/ci/cleanup-disk-space.sh deleted file mode 100755 index c4c6d4ab39f3..000000000000 --- a/dev-support/ci/cleanup-disk-space.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# This script frees up disk space on CI systems by removing unneeded packages -# and large directories. This is particularly useful for builds that require -# significant disk space, such as RocksDB native builds. -# - -echo "==============================================================================" -echo "Freeing up disk space on CI system" -echo "==============================================================================" - -echo "Listing 100 largest packages" -dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100 -df -h - -echo "Removing large packages" -sudo apt-get remove -y '^ghc-8.*' -sudo apt-get remove -y 'php.*' -sudo apt-get remove -y azure-cli google-cloud-sdk hhvm google-chrome-stable firefox powershell mono-devel -sudo apt-get autoremove -y -sudo apt-get clean - -df -h - -echo "Removing large directories" -# Remove .NET SDK (typically 15GB+) -rm -rf /usr/share/dotnet/ - -# Remove Maven repository cache (if needed) -if [ "${CLEAN_MAVEN_CACHE}" = "true" ]; then - echo "Cleaning Maven repository cache" - rm -rf ~/.m2/repository -fi - -# Remove RocksDB build artifacts (if needed) -if [ "${CLEAN_ROCKSDB_BUILD}" = "true" ]; then - echo "Cleaning RocksDB build artifacts" - find . -name "rocksdb-*" -type d -exec rm -rf {} + -fi - -df -h - -echo "==============================================================================" -echo "Disk space cleanup completed" -echo "==============================================================================" \ No newline at end of file From 122e32175f308b7582d1c366eb77c2694b94d1dd Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 4 Jun 2025 12:10:59 +0800 Subject: [PATCH 23/31] Enhance getCompoundKeyRangeStatsFromRange method in OBSTableCompactor by ensuring proper closure of key range stats list, improving the accuracy of compound key range statistics retrieval. --- .../apache/hadoop/ozone/om/compaction/OBSTableCompactor.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java index 836f4ddbd32f..036b3f73e9ec 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java @@ -260,6 +260,8 @@ private CompoundKeyRangeStats getCompoundKeyRangeStatsFromRange(KeyRange range) KeyRange keyRange = new KeyRange(meta.smallestKey(), meta.largestKey()); KeyRangeStats stats = KeyRangeStats.fromTableProperties(entry.getValue()); keyRangeStatsList.add(Pair.of(keyRange, stats)); + } + CompoundKeyRangeStats result = new CompoundKeyRangeStats(keyRangeStatsList); return result; } From 122c21ddd759aa4c45912500eb6797f2b0fb0d98 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 6 Jun 2025 14:05:19 +0800 Subject: [PATCH 24/31] Fix some logic in OBS table compactor --- .../apache/hadoop/hdds/utils/db/DBStore.java | 1 - .../om/compaction/OBSTableCompactor.java | 20 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java index 85fdccd807a7..5002a5e8c0bd 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java @@ -144,7 +144,6 @@ Map getPropertiesOfTableInRange(String tableName, Strin Map getPropertiesOfTableInRange(String tableName, List ranges) throws IOException; - /** * Moves a key from the Source Table to the destination Table. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java index 036b3f73e9ec..6c55871b075d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java @@ -68,7 +68,7 @@ protected void collectRangesNeedingCompaction(List ranges) { try { Pair preparedRange = prepareRanges(); if (preparedRange == null) { - continue; + break; } // TODO: merge consecutive ranges if they aren't too big, notice the situation that iterator has been reset // to be more specific, @@ -116,8 +116,10 @@ private KeyRange getNextBucketBound() { String bucketStartKey = iterator.next().getKey().getCacheKey(), bucketEndKey; if (iterator.hasNext()) { bucketEndKey = iterator.next().getKey().getCacheKey(); + currentBucketUpperBound = bucketEndKey; } else { bucketEndKey = StringUtils.getKeyPrefixUpperBound(bucketStartKey); + currentBucketUpperBound = null; } return new KeyRange(bucketStartKey, bucketEndKey); } @@ -149,7 +151,7 @@ private Pair prepareRanges() throws IOException { // Get compound stats for the range CompoundKeyRangeStats compoundStats = getCompoundKeyRangeStatsFromRange(currentRange); if (compoundStats.isEmpty()) { - return null; + return Pair.of(currentRange, new KeyRangeStats()); } if (compoundStats.getNumEntries() <= getMaxCompactionEntries()) { @@ -161,7 +163,7 @@ private Pair prepareRanges() throws IOException { compoundStats.getKeyRangeStatsList(), getMaxCompactionEntries()); if (splittedRanges == null || splittedRanges.isEmpty()) { LOG.warn("splitted ranges is null or empty, return null, current range: {}", currentRange); - return null; + return Pair.of(currentRange, compoundStats.getCompoundStats()); } Pair squashedRange = squashRanges(splittedRanges); String squashedRangeEndKeyUpperBound = StringUtils.getKeyPrefixUpperBound(squashedRange.getLeft().getEndKey()); @@ -233,7 +235,8 @@ private Pair squashRanges( maxKey = StringUtils.max(maxKey, range.getLeft().getEndKey()); stats.add(range.getRight()); } - return Pair.of(new KeyRange(minKey, maxKey), stats); + KeyRange squashedRange = new KeyRange(minKey, maxKey); + return Pair.of(squashedRange, stats); } private CompoundKeyRangeStats getCompoundKeyRangeStatsFromRange(KeyRange range) throws IOException { @@ -243,18 +246,17 @@ private CompoundKeyRangeStats getCompoundKeyRangeStatsFromRange(KeyRange range) // https://github.com/facebook/rocksdb/blob/09cd25f76305f2110131f51068656ab392dc2bf5/db/version_set.cc#L1744-L1768 // https://github.com/facebook/rocksdb/blob/a00391c72996a5dbdd93a621dbc53719c13b05c4/file/filename.cc#L140-L150 for (LiveFileMetaData metaData : liveFileMetaDataList) { - fileMap.put(metaData.fileName(), metaData); + fileMap.put(metaData.path() + metaData.fileName(), metaData); } Map propsMap = getDBStore().getPropertiesOfTableInRange(getTableName(), Collections.singletonList(range)); for (Map.Entry entry : propsMap.entrySet()) { - String fullPath = entry.getKey(); - String fileName = fullPath.substring(fullPath.lastIndexOf('/') + 1); - LiveFileMetaData meta = fileMap.get(fileName); + String filePath = entry.getKey(); + LiveFileMetaData meta = fileMap.get(filePath); if (meta == null) { - LOG.warn("LiveFileMetaData not found for file {}", fullPath); + LOG.warn("LiveFileMetaData not found for file {}", filePath); continue; } KeyRange keyRange = new KeyRange(meta.smallestKey(), meta.largestKey()); From b45fe1ae0e3369284dfdcbe19c23c866e4baaaa5 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 6 Jun 2025 14:22:09 +0800 Subject: [PATCH 25/31] add log --- .../apache/hadoop/hdds/utils/db/RDBStore.java | 11 +++ .../om/compaction/OBSTableCompactor.java | 70 +++++++++++++++---- .../om/service/RangeCompactionService.java | 1 + 3 files changed, 68 insertions(+), 14 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index 7d9ea37b53d3..4f5fdba663ae 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -237,9 +237,11 @@ public void compactTable(String tableName) throws IOException { @Override public void compactTable(String tableName, String startKey, String endKey) throws IOException { + LOG.info("Starting table compaction for table: {}, range: [{} - {}]", tableName, startKey, endKey); try (ManagedCompactRangeOptions options = new ManagedCompactRangeOptions()) { compactTable(tableName, startKey, endKey, options); } + LOG.info("Completed table compaction for table: {}, range: [{} - {}]", tableName, startKey, endKey); } @Override @@ -254,27 +256,35 @@ public void compactTable(String tableName, ManagedCompactRangeOptions options) t @Override public void compactTable(String tableName, String startKey, String endKey, ManagedCompactRangeOptions options) throws IOException { + LOG.info("Starting table compaction with options for table: {}, range: [{} - {}]", tableName, startKey, endKey); ColumnFamily columnFamily = db.getColumnFamily(tableName); if (columnFamily == null) { + LOG.error("Table not found for compaction: {}", tableName); throw new IOException("No such table in this DB. TableName : " + tableName); } db.compactRange(columnFamily, StringUtils.string2Bytes(startKey), StringUtils.string2Bytes(endKey), options); + LOG.info("Completed table compaction with options for table: {}, range: [{} - {}]", tableName, startKey, endKey); } @Override public Map getPropertiesOfTableInRange(String tableName, String startKey, String endKey) throws IOException { + LOG.info("Getting table properties for table: {}, range: [{} - {}]", tableName, startKey, endKey); Map result = getPropertiesOfTableInRange(tableName, Collections.singletonList(new KeyRange(startKey, endKey))); + LOG.info("Retrieved {} table properties for table: {}, range: [{} - {}]", + result.size(), tableName, startKey, endKey); return result; } @Override public Map getPropertiesOfTableInRange(String tableName, List ranges) throws IOException { + LOG.info("Getting table properties for table: {}, number of ranges: {}", tableName, ranges.size()); ColumnFamily columnFamily = db.getColumnFamily(tableName); if (columnFamily == null) { + LOG.error("Table not found for getting properties: {}", tableName); throw new IOException("No such table in this DB. TableName : " + tableName); } @@ -288,6 +298,7 @@ public Map getPropertiesOfTableInRange(String tableName managedSlices.add(end); rocksRanges.add(new Range(start, end)); } + LOG.info("Converted {} ranges to RocksDB ranges, start to get properties", rocksRanges.size()); return db.getPropertiesOfColumnFamilyInRange(columnFamily, rocksRanges); } catch (RocksDatabaseException e) { throw new IOException("Failed to get properties of table in range", e); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java index 6c55871b075d..cc803f2cd9b2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java @@ -53,10 +53,15 @@ public OBSTableCompactor(CompactorBuilder builder) { @Override public void run() { try { + LOG.info("Starting range compaction scan for table {}", getTableName()); List ranges = getRangesNeedingCompaction(); + LOG.info("Found {} ranges needing compaction for table {}, submitting to compaction queue", + ranges.size(), getTableName()); for (KeyRange range : ranges) { + LOG.info("Submitting range [{}] for compaction", range); addRangeCompactionTask(range); } + LOG.info("Completed range compaction scan for table {}", getTableName()); } catch (Exception e) { LOG.error("Failed to run range compaction for table {}", getTableName(), e); } @@ -64,10 +69,13 @@ public void run() { @Override protected void collectRangesNeedingCompaction(List ranges) { + LOG.info("Starting to collect ranges needing compaction for table {}", getTableName()); for (int i = 0; i < getRangesPerRun(); i++) { try { + LOG.info("Preparing range {} of {} for table {}", i + 1, getRangesPerRun(), getTableName()); Pair preparedRange = prepareRanges(); if (preparedRange == null) { + LOG.info("No more ranges to prepare for table {}", getTableName()); break; } // TODO: merge consecutive ranges if they aren't too big, notice the situation that iterator has been reset @@ -93,12 +101,17 @@ protected void collectRangesNeedingCompaction(List ranges) { // then **set temp range to a range that composed of temp range and the current range** // (notice the difference between point 2, 3) if (needsCompaction(preparedRange.getRight(), getMinTombstones(), getTombstoneRatio())) { + LOG.info("Range [{}] needs compaction with stats: {}", preparedRange.getLeft(), preparedRange.getRight()); ranges.add(preparedRange.getLeft()); + } else { + LOG.info("Range [{}] does not need compaction with stats: {}", preparedRange.getLeft(), + preparedRange.getRight()); } } catch (IOException e) { - LOG.error("Failed to prepare ranges for compaction", e); + LOG.error("Failed to prepare ranges for compaction for table {}", getTableName(), e); } } + LOG.info("Collected {} ranges needing compaction for table {}", ranges.size(), getTableName()); } /** @@ -107,9 +120,11 @@ protected void collectRangesNeedingCompaction(List ranges) { * @return the next bucket bound, or null if reach the end of the iterator */ private KeyRange getNextBucketBound() { + LOG.info("Getting next bucket bound starting from {}", currentBucketUpperBound); Iterator, CacheValue>> iterator = getMetadataManager() .getBucketIterator(currentBucketUpperBound); if (!iterator.hasNext()) { + LOG.info("No more bucket bounds found"); return null; } @@ -117,9 +132,11 @@ private KeyRange getNextBucketBound() { if (iterator.hasNext()) { bucketEndKey = iterator.next().getKey().getCacheKey(); currentBucketUpperBound = bucketEndKey; + LOG.info("Found bucket range: [{} - {}]", bucketStartKey, bucketEndKey); } else { bucketEndKey = StringUtils.getKeyPrefixUpperBound(bucketStartKey); currentBucketUpperBound = null; + LOG.info("Found last bucket range: [{} - {}]", bucketStartKey, bucketEndKey); } return new KeyRange(bucketStartKey, bucketEndKey); } @@ -131,11 +148,13 @@ private KeyRange getNextBucketBound() { * @throws IOException if an error occurs */ private Pair prepareRanges() throws IOException { + LOG.info("Preparing ranges for compaction"); // Get the initial range to process KeyRange currentRange; if (nextKey != null) { // Continue from last split point + LOG.info("Continuing from last split point: {}", nextKey); currentRange = new KeyRange(nextKey, currentBucketUpperBound); // clean up the nextKey nextKey = null; @@ -143,26 +162,31 @@ private Pair prepareRanges() throws IOException { // Start from next bucket boundary KeyRange bucketBound = getNextBucketBound(); if (bucketBound == null) { + LOG.info("No more bucket bounds to process"); return null; } currentRange = new KeyRange(bucketBound.getStartKey(), bucketBound.getEndKey()); + LOG.info("Starting new bucket range: [{}]", currentRange); } // Get compound stats for the range CompoundKeyRangeStats compoundStats = getCompoundKeyRangeStatsFromRange(currentRange); if (compoundStats.isEmpty()) { + LOG.info("Range [{}] has no entries, returning empty range", currentRange); return Pair.of(currentRange, new KeyRangeStats()); } + LOG.info("Range [{}] has {} entries", currentRange, compoundStats.getNumEntries()); if (compoundStats.getNumEntries() <= getMaxCompactionEntries()) { + LOG.info("Range [{}] fits within max entries limit of {}", currentRange, getMaxCompactionEntries()); return Pair.of(currentRange, compoundStats.getCompoundStats()); } else { - LOG.info("max compaction entries is {}, but range {} has {} entries, splitting", - getMaxCompactionEntries(), currentRange, compoundStats.getNumEntries()); + LOG.info("Range [{}] exceeds max entries limit ({} > {}), splitting", + currentRange, compoundStats.getNumEntries(), getMaxCompactionEntries()); List> splittedRanges = findFitRanges( compoundStats.getKeyRangeStatsList(), getMaxCompactionEntries()); if (splittedRanges == null || splittedRanges.isEmpty()) { - LOG.warn("splitted ranges is null or empty, return null, current range: {}", currentRange); + LOG.info("Failed to split range [{}] into valid sub-ranges, returning original range", currentRange); return Pair.of(currentRange, compoundStats.getCompoundStats()); } Pair squashedRange = squashRanges(splittedRanges); @@ -170,11 +194,10 @@ private Pair prepareRanges() throws IOException { if (currentRange.getEndKey().compareTo(squashedRangeEndKeyUpperBound) > 0) { // if the squashed range is not the last range, then we need to split the range nextKey = squashedRangeEndKeyUpperBound; - LOG.info("squashed range [{}] is not the last range, need to split, next key: {}", - squashedRange, nextKey); + LOG.info("Split range [{}] into [{}] and will continue from {}", + currentRange, squashedRange.getLeft(), nextKey); } else { - LOG.info("squashed range [{}] is the last range, no need to split, " + - "probably means there are some SSTs that are ignored", squashedRange); + LOG.info("Split range [{}] into [{}] (final range)", currentRange, squashedRange.getLeft()); } return squashedRange; } @@ -186,35 +209,36 @@ private Pair prepareRanges() throws IOException { */ private List> findFitRanges( List> ranges, long maxEntries) { - + LOG.info("Finding ranges that fit within {} entries", maxEntries); ranges.sort((a, b) -> a.getLeft().getStartKey().compareTo(b.getLeft().getStartKey())); List> out = new ArrayList<>(); - KeyRangeStats chunkStats = new KeyRangeStats(); for (Pair range : ranges) { long n = range.getRight().getNumEntries(); + LOG.info("Processing range [{}] with {} entries", range.getLeft(), n); // this single SST already exceeds the threshold if (n > maxEntries) { - LOG.warn("Single SST [{}] holds {} entries (> maxEntries {}), it will be ignored.", - range.getLeft(), n, maxEntries); + LOG.warn("Range [{}] exceeds max entries ({} > {}), skipping", range.getLeft(), n, maxEntries); continue; } // adding this SST would overflow the threshold if (chunkStats.getNumEntries() > 0 && chunkStats.getNumEntries() + n > maxEntries) { - LOG.info("Chunk stats [{}] + range [{}]'s numEntries [{}] > maxEntries [{}], fit ranges found", - chunkStats, range, n, maxEntries); + LOG.info("Current chunk ({} entries) + range [{}] ({} entries) exceeds max entries ({}), stopping", + chunkStats.getNumEntries(), range.getLeft(), n, maxEntries); return out; } /* add current SST into (possibly new) chunk */ out.add(range); chunkStats.add(range.getRight()); + LOG.info("Added range [{}] to chunk, total entries: {}", range.getLeft(), chunkStats.getNumEntries()); } + LOG.info("Found {} ranges that fit within {} entries", out.size(), maxEntries); return out; } @@ -225,6 +249,7 @@ private List> findFitRanges( */ private Pair squashRanges( List> list) { + LOG.info("Squashing {} ranges", list.size()); Preconditions.checkNotNull(list, "list is null"); Preconditions.checkArgument(!list.isEmpty(), "list is empty"); @@ -234,23 +259,30 @@ private Pair squashRanges( minKey = StringUtils.min(minKey, range.getLeft().getStartKey()); maxKey = StringUtils.max(maxKey, range.getLeft().getEndKey()); stats.add(range.getRight()); + LOG.info("Squashing range [{}] with {} entries", range.getLeft(), range.getRight().getNumEntries()); } KeyRange squashedRange = new KeyRange(minKey, maxKey); + LOG.info("Squashed {} ranges into [{}] with {} total entries", + list.size(), squashedRange, stats.getNumEntries()); return Pair.of(squashedRange, stats); } private CompoundKeyRangeStats getCompoundKeyRangeStatsFromRange(KeyRange range) throws IOException { + LOG.info("Getting compound stats for range [{}]", range); List> keyRangeStatsList = new ArrayList<>(); List liveFileMetaDataList = ((RDBStore)getDBStore()).getDb().getLiveFilesMetaData(); Map fileMap = new HashMap<>(); // https://github.com/facebook/rocksdb/blob/09cd25f76305f2110131f51068656ab392dc2bf5/db/version_set.cc#L1744-L1768 // https://github.com/facebook/rocksdb/blob/a00391c72996a5dbdd93a621dbc53719c13b05c4/file/filename.cc#L140-L150 + LOG.info("Processing {} live files", liveFileMetaDataList.size()); for (LiveFileMetaData metaData : liveFileMetaDataList) { fileMap.put(metaData.path() + metaData.fileName(), metaData); } + LOG.info("Getting table properties for range [{}]", range); Map propsMap = getDBStore().getPropertiesOfTableInRange(getTableName(), Collections.singletonList(range)); + LOG.info("Found {} table properties in range", propsMap.size()); for (Map.Entry entry : propsMap.entrySet()) { String filePath = entry.getKey(); @@ -262,9 +294,19 @@ private CompoundKeyRangeStats getCompoundKeyRangeStatsFromRange(KeyRange range) KeyRange keyRange = new KeyRange(meta.smallestKey(), meta.largestKey()); KeyRangeStats stats = KeyRangeStats.fromTableProperties(entry.getValue()); keyRangeStatsList.add(Pair.of(keyRange, stats)); + LOG.info("Added file {} with range [{}] and {} entries", + filePath, keyRange, stats.getNumEntries()); } + LOG.info("Key range stats list: {}", keyRangeStatsList); + CompoundKeyRangeStats result = new CompoundKeyRangeStats(keyRangeStatsList); + if (!result.isEmpty()) { + LOG.info("Compound stats for range [{}]: {} entries across {} files", + range, result.getNumEntries(), keyRangeStatsList.size()); + } else { + LOG.info("Range [{}] has no entries", range); + } return result; } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java index 79925abbc415..b8bfa277bd71 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java @@ -138,6 +138,7 @@ private void scheduleCompactionChecks() { ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); compactorExecutors.put(tableName, executor); + LOG.info("Scheduling range compaction compactor for table {}", tableName); executor.scheduleWithFixedDelay( compactor::run, checkIntervalMs, // TODO: randomize the start time From 7d18be3807a437ba1e3b671945130d065015da7f Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sat, 7 Jun 2025 05:03:31 +0000 Subject: [PATCH 26/31] Update Grafana dashboard JSON files to replace Prometheus datasource UID from "${DS_PROMETHEUS}" to "-- Grafana --" for consistency. --- .../Ozone - OMComittedIndexMetrics.json | 4 +- .../Ozone - Ozone Manager RocksDB.json | 126 +++++++++--------- 2 files changed, 65 insertions(+), 65 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - OMComittedIndexMetrics.json b/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - OMComittedIndexMetrics.json index 9ff886cbeec2..4f259acc351b 100644 --- a/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - OMComittedIndexMetrics.json +++ b/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - OMComittedIndexMetrics.json @@ -55,7 +55,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -134,7 +134,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "editorMode": "code", "exemplar": false, diff --git a/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - Ozone Manager RocksDB.json b/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - Ozone Manager RocksDB.json index 4005c50a5a82..fadc4b2b579c 100644 --- a/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - Ozone Manager RocksDB.json +++ b/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - Ozone Manager RocksDB.json @@ -88,7 +88,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -142,7 +142,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -165,7 +165,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -219,7 +219,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -238,7 +238,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -292,7 +292,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -311,7 +311,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -365,7 +365,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -384,7 +384,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -438,7 +438,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -457,7 +457,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -511,7 +511,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -530,7 +530,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -584,7 +584,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -616,7 +616,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -670,7 +670,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -689,7 +689,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -743,7 +743,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -775,7 +775,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -829,7 +829,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -861,7 +861,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "gridPos": { "h": 3, @@ -899,7 +899,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -979,7 +979,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -998,7 +998,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -1077,7 +1077,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -1096,7 +1096,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -1175,7 +1175,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -1194,7 +1194,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -1273,7 +1273,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -1292,7 +1292,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -1372,7 +1372,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -1391,7 +1391,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -1471,7 +1471,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -1490,7 +1490,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -1569,7 +1569,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -1601,7 +1601,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -1681,7 +1681,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -1700,7 +1700,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -1780,7 +1780,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -1799,7 +1799,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -1879,7 +1879,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -1898,7 +1898,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -1978,7 +1978,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -1997,7 +1997,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -2077,7 +2077,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -2096,7 +2096,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -2176,7 +2176,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -2208,7 +2208,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "description": "", "fieldConfig": { @@ -2289,7 +2289,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -2308,7 +2308,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -2388,7 +2388,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -2407,7 +2407,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "description": "", "fieldConfig": { @@ -2488,7 +2488,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -2507,7 +2507,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -2587,7 +2587,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -2619,7 +2619,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -2699,7 +2699,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -2718,7 +2718,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -2798,7 +2798,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -2817,7 +2817,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -2897,7 +2897,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", @@ -2916,7 +2916,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "fieldConfig": { "defaults": { @@ -2996,7 +2996,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "-- Grafana --" }, "disableTextWrap": false, "editorMode": "builder", From ca48dd75328781e515235fa5040466cec2e20b74 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sat, 7 Jun 2025 15:54:09 +0800 Subject: [PATCH 27/31] Remove uid: "--Grafana--" --- .../Ozone - OMComittedIndexMetrics.json | 6 +- .../Ozone - Ozone Manager RocksDB.json | 189 ++++++------------ 2 files changed, 65 insertions(+), 130 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - OMComittedIndexMetrics.json b/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - OMComittedIndexMetrics.json index 4f259acc351b..2fb387afa10e 100644 --- a/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - OMComittedIndexMetrics.json +++ b/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - OMComittedIndexMetrics.json @@ -54,8 +54,7 @@ "panels": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -133,8 +132,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "editorMode": "code", "exemplar": false, diff --git a/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - Ozone Manager RocksDB.json b/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - Ozone Manager RocksDB.json index fadc4b2b579c..86006979fa2a 100644 --- a/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - Ozone Manager RocksDB.json +++ b/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - Ozone Manager RocksDB.json @@ -87,8 +87,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -141,8 +140,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -164,8 +162,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -218,8 +215,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -237,8 +233,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -291,8 +286,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -310,8 +304,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -364,8 +357,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -383,8 +375,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -437,8 +428,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -456,8 +446,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -510,8 +499,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -529,8 +517,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -583,8 +570,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -615,8 +601,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -669,8 +654,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -688,8 +672,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -742,8 +725,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -774,8 +756,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -828,8 +809,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -860,8 +840,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "gridPos": { "h": 3, @@ -898,8 +877,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -978,8 +956,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -997,8 +974,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -1076,8 +1052,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -1095,8 +1070,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -1174,8 +1148,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -1193,8 +1166,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -1272,8 +1244,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -1291,8 +1262,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -1371,8 +1341,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -1390,8 +1359,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -1470,8 +1438,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -1489,8 +1456,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -1568,8 +1534,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -1600,8 +1565,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -1680,8 +1644,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -1699,8 +1662,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -1779,8 +1741,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -1798,8 +1759,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -1878,8 +1838,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -1897,8 +1856,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -1977,8 +1935,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -1996,8 +1953,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -2076,8 +2032,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -2095,8 +2050,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -2175,8 +2129,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -2207,8 +2160,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "description": "", "fieldConfig": { @@ -2288,8 +2240,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -2307,8 +2258,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -2387,8 +2337,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -2406,8 +2355,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "description": "", "fieldConfig": { @@ -2487,8 +2435,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -2506,8 +2453,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -2586,8 +2532,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -2618,8 +2563,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -2698,8 +2642,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -2717,8 +2660,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -2797,8 +2739,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -2816,8 +2757,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -2896,8 +2836,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", @@ -2915,8 +2854,7 @@ }, { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "fieldConfig": { "defaults": { @@ -2995,8 +2933,7 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "-- Grafana --" + "type": "prometheus" }, "disableTextWrap": false, "editorMode": "builder", From 91770bd26f49c94a2706e26db7006b902b801baf Mon Sep 17 00:00:00 2001 From: peterxcli Date: Mon, 9 Jun 2025 08:23:43 +0000 Subject: [PATCH 28/31] Disable the range compaction service by default and add a new benchmark generator for key operations in Ozone. Update related configuration keys and ensure proper handling of bucket bounds in the compactor. --- .../src/main/resources/ozone-default.xml | 2 +- .../apache/hadoop/ozone/om/OMConfigKeys.java | 2 +- .../ozone/freon/OmKeyBenchGenerator.java | 289 ++++++++++++++++++ .../om/compaction/OBSTableCompactor.java | 3 +- 4 files changed, 293 insertions(+), 3 deletions(-) create mode 100644 hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/OmKeyBenchGenerator.java diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 616dd4247d3a..2b604442391a 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -4786,7 +4786,7 @@ ozone.om.range.compaction.service.enabled - true + false OZONE, OM, PERFORMANCE Enable or disable the range compaction service that compacts specific ranges of data based on tombstone ratio. This service helps improve performance and reduce storage overhead by compacting ranges with high tombstone density. diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index d2fa3cbd6b48..d64a34f589a6 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -661,7 +661,7 @@ public final class OMConfigKeys { * Configuration keys for RocksDB range compaction service. */ public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_ENABLED = "ozone.om.range.compaction.service.enabled"; - public static final boolean OZONE_OM_RANGE_COMPACTION_SERVICE_ENABLED_DEFAULT = true; + public static final boolean OZONE_OM_RANGE_COMPACTION_SERVICE_ENABLED_DEFAULT = false; public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_INTERVAL = "ozone.om.range.compaction.service.interval"; public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_INTERVAL_DEFAULT = "30s"; public static final String OZONE_OM_RANGE_COMPACTION_SERVICE_TIMEOUT = "ozone.om.range.compaction.service.timeout"; diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/OmKeyBenchGenerator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/OmKeyBenchGenerator.java new file mode 100644 index 000000000000..f7d192d6960e --- /dev/null +++ b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/OmKeyBenchGenerator.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.freon; + +import static java.util.Collections.emptyMap; + +import com.codahale.metrics.MetricFilter; +import com.codahale.metrics.Timer; +import java.io.OutputStream; +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.LongAdder; +import java.util.function.Supplier; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageSize; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; +import org.kohsuke.MetaInfServices; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; + +/** + * Key‑level benchmark that supports CREATE, DELETE and LIST with user‑defined + * weights. It now keeps a concurrent set of *live* keys so DELETE/LIST always + * operate on existing entries; when the set is empty those two ops are skipped. + */ +@Command(name = "omkeybench", + description = "Generate CREATE/DELETE/LIST load toward OM with weighted mix.", + versionProvider = HddsVersionProvider.class, + mixinStandardHelpOptions = true, + showDefaultValues = true) +@MetaInfServices(FreonSubcommand.class) +public class OmKeyBenchGenerator extends BaseFreonGenerator implements Runnable { + + // ============ CLI ============ + + @Option(names = { "-v", "--volume" }, defaultValue = "vol1", description = "Volume name (created if absent)") + private String volume; + + @Option(names = { "-b", "--bucket" }, defaultValue = "bucket1", description = "Bucket name (created if absent)") + private String bucketName; + + @Option(names = "--weights", description = "Comma‑separated weights, e.g. create:5,delete:3,list:2") + private String weightSpec = "create:1,delete:1,list:1"; + + @Option(names = { "--batch-size" }, defaultValue = "1000", description = "Batch size for LIST_KEYS request") + private int batchSize; + + @Option(names = { "-s", "--size" }, defaultValue = "0", description = "Payload size when creating a key (bytes) " + + StorageSizeConverter.STORAGE_SIZE_DESCRIPTION, converter = StorageSizeConverter.class) + private StorageSize dataSize; + + @Mixin + private FreonReplicationOptions replicationOpt; + + // ------------ runtime state ------------ + private final EnumMap weights = new EnumMap<>(Op.class); + private double pCreate; // cumulative boundaries in [0,1) + private double pDelete; + + private OzoneBucket bucket; + private ReplicationConfig replConf; + private OzoneManagerProtocol om; + + private ContentGenerator contentGen; + private final LongAdder createdCounter = new LongAdder(); + private final LongAdder deletedCounter = new LongAdder(); + + /** Set of currently *live* keys for quick random sampling. */ + private final ConcurrentSkipListSet liveKeys = new ConcurrentSkipListSet<>(); + + @Override + public void run() { + try { + parseWeights(); + init(); + contentGen = new ContentGenerator(dataSize.toBytes(), 4096); + + OzoneConfiguration conf = createOzoneConfiguration(); + replConf = replicationOpt.fromParamsOrConfig(conf); + + try (OzoneClient client = createOzoneClient(null, conf)) { + ensureVolumeAndBucketExist(client, volume, bucketName); + bucket = client.getObjectStore().getVolume(volume).getBucket(bucketName); + om = createOmClient(conf, null); + runTests(this::oneIteration); + } finally { + if (om != null) { + om.close(); + } + } + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + /* ---------------- weight parsing ---------------- */ + + private void parseWeights() { + Arrays.stream(weightSpec.split(",")) + .forEach(pair -> { + String[] kv = pair.trim().split(":"); + if (kv.length != 2) { + throw new IllegalArgumentException("Bad --weights element: " + pair); + } + Op op = Op.valueOf(kv[0].toUpperCase()); + int w = Integer.parseInt(kv[1]); + if (w < 0) { + throw new IllegalArgumentException("Negative weight: " + pair); + } + weights.put(op, w); + }); + if (weights.isEmpty()) { + throw new IllegalArgumentException("No weights specified"); + } + int total = weights.values().stream().mapToInt(Integer::intValue).sum(); + pCreate = weights.getOrDefault(Op.CREATE, 0) / (double) total; + pDelete = pCreate + weights.getOrDefault(Op.DELETE, 0) / (double) total; + } + + /* ---------------- main loop ---------------- */ + + private void oneIteration(long globalCounter) throws Exception { + double r = ThreadLocalRandom.current().nextDouble(); + Op op = (r < pCreate) ? Op.CREATE : (r < pDelete ? Op.DELETE : Op.LIST); + + switch (op) { + case CREATE: + createKey(globalCounter); + break; + case DELETE: + deleteRandomKey(); + break; + case LIST: + listRandom(); + break; + default: + throw new IllegalStateException(); + } + } + + /* ---------------- operations ---------------- */ + + private void createKey(long counter) throws Exception { + String key = formatKey(counter); + Timer timer = getMetrics().timer(Op.CREATE.name()); + timer.time(() -> { + try (OutputStream os = bucket.createKey(key, dataSize.toBytes(), replConf, emptyMap())) { + contentGen.write(os); + } + createdCounter.increment(); + liveKeys.add(key); + return null; + }); + } + + private void deleteRandomKey() throws Exception { + if (liveKeys.isEmpty()) { + return; // nothing to delete now + } + String key = pickRandomKey(); + if (key == null) { + return; // race condition; skip + } + OmKeyArgs keyArgs = new OmKeyArgs.Builder() + .setVolumeName(volume) + .setBucketName(bucketName) + .setKeyName(key) + .build(); + Timer timer = getMetrics().timer(Op.DELETE.name()); + timer.time(() -> { + try { + om.deleteKey(keyArgs); + if (liveKeys.remove(key)) { + deletedCounter.increment(); + } + } catch (OMException ex) { + if (ex.getResult() != OMException.ResultCodes.KEY_NOT_FOUND) { + throw ex; + } + } + return null; + }); + } + + private void listRandom() throws Exception { + if (liveKeys.isEmpty()) { + return; // nothing to list + } + String start = pickRandomKey(); + if (start == null) { + return; + } + Timer timer = getMetrics().timer(Op.LIST.name()); + timer.time(() -> { + om.listKeys(volume, bucketName, start, "", batchSize); + return null; // ignore size check when data sparse + }); + } + /* ---------------- helpers ---------------- */ + + private String pickRandomKey() { + int size = liveKeys.size(); + if (size == 0) { + return null; + } + int index = ThreadLocalRandom.current().nextInt(size); + return liveKeys.stream().skip(index).findFirst().orElse(null); + } + + private static String formatKey(long n) { + return StringUtils.leftPad(Long.toString(n), 19, '0'); + } + + @Override + public Supplier realTimeStatusSupplier() { + final Map maxValueRecorder = new HashMap<>(); + final Map valueRecorder = new HashMap<>(); + final Map instantsRecorder = new HashMap<>(); + return () -> { + StringBuilder sb = new StringBuilder(); + sb.append(String.format("live=%d created=%d deleted=%d", liveKeys.size(), + createdCounter.sum(), deletedCounter.sum())); + + // Add rate information for each operation type + for (Map.Entry entry + : getMetrics().getTimers(MetricFilter.ALL).entrySet()) { + String name = entry.getKey(); + long maxValue = maxValueRecorder.getOrDefault(name, -1L); + long preValue = valueRecorder.getOrDefault(name, 0L); + Instant preInstant = instantsRecorder.getOrDefault(name, Instant.now()); + + long curValue = entry.getValue().getCount(); + Instant now = Instant.now(); + long duration = Duration.between(preInstant, now).getSeconds(); + long rate = ((curValue - preValue) / (duration == 0 ? 1 : duration)); + maxValue = Math.max(rate, maxValue); + + maxValueRecorder.put(name, maxValue); + valueRecorder.put(name, curValue); + instantsRecorder.put(name, now); + sb.append(' ') + .append(name) + .append(": rate ") + .append(rate) + .append(" max ") + .append(maxValue); + } + return sb.toString(); + }; + } + + @Override + public boolean allowEmptyPrefix() { + return true; + } + + enum Op { + CREATE, DELETE, LIST + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java index cc803f2cd9b2..a6a4d587763e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/OBSTableCompactor.java @@ -125,6 +125,7 @@ private KeyRange getNextBucketBound() { .getBucketIterator(currentBucketUpperBound); if (!iterator.hasNext()) { LOG.info("No more bucket bounds found"); + currentBucketUpperBound = null; return null; } @@ -135,7 +136,7 @@ private KeyRange getNextBucketBound() { LOG.info("Found bucket range: [{} - {}]", bucketStartKey, bucketEndKey); } else { bucketEndKey = StringUtils.getKeyPrefixUpperBound(bucketStartKey); - currentBucketUpperBound = null; + currentBucketUpperBound = bucketEndKey; LOG.info("Found last bucket range: [{} - {}]", bucketStartKey, bucketEndKey); } return new KeyRange(bucketStartKey, bucketEndKey); From e10d58adda757bba4e5057cf271e011d94b8b192 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 12 Jun 2025 06:03:31 +0000 Subject: [PATCH 29/31] Add max live keys limit to OmKeyBenchGenerator and optimize key retrieval. Enhance logging in AbstractCompactor and synchronize start method in RangeCompactionService. --- .../ozone/freon/OmKeyBenchGenerator.java | 33 ++++++++++++++++--- .../om/compaction/AbstractCompactor.java | 13 ++++++-- .../om/service/RangeCompactionService.java | 3 +- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/OmKeyBenchGenerator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/OmKeyBenchGenerator.java index f7d192d6960e..d3883de7ac98 100644 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/OmKeyBenchGenerator.java +++ b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/OmKeyBenchGenerator.java @@ -78,6 +78,10 @@ public class OmKeyBenchGenerator extends BaseFreonGenerator implements Runnable + StorageSizeConverter.STORAGE_SIZE_DESCRIPTION, converter = StorageSizeConverter.class) private StorageSize dataSize; + @Option(names = { "--max-live-keys" }, defaultValue = "100000", + description = "Maximum number of live keys to maintain for DELETE/LIST operations") + private int maxLiveKeys; + @Mixin private FreonReplicationOptions replicationOpt; @@ -169,6 +173,11 @@ private void oneIteration(long globalCounter) throws Exception { /* ---------------- operations ---------------- */ + /** + * Creates a key and optionally adds it to the liveKeys set for future DELETE/LIST operations. + * When the liveKeys set reaches maxLiveKeys limit, new keys are still created but not added + * to the set, preventing performance degradation from large set operations. + */ private void createKey(long counter) throws Exception { String key = formatKey(counter); Timer timer = getMetrics().timer(Op.CREATE.name()); @@ -177,7 +186,11 @@ private void createKey(long counter) throws Exception { contentGen.write(os); } createdCounter.increment(); - liveKeys.add(key); + + // Only add to liveKeys if we haven't reached the limit + if (liveKeys.size() < maxLiveKeys) { + liveKeys.add(key); + } return null; }); } @@ -232,8 +245,15 @@ private String pickRandomKey() { if (size == 0) { return null; } - int index = ThreadLocalRandom.current().nextInt(size); - return liveKeys.stream().skip(index).findFirst().orElse(null); + + // Convert to array for O(1) random access - more efficient than stream().skip() + String[] keysArray = liveKeys.toArray(new String[0]); + if (keysArray.length == 0) { + return null; // Race condition check + } + + int index = ThreadLocalRandom.current().nextInt(keysArray.length); + return keysArray[index]; } private static String formatKey(long n) { @@ -247,9 +267,14 @@ public Supplier realTimeStatusSupplier() { final Map instantsRecorder = new HashMap<>(); return () -> { StringBuilder sb = new StringBuilder(); - sb.append(String.format("live=%d created=%d deleted=%d", liveKeys.size(), + int currentLiveKeys = liveKeys.size(); + sb.append(String.format("live=%d/%d created=%d deleted=%d", currentLiveKeys, maxLiveKeys, createdCounter.sum(), deletedCounter.sum())); + if (currentLiveKeys >= maxLiveKeys) { + sb.append(" [LIMIT_REACHED]"); + } + // Add rate information for each operation type for (Map.Entry entry : getMetrics().getTimers(MetricFilter.ALL).entrySet()) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/AbstractCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/AbstractCompactor.java index 1b552d57efdb..0321d74584f9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/AbstractCompactor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/AbstractCompactor.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hdds.utils.BackgroundTaskResult; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.KeyRange; +import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.rocksdb.TableProperties; import org.slf4j.Logger; @@ -65,9 +66,11 @@ public String getTableName() { @Override public void compactRange(KeyRange range) { - try { + try (ManagedCompactRangeOptions options = new ManagedCompactRangeOptions()) { LOG.info("Compacting range {} for table {}", range, tableName); - dbStore.compactTable(tableName, range.getStartKey(), range.getEndKey()); + options.setBottommostLevelCompaction(ManagedCompactRangeOptions.BottommostLevelCompaction.kForce); + options.setExclusiveManualCompaction(true); + dbStore.compactTable(tableName, range.getStartKey(), range.getEndKey(), options); } catch (Exception e) { LOG.error("Failed to compact range {} for table {}", range, tableName, e); } @@ -123,7 +126,11 @@ protected DBStore getDBStore() { } protected void addRangeCompactionTask(KeyRange range) { + LOG.info("Adding range compaction task for range {} in table {}, compactRangeQueue size: {}", + range, tableName, compactRangeQueue.size()); compactRangeQueue.add(new CompactionTask(range)); + LOG.info("Added range compaction task for range {} in table {}, compactRangeQueue size: {}", + range, tableName, compactRangeQueue.size()); } private class CompactionTask implements BackgroundTask { @@ -143,7 +150,7 @@ public int getPriority() { public BackgroundTaskResult call() throws Exception { LOG.debug("Running compaction for range {} in table {}", range, tableName); - dbStore.compactTable(tableName, range.getStartKey(), range.getEndKey()); + compactRange(range); return () -> 1; } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java index b8bfa277bd71..d4b4bc1e4757 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RangeCompactionService.java @@ -81,7 +81,8 @@ public BackgroundTaskQueue getTasks() { } @Override - public void start() { + public synchronized void start() { + super.start(); scheduleCompactionChecks(); } From 4a8b81ccf5c97968cc29108249cf836eb8cc5d05 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 4 Jul 2025 21:59:04 +0800 Subject: [PATCH 30/31] Implement FSO compaction logic in FSOTableCompactor, introducing directory-aware range processing and adaptive splitting based on directory hierarchy. Enhance logging for better traceability during compaction operations. --- .../om/compaction/FSOTableCompactor.java | 639 +++++++++++++++--- 1 file changed, 543 insertions(+), 96 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOTableCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOTableCompactor.java index 9161ae7e1417..0a1c303a78ad 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOTableCompactor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOTableCompactor.java @@ -17,118 +17,565 @@ package org.apache.hadoop.ozone.om.compaction; -// import java.util.Iterator; +import com.google.common.base.Preconditions; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; import java.util.List; -// import java.util.Map; +import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.utils.db.KeyRange; -// import org.apache.hadoop.hdds.utils.db.cache.CacheKey; -// import org.apache.hadoop.hdds.utils.db.cache.CacheValue; -// import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -// import org.slf4j.Logger; -// import org.slf4j.LoggerFactory; +import org.apache.hadoop.hdds.utils.db.RDBStore; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.rocksdb.LiveFileMetaData; +import org.rocksdb.TableProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Compactor for FSO layout that compacts based on bucket and parent ID ranges. + * + * Unlike OBS compactor which uses bucket boundaries, FSO compactor uses a hierarchical + * approach based on parent-child relationships in the directory structure. + * + * Key features: + * 1. Parent ID-based splitting: Creates ranges based on directory hierarchy + * 2. Depth-aware compaction: Prioritizes shallow directories over deep ones + * 3. Directory-aware merging: Keeps related files/subdirectories together + * 4. Adaptive range sizing: Adjusts range size based on directory entry count */ public class FSOTableCompactor extends AbstractCompactor { - // private static final Logger LOG = LoggerFactory.getLogger(FSOTableCompactor.class); - - // private String nextBucket = null; - // private String nextParentId = null; - // private String nextKey; + private static final Logger LOG = LoggerFactory.getLogger(FSOTableCompactor.class); + private static final String OM_KEY_PREFIX = "/"; + + // Pagination state for resuming compaction + private Long currentVolumeId; + private Long currentBucketId; + private Long currentParentId; + private String resumeKey; + + // Cache for directory hierarchy information + private final NavigableMap directoryCache = new TreeMap<>(); + + // Configuration for adaptive splitting + private static final int MAX_DIRECTORY_DEPTH = 10; + private static final double DEPTH_WEIGHT_FACTOR = 0.8; // Prioritize shallow directories public FSOTableCompactor(CompactorBuilder builder) { super(builder); } + + /** + * Represents cached metadata about a directory. + */ + private static class DirectoryMetadata { + private final long objectId; + private final long parentId; + private final int depth; + private final String path; + private long entryCount; + private long tombstoneCount; + + DirectoryMetadata(long objectId, long parentId, int depth, String path) { + this.objectId = objectId; + this.parentId = parentId; + this.depth = depth; + this.path = path; + this.entryCount = 0; + this.tombstoneCount = 0; + } + + double getCompactionPriority(double tombstoneRatio) { + if (entryCount == 0) return 0; + double tombstoneScore = (double) tombstoneCount / entryCount; + // Prioritize shallow directories by reducing score based on depth + return tombstoneScore * Math.pow(DEPTH_WEIGHT_FACTOR, depth); + } + } @Override public void run() { + try { + LOG.info("Starting FSO range compaction scan for table {}", getTableName()); + List ranges = getRangesNeedingCompaction(); + LOG.info("Found {} ranges needing compaction for table {}, submitting to compaction queue", + ranges.size(), getTableName()); + for (KeyRange range : ranges) { + LOG.info("Submitting range [{}] for compaction", range); + addRangeCompactionTask(range); + } + LOG.info("Completed FSO range compaction scan for table {}", getTableName()); + } catch (Exception e) { + LOG.error("Failed to run FSO range compaction for table {}", getTableName(), e); + } } @Override protected void collectRangesNeedingCompaction(List ranges) { - // Iterator, CacheValue>> bucketIterator = getBucketIterator(); - // while (bucketIterator.hasNext()) { - // Map.Entry, CacheValue> entry = bucketIterator.next(); - // String bucketKey = entry.getKey().getCacheKey(); - // OmBucketInfo bucketInfo = entry.getValue().getCacheValue(); - - // if (nextBucket != null && !nextBucket.equals(bucketKey)) { - // continue; - // } - - // // For FSO, we need to handle parent IDs - // if (nextParentId != null) { - // // Continue with the same bucket but different parent ID - // KeyRange parentRange = new KeyRange( - // getParentKey(bucketKey, nextParentId), - // getNextParentKey(bucketKey, nextParentId)); - // processRange(parentRange, ranges); - // } else { - // // Start with the first parent ID for this bucket - // String firstParentId = getFirstParentId(bucketKey); - // if (firstParentId != null) { - // KeyRange parentRange = new KeyRange( - // getParentKey(bucketKey, firstParentId), - // getNextParentKey(bucketKey, firstParentId)); - // processRange(parentRange, ranges); - // } - // } - // } - } - - // private void processRange(KeyRange range, List ranges) { - // KeyRangeStats stats = getRangeStats(range); - - // if (stats.getNumEntries() <= getMaxEntriesSum()) { - // if (needsCompaction(stats, getMinTombstones(), getTombstoneRatio())) { - // ranges.add(range); - // } - // nextParentId = null; - // // nextKey = null; - // } else { - // String splitKey = findSplitKey(range); - // if (splitKey != null) { - // KeyRange splitRange = new KeyRange(range.getStartKey(), splitKey); - // KeyRangeStats splitStats = getRangeStats(splitRange); - // if (needsCompaction(splitStats, getMinTombstones(), getTombstoneRatio())) { - // ranges.add(splitRange); - // } - // // nextKey = splitKey; - // } - // } - // } - - // private Iterator, CacheValue>> getBucketIterator() { - // if (nextBucket != null) { - // return getMetadataManager().getBucketIterator(nextBucket); - // } - // return getMetadataManager().getBucketIterator(); - // } - - // private String getParentKey(String bucketKey, String parentId) { - // // Format: /volumeId/bucketId/parentId/ - // return bucketKey + "/" + parentId + "/"; - // } - - // private String getNextParentKey(String bucketKey, String parentId) { - // // Format: /volumeId/bucketId/parentId/ + 1 - // return bucketKey + "/" + parentId + "/\0"; - // } - - // private String getFirstParentId(String bucketKey) { - // // This is a simplified version - in reality, you'd need to implement - // // a proper way to get the first parent ID for a bucket - // return "0"; - // } - - // private String findSplitKey(KeyRange range) { - // // Binary search to find a split key that keeps the range under maxEntriesSum - // String startKey = range.getStartKey(); - // String endKey = range.getEndKey(); - - // // This is a simplified version - in reality, you'd need to implement - // // a proper binary search based on your key format - // return startKey + "\0"; - // } + LOG.info("Starting to collect FSO ranges needing compaction for table {}", getTableName()); + + try { + // First, build directory hierarchy cache + buildDirectoryCache(); + + // Process ranges based on directory hierarchy + for (int i = 0; i < getRangesPerRun(); i++) { + Pair rangeAndStats = prepareNextRange(); + if (rangeAndStats == null) { + LOG.info("No more ranges to process for table {}", getTableName()); + break; + } + + if (needsCompaction(rangeAndStats.getRight(), getMinTombstones(), getTombstoneRatio())) { + LOG.info("Range [{}] needs compaction with stats: {}", + rangeAndStats.getLeft(), rangeAndStats.getRight()); + ranges.add(rangeAndStats.getLeft()); + } else { + LOG.info("Range [{}] does not need compaction with stats: {}", + rangeAndStats.getLeft(), rangeAndStats.getRight()); + } + } + } catch (IOException e) { + LOG.error("Failed to collect FSO ranges for compaction for table {}", getTableName(), e); + } + + LOG.info("Collected {} FSO ranges needing compaction for table {}", ranges.size(), getTableName()); + } + + /** + * Build a cache of directory metadata for efficient range calculation. + */ + private void buildDirectoryCache() throws IOException { + if (getTableName().equals(getMetadataManager().getDirectoryTableName())) { + LOG.info("Building directory cache for FSO compaction"); + + Table dirTable = getMetadataManager().getDirectoryTable(); + try (TableIterator> iter = + dirTable.iterator()) { + + int count = 0; + while (iter.hasNext() && count < 1000) { // Limit cache size + Table.KeyValue kv = iter.next(); + OmDirectoryInfo dirInfo = kv.getValue(); + + // Calculate depth by counting parent traversals + int depth = calculateDirectoryDepth(dirInfo); + DirectoryMetadata metadata = new DirectoryMetadata( + dirInfo.getObjectID(), + dirInfo.getParentObjectID(), + depth, + kv.getKey() + ); + + directoryCache.put(dirInfo.getObjectID(), metadata); + count++; + } + } + + LOG.info("Built directory cache with {} entries", directoryCache.size()); + } + } + + /** + * Calculate the depth of a directory in the hierarchy. + */ + private int calculateDirectoryDepth(OmDirectoryInfo dirInfo) { + int depth = 0; + long parentId = dirInfo.getParentObjectID(); + + // Traverse up the hierarchy until we reach bucket level + while (depth < MAX_DIRECTORY_DEPTH && directoryCache.containsKey(parentId)) { + DirectoryMetadata parent = directoryCache.get(parentId); + parentId = parent.parentId; + depth++; + } + + return depth; + } + + /** + * Prepare the next range for compaction based on FSO hierarchy. + */ + private Pair prepareNextRange() throws IOException { + LOG.info("Preparing next FSO range for compaction"); + + // Determine starting point + KeyRange currentRange; + if (resumeKey != null) { + // Resume from last split point + LOG.info("Resuming from last split point: {}", resumeKey); + String endKey = findNextParentBoundary(resumeKey); + currentRange = new KeyRange(resumeKey, endKey); + resumeKey = null; + } else { + // Find next parent ID range to process + currentRange = getNextParentIdRange(); + if (currentRange == null) { + return null; + } + } + + // Get stats and handle splitting if needed + return processRangeWithSplitting(currentRange); + } + + /** + * Get the next parent ID range to process. + */ + private KeyRange getNextParentIdRange() throws IOException { + LOG.info("Getting next parent ID range"); + + // If we have a current position, continue from there + if (currentVolumeId != null && currentBucketId != null && currentParentId != null) { + return getNextParentRange(currentVolumeId, currentBucketId, currentParentId); + } + + // Otherwise, start from the beginning + return getFirstParentRange(); + } + + /** + * Get the first parent range in the table. + */ + private KeyRange getFirstParentRange() throws IOException { + // For FSO tables, we need to iterate through volume/bucket combinations + Iterator, CacheValue>> bucketIter = + getMetadataManager().getBucketIterator(); + + if (!bucketIter.hasNext()) { + LOG.info("No buckets found for FSO compaction"); + return null; + } + + Map.Entry, CacheValue> bucketEntry = bucketIter.next(); + OmBucketInfo bucketInfo = bucketEntry.getValue().getCacheValue(); + + currentVolumeId = bucketInfo.getVolumeId(); + currentBucketId = bucketInfo.getObjectID(); + currentParentId = bucketInfo.getObjectID(); // Start with bucket as parent + + String startKey = buildFSOKey(currentVolumeId, currentBucketId, currentParentId, ""); + String endKey = buildFSOKey(currentVolumeId, currentBucketId, currentParentId + 1, ""); + + LOG.info("First parent range: volumeId={}, bucketId={}, parentId={}, range=[{} - {}]", + currentVolumeId, currentBucketId, currentParentId, startKey, endKey); + + return new KeyRange(startKey, endKey); + } + + /** + * Get the next parent range after the current one. + */ + private KeyRange getNextParentRange(long volumeId, long bucketId, long parentId) throws IOException { + // For simplicity, increment parent ID + // In production, this would involve more sophisticated directory traversal + currentParentId = parentId + 1; + + // Check if we need to move to next bucket + if (currentParentId > parentId + 1000) { // Arbitrary limit for demo + return moveToNextBucket(); + } + + String startKey = buildFSOKey(volumeId, bucketId, currentParentId, ""); + String endKey = buildFSOKey(volumeId, bucketId, currentParentId + 1, ""); + + LOG.info("Next parent range: volumeId={}, bucketId={}, parentId={}, range=[{} - {}]", + volumeId, bucketId, currentParentId, startKey, endKey); + + return new KeyRange(startKey, endKey); + } + + /** + * Move to the next bucket for compaction. + */ + private KeyRange moveToNextBucket() throws IOException { + // Reset state and find next bucket + currentParentId = null; + + Iterator, CacheValue>> bucketIter = + getMetadataManager().getBucketIterator(buildBucketKey(currentVolumeId, currentBucketId)); + + if (!bucketIter.hasNext()) { + LOG.info("No more buckets to process"); + currentVolumeId = null; + currentBucketId = null; + return null; + } + + Map.Entry, CacheValue> bucketEntry = bucketIter.next(); + OmBucketInfo bucketInfo = bucketEntry.getValue().getCacheValue(); + + currentVolumeId = bucketInfo.getVolumeId(); + currentBucketId = bucketInfo.getObjectID(); + currentParentId = bucketInfo.getObjectID(); + + String startKey = buildFSOKey(currentVolumeId, currentBucketId, currentParentId, ""); + String endKey = buildFSOKey(currentVolumeId, currentBucketId, currentParentId + 1, ""); + + return new KeyRange(startKey, endKey); + } + + /** + * Process a range with potential splitting if it's too large. + */ + private Pair processRangeWithSplitting(KeyRange range) throws IOException { + CompoundKeyRangeStats compoundStats = getCompoundKeyRangeStatsFromRange(range); + + if (compoundStats.isEmpty()) { + LOG.info("Range [{}] has no entries", range); + return Pair.of(range, new KeyRangeStats()); + } + + LOG.info("Range [{}] has {} entries", range, compoundStats.getNumEntries()); + + // Check if range needs splitting + if (compoundStats.getNumEntries() <= getMaxCompactionEntries()) { + return Pair.of(range, compoundStats.getCompoundStats()); + } + + // Split range using FSO-aware logic + LOG.info("Range [{}] exceeds max entries ({} > {}), applying FSO splitting", + range, compoundStats.getNumEntries(), getMaxCompactionEntries()); + + List> splitRanges = splitRangeByDirectoryBoundaries( + range, compoundStats, getMaxCompactionEntries()); + + if (splitRanges.isEmpty()) { + // Fallback to SST-based splitting + splitRanges = findFitRanges(compoundStats.getKeyRangeStatsList(), getMaxCompactionEntries()); + } + + if (splitRanges.isEmpty()) { + LOG.warn("Failed to split range [{}], returning original", range); + return Pair.of(range, compoundStats.getCompoundStats()); + } + + // Take the first split range and save resume point + Pair firstRange = squashRanges(splitRanges); + String nextBoundary = findNextParentBoundary(firstRange.getLeft().getEndKey()); + + if (range.getEndKey().compareTo(nextBoundary) > 0) { + resumeKey = nextBoundary; + LOG.info("Split range [{}] into [{}], will resume from {}", + range, firstRange.getLeft(), resumeKey); + } + + return firstRange; + } + + /** + * Split a range based on directory boundaries for better FSO alignment. + */ + private List> splitRangeByDirectoryBoundaries( + KeyRange range, CompoundKeyRangeStats stats, long maxEntries) { + + LOG.info("Attempting directory-aware splitting for range [{}]", range); + List> result = new ArrayList<>(); + + // Group SST files by directory boundaries + Map>> directoryGroups = new HashMap<>(); + + for (Pair sstRange : stats.getKeyRangeStatsList()) { + String dirBoundary = extractDirectoryBoundary(sstRange.getLeft().getStartKey()); + directoryGroups.computeIfAbsent(dirBoundary, k -> new ArrayList<>()).add(sstRange); + } + + // Build ranges respecting directory boundaries + long currentEntries = 0; + List> currentGroup = new ArrayList<>(); + + for (Map.Entry>> entry : directoryGroups.entrySet()) { + List> dirSsts = entry.getValue(); + long dirEntries = dirSsts.stream() + .mapToLong(p -> p.getRight().getNumEntries()) + .sum(); + + if (currentEntries > 0 && currentEntries + dirEntries > maxEntries) { + // Current group is full, save it + if (!currentGroup.isEmpty()) { + result.add(squashRanges(currentGroup)); + } + currentGroup = new ArrayList<>(); + currentEntries = 0; + } + + currentGroup.addAll(dirSsts); + currentEntries += dirEntries; + } + + // Add remaining group + if (!currentGroup.isEmpty() && currentEntries <= maxEntries) { + result.add(squashRanges(currentGroup)); + } + + LOG.info("Directory-aware splitting produced {} ranges", result.size()); + return result; + } + + /** + * Extract directory boundary from an FSO key. + */ + private String extractDirectoryBoundary(String key) { + // FSO key format: /volumeId/bucketId/parentId/name + String[] parts = key.split(OM_KEY_PREFIX); + if (parts.length >= 4) { + // Return up to parent ID to group by directory + return OM_KEY_PREFIX + parts[1] + OM_KEY_PREFIX + parts[2] + OM_KEY_PREFIX + parts[3]; + } + return key; + } + + /** + * Find the next parent boundary after the given key. + */ + private String findNextParentBoundary(String key) { + // Extract parent ID and increment + String[] parts = key.split(OM_KEY_PREFIX); + if (parts.length >= 4) { + try { + long parentId = Long.parseLong(parts[3]); + return OM_KEY_PREFIX + parts[1] + OM_KEY_PREFIX + parts[2] + + OM_KEY_PREFIX + (parentId + 1) + OM_KEY_PREFIX; + } catch (NumberFormatException e) { + LOG.warn("Failed to parse parent ID from key: {}", key); + } + } + return StringUtils.getKeyPrefixUpperBound(key); + } + + /** + * Build an FSO key from components. + */ + private String buildFSOKey(long volumeId, long bucketId, long parentId, String name) { + return OM_KEY_PREFIX + volumeId + OM_KEY_PREFIX + bucketId + + OM_KEY_PREFIX + parentId + OM_KEY_PREFIX + name; + } + + /** + * Build a bucket key for iteration. + */ + private String buildBucketKey(long volumeId, long bucketId) { + return OM_KEY_PREFIX + volumeId + OM_KEY_PREFIX + bucketId; + } + + + /** + * Find ranges that fit within the max entries limit. + */ + private List> findFitRanges( + List> ranges, long maxEntries) { + + LOG.info("Finding ranges that fit within {} entries", maxEntries); + ranges.sort((a, b) -> a.getLeft().getStartKey().compareTo(b.getLeft().getStartKey())); + + List> out = new ArrayList<>(); + KeyRangeStats chunkStats = new KeyRangeStats(); + + for (Pair range : ranges) { + long n = range.getRight().getNumEntries(); + + if (n > maxEntries) { + LOG.warn("Range [{}] exceeds max entries ({} > {}), skipping", + range.getLeft(), n, maxEntries); + continue; + } + + if (chunkStats.getNumEntries() > 0 && chunkStats.getNumEntries() + n > maxEntries) { + LOG.info("Current chunk + range would exceed max entries, stopping"); + return out; + } + + out.add(range); + chunkStats.add(range.getRight()); + } + + return out; + } + + /** + * Squash multiple ranges into a single range. + */ + private Pair squashRanges(List> list) { + Preconditions.checkNotNull(list, "list is null"); + Preconditions.checkArgument(!list.isEmpty(), "list is empty"); + + String minKey = list.get(0).getLeft().getStartKey(); + String maxKey = list.get(0).getLeft().getEndKey(); + KeyRangeStats stats = new KeyRangeStats(); + + for (Pair range : list) { + minKey = StringUtils.min(minKey, range.getLeft().getStartKey()); + maxKey = StringUtils.max(maxKey, range.getLeft().getEndKey()); + stats.add(range.getRight()); + } + + return Pair.of(new KeyRange(minKey, maxKey), stats); + } + + /** + * Get compound statistics for a key range. + */ + private CompoundKeyRangeStats getCompoundKeyRangeStatsFromRange(KeyRange range) throws IOException { + LOG.info("Getting compound stats for FSO range [{}]", range); + + List> keyRangeStatsList = new ArrayList<>(); + List liveFileMetaDataList = ((RDBStore)getDBStore()).getDb().getLiveFilesMetaData(); + Map fileMap = new HashMap<>(); + + for (LiveFileMetaData metaData : liveFileMetaDataList) { + fileMap.put(metaData.path() + metaData.fileName(), metaData); + } + + Map propsMap = getDBStore().getPropertiesOfTableInRange( + getTableName(), Collections.singletonList(range)); + + for (Map.Entry entry : propsMap.entrySet()) { + String filePath = entry.getKey(); + LiveFileMetaData meta = fileMap.get(filePath); + if (meta == null) { + LOG.warn("LiveFileMetaData not found for file {}", filePath); + continue; + } + + KeyRange keyRange = new KeyRange(meta.smallestKey(), meta.largestKey()); + KeyRangeStats stats = KeyRangeStats.fromTableProperties(entry.getValue()); + keyRangeStatsList.add(Pair.of(keyRange, stats)); + + // Update directory cache if available + updateDirectoryCacheWithStats(keyRange, stats); + } + + return new CompoundKeyRangeStats(keyRangeStatsList); + } + + /** + * Update directory cache with statistics from SST files. + */ + private void updateDirectoryCacheWithStats(KeyRange range, KeyRangeStats stats) { + String startKey = range.getStartKey(); + String[] parts = startKey.split(OM_KEY_PREFIX); + + if (parts.length >= 4) { + try { + long parentId = Long.parseLong(parts[3]); + DirectoryMetadata dirMeta = directoryCache.get(parentId); + if (dirMeta != null) { + dirMeta.entryCount += stats.getNumEntries(); + dirMeta.tombstoneCount += stats.getNumDeletions(); + } + } catch (NumberFormatException e) { + // Ignore parsing errors + } + } + } } From bc666bd4b3e70f1bb404b2b8cb988d0a0d776ccc Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 28 Aug 2025 10:33:03 +0000 Subject: [PATCH 31/31] fix compile error --- .../om/compaction/FSOTableCompactor.java | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOTableCompactor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOTableCompactor.java index 0a1c303a78ad..e140cd4aa569 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOTableCompactor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/compaction/FSOTableCompactor.java @@ -95,11 +95,28 @@ private static class DirectoryMetadata { this.tombstoneCount = 0; } - double getCompactionPriority(double tombstoneRatio) { - if (entryCount == 0) return 0; - double tombstoneScore = (double) tombstoneCount / entryCount; - // Prioritize shallow directories by reducing score based on depth - return tombstoneScore * Math.pow(DEPTH_WEIGHT_FACTOR, depth); + public long getObjectId() { + return objectId; + } + + public long getParentId() { + return parentId; + } + + public int getDepth() { + return depth; + } + + public String getPath() { + return path; + } + + public long getEntryCount() { + return entryCount; + } + + public long getTombstoneCount() { + return tombstoneCount; } } @@ -156,7 +173,7 @@ protected void collectRangesNeedingCompaction(List ranges) { * Build a cache of directory metadata for efficient range calculation. */ private void buildDirectoryCache() throws IOException { - if (getTableName().equals(getMetadataManager().getDirectoryTableName())) { + if (getTableName().equals(getMetadataManager().getDirectoryTable().getName())) { LOG.info("Building directory cache for FSO compaction"); Table dirTable = getMetadataManager().getDirectoryTable(); @@ -260,7 +277,7 @@ private KeyRange getFirstParentRange() throws IOException { Map.Entry, CacheValue> bucketEntry = bucketIter.next(); OmBucketInfo bucketInfo = bucketEntry.getValue().getCacheValue(); - currentVolumeId = bucketInfo.getVolumeId(); + currentVolumeId = getMetadataManager().getVolumeId(bucketInfo.getVolumeName()); currentBucketId = bucketInfo.getObjectID(); currentParentId = bucketInfo.getObjectID(); // Start with bucket as parent @@ -315,7 +332,7 @@ private KeyRange moveToNextBucket() throws IOException { Map.Entry, CacheValue> bucketEntry = bucketIter.next(); OmBucketInfo bucketInfo = bucketEntry.getValue().getCacheValue(); - currentVolumeId = bucketInfo.getVolumeId(); + currentVolumeId = getMetadataManager().getVolumeId(bucketInfo.getVolumeName()); currentBucketId = bucketInfo.getObjectID(); currentParentId = bucketInfo.getObjectID(); @@ -468,7 +485,6 @@ private String buildBucketKey(long volumeId, long bucketId) { return OM_KEY_PREFIX + volumeId + OM_KEY_PREFIX + bucketId; } - /** * Find ranges that fit within the max entries limit. */ @@ -571,7 +587,7 @@ private void updateDirectoryCacheWithStats(KeyRange range, KeyRangeStats stats) DirectoryMetadata dirMeta = directoryCache.get(parentId); if (dirMeta != null) { dirMeta.entryCount += stats.getNumEntries(); - dirMeta.tombstoneCount += stats.getNumDeletions(); + dirMeta.tombstoneCount += stats.getNumDeletion(); } } catch (NumberFormatException e) { // Ignore parsing errors