Skip to content

HDDS-12193. Provide a configurable option to disable RDBStoreMetrics for Snapshotted DB. #7982

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adoroszlai merged 7 commits into from
Mar 5, 2025
Merged

HDDS-12193. Provide a configurable option to disable RDBStoreMetrics for Snapshotted DB. #7982

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a456a97
HDDS-12193. Provide a configurable option to disable RDBStoreMetrics.
Feb 27, 2025
70d978a
HDDS-12193. Fixed findbugs issue.
Feb 27, 2025
57cf760
Merge branch 'apache:master' into HDDS-12193
SaketaChalamchala Feb 27, 2025
4da7e05
HDDS-12193. Configuration to disable Rocks DB metrics collection for …
Feb 28, 2025
0c2485a
HDDS-12193. Used OMConfigKeys. Fixed checkstyle.
Mar 3, 2025
19cdf6a
Merge branch 'apache:master' into HDDS-12193
SaketaChalamchala Mar 4, 2025
1d334c4
HDDS-12193. Disable Snaoshot DB metrics by default.
Mar 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions hadoop-hdds/common/src/main/resources/ozone-default.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1518,6 +1518,16 @@
to the OM.
</description>
</property>

<property>
<name>ozone.om.snapshot.rocksdb.metrics.enabled</name>
<value>true</value>
Copy link
Contributor

@hemantk-12 hemantk-12 Mar 4, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we disable it but default and enable it on demand? Because when snapshot size grows, it will background services like SnapshotDeletingService.

Copy link
Contributor Author

@SaketaChalamchala SaketaChalamchala Mar 4, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, that sounds good.

<tag>OZONE, OM</tag>
<description>
Skip collecting RocksDBStore metrics for Snapshotted DB.
</description>
</property>

<property>
<name>hdds.rest.http-address</name>
<value>0.0.0.0:9880</value>
Expand Down
8 changes: 7 additions & 1 deletion hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ public final class DBStoreBuilder {
private final DBProfile defaultCfProfile;
private boolean enableCompactionDag;
private boolean createCheckpointDirs = true;
private boolean enableRocksDbMetrics = true;
// this is to track the total size of dbUpdates data since sequence
// number in request to avoid increase in heap memory.
private long maxDbUpdatesSizeThreshold;
Expand Down Expand Up @@ -232,7 +233,7 @@ public DBStore build() throws IOException {
return new RDBStore(dbFile, rocksDBOption, statistics, writeOptions, tableConfigs,
registry.build(), openReadOnly, maxFSSnapshots, dbJmxBeanNameName,
enableCompactionDag, maxDbUpdatesSizeThreshold, createCheckpointDirs,
configuration, threadNamePrefix);
configuration, threadNamePrefix, enableRocksDbMetrics);
} finally {
tableConfigs.forEach(TableConfig::close);
}
Expand Down Expand Up @@ -302,6 +303,11 @@ public DBStoreBuilder setCreateCheckpointDirs(boolean createCheckpointDirs) {
this.createCheckpointDirs = createCheckpointDirs;
return this;
}

public DBStoreBuilder setEnableRocksDbMetrics(boolean enableRocksDbMetrics) {
this.enableRocksDbMetrics = enableRocksDbMetrics;
return this;
}
/**
* Set the {@link ManagedDBOptions} and default
* {@link ManagedColumnFamilyOptions} based on {@code prof}.
Expand Down
18 changes: 12 additions & 6 deletions hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,8 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, ManagedStatistics stati
String dbJmxBeanName, boolean enableCompactionDag,
long maxDbUpdatesSizeThreshold,
boolean createCheckpointDirs,
ConfigurationSource configuration, String threadNamePrefix)
ConfigurationSource configuration, String threadNamePrefix,
boolean enableRocksDBMetrics)

throws IOException {
this.threadNamePrefix = threadNamePrefix;
Expand Down Expand Up @@ -118,13 +119,18 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, ManagedStatistics stati
dbJmxBeanName = dbFile.getName();
}
// Use statistics instead of dbOptions.statistics() to avoid repeated init.
metrics = RocksDBStoreMetrics.create(statistics, db, dbJmxBeanName);
if (metrics == null) {
LOG.warn("Metrics registration failed during RocksDB init, " +
if (!enableRocksDBMetrics) {
LOG.debug("Skipped Metrics registration during RocksDB init, " +
"db path :{}", dbJmxBeanName);
} else {
LOG.debug("Metrics registration succeed during RocksDB init, " +
"db path :{}", dbJmxBeanName);
metrics = RocksDBStoreMetrics.create(statistics, db, dbJmxBeanName);
if (metrics == null) {
LOG.warn("Metrics registration failed during RocksDB init, " +
"db path :{}", dbJmxBeanName);
} else {
LOG.debug("Metrics registration succeed during RocksDB init, " +
"db path :{}", dbJmxBeanName);
}
}

// Create checkpoints and snapshot directories if not exists.
Expand Down
2 changes: 1 addition & 1 deletion hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ public static RDBStore newRDBStore(File dbFile, ManagedDBOptions options,
throws IOException {
return new RDBStore(dbFile, options, null, new ManagedWriteOptions(), families,
CodecRegistry.newBuilder().build(), false, 1000, null, false,
maxDbUpdatesSizeThreshold, true, null, "");
maxDbUpdatesSizeThreshold, true, null, "", true);
}

public static final int MAX_DB_UPDATES_SIZE_THRESHOLD = 80;
Expand Down
4 changes: 4 additions & 0 deletions hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,10 @@ private OMConfigKeys() {
public static final String OZONE_OM_METRICS_SAVE_INTERVAL =
"ozone.om.save.metrics.interval";
public static final String OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT = "5m";
public static final String OZONE_OM_SNAPSHOT_ROCKSDB_METRICS_ENABLED =
"ozone.om.snapshot.rocksdb.metrics.enabled";
public static final boolean
OZONE_OM_SNAPSHOT_ROCKSDB_METRICS_ENABLED_DEFAULT = true;

/**
* OM Ratis related configurations.
Expand Down
26 changes: 17 additions & 9 deletions ...p-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_FS_SNAPSHOT_MAX_LIMIT_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_ROCKSDB_METRICS_ENABLED;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_ROCKSDB_METRICS_ENABLED_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_CHECKPOINT_DIR_CREATION_POLL_TIMEOUT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_CHECKPOINT_DIR_CREATION_POLL_TIMEOUT_DEFAULT;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND;
Expand Down Expand Up @@ -392,7 +394,8 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name)
omEpoch = 0;
int maxOpenFiles = conf.getInt(OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES, OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES_DEFAULT);

setStore(loadDB(conf, dir, name, true, Optional.of(Boolean.TRUE), maxOpenFiles, false, false));
setStore(loadDB(conf, dir, name, true, Optional.of(Boolean.TRUE),
maxOpenFiles, false, false, true));
initializeOmTables(CacheType.PARTIAL_CACHE, false);
perfMetrics = null;
}
Expand Down Expand Up @@ -425,7 +428,9 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name)
checkSnapshotDirExist(checkpoint);
}
setStore(loadDB(conf, metaDir, dbName, false,
java.util.Optional.of(Boolean.TRUE), maxOpenFiles, false, false));
java.util.Optional.of(Boolean.TRUE), maxOpenFiles, false, false,
conf.getBoolean(OZONE_OM_SNAPSHOT_ROCKSDB_METRICS_ENABLED,
OZONE_OM_SNAPSHOT_ROCKSDB_METRICS_ENABLED_DEFAULT)));
initializeOmTables(CacheType.PARTIAL_CACHE, false);
} catch (IOException e) {
stop();
Expand Down Expand Up @@ -571,16 +576,18 @@ public void start(OzoneConfiguration configuration) throws IOException {
}

public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, int maxOpenFiles) throws IOException {
return loadDB(configuration, metaDir, OM_DB_NAME, false, java.util.Optional.empty(), maxOpenFiles, true, true);
return loadDB(configuration, metaDir, OM_DB_NAME, false,
java.util.Optional.empty(), maxOpenFiles, true, true, true);
}

@SuppressWarnings("checkstyle:parameternumber")
public static DBStore loadDB(OzoneConfiguration configuration, File metaDir,
String dbName, boolean readOnly,
java.util.Optional<Boolean> disableAutoCompaction,
int maxOpenFiles,
boolean enableCompactionDag,
boolean createCheckpointDirs)
String dbName, boolean readOnly,
java.util.Optional<Boolean> disableAutoCompaction,
int maxOpenFiles,
boolean enableCompactionDag,
boolean createCheckpointDirs,
boolean enableRocksDBMetrics)
throws IOException {
final int maxFSSnapshots = configuration.getInt(
OZONE_OM_FS_SNAPSHOT_MAX_LIMIT, OZONE_OM_FS_SNAPSHOT_MAX_LIMIT_DEFAULT);
Expand All @@ -593,7 +600,8 @@ public static DBStore loadDB(OzoneConfiguration configuration, File metaDir,
.setMaxFSSnapshots(maxFSSnapshots)
.setEnableCompactionDag(enableCompactionDag)
.setCreateCheckpointDirs(createCheckpointDirs)
.setMaxNumberOfOpenFiles(maxOpenFiles);
.setMaxNumberOfOpenFiles(maxOpenFiles)
.setEnableRocksDbMetrics(enableRocksDBMetrics);
disableAutoCompaction.ifPresent(
dbStoreBuilder::disableDefaultCFAutoCompaction);
return addOMTablesAndCodecs(dbStoreBuilder).build();
Expand Down
17 changes: 16 additions & 1 deletion ...p-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,13 @@
import java.util.Set;
import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.TimeoutException;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.HddsWhiteboxTestUtils;
import org.apache.hadoop.hdds.utils.db.DBStore;
import org.apache.hadoop.hdds.utils.db.RDBBatchOperation;
import org.apache.hadoop.hdds.utils.db.RDBStore;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
Expand All @@ -78,6 +80,7 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.io.TempDir;
import org.slf4j.event.Level;

/**
* Unit test ozone snapshot manager.
Expand Down Expand Up @@ -108,6 +111,8 @@ void init(@TempDir File tempDir) throws Exception {
// Only allow one entry in cache so each new one causes an eviction
configuration.setInt(
OMConfigKeys.OZONE_OM_SNAPSHOT_CACHE_MAX_SIZE, 1);
configuration.setBoolean(
OMConfigKeys.OZONE_OM_SNAPSHOT_ROCKSDB_METRICS_ENABLED, false);

OmTestManagers omTestManagers = new OmTestManagers(configuration);
om = omTestManagers.getOzoneManager();
Expand Down Expand Up @@ -135,8 +140,12 @@ public void testSnapshotFeatureFlagSafetyCheck() throws IOException {
}

@Test
public void testCloseOnEviction() throws IOException {
public void testCloseOnEviction() throws IOException,
InterruptedException, TimeoutException {

GenericTestUtils.setLogLevel(RDBStore.getLogger(), Level.DEBUG);
GenericTestUtils.LogCapturer logCapture =
GenericTestUtils.LogCapturer.captureLogs(RDBStore.getLogger());
// set up db tables
Table<String, OmVolumeArgs> volumeTable = mock(Table.class);
Table<String, OmBucketInfo> bucketTable = mock(Table.class);
Expand Down Expand Up @@ -213,6 +222,12 @@ public void testCloseOnEviction() throws IOException {

// confirm store was closed
verify(firstSnapshotStore, timeout(3000).times(1)).close();

// Verify RocksDBStoreMetrics registration is skipped.
String msg = "Skipped Metrics registration during RocksDB init";
GenericTestUtils.waitFor(() -> {
return logCapture.getOutput().contains(msg);
}, 100, 30_000);
}

@BeforeEach
Expand Down