diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java index 3d5d785f869c..5427f2093663 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java @@ -107,6 +107,7 @@ public final class DBStoreBuilder { private boolean enableCompactionLog; private long maxTimeAllowedForSnapshotInDag; private long pruneCompactionDagDaemonRunInterval; + private boolean createCheckpointDirs = true; // this is to track the total size of dbUpdates data since sequence // number in request to avoid increase in heap memory. private long maxDbUpdatesSizeThreshold; @@ -228,7 +229,8 @@ public DBStore build() throws IOException { return new RDBStore(dbFile, rocksDBOption, writeOptions, tableConfigs, registry, openReadOnly, maxFSSnapshots, dbJmxBeanNameName, enableCompactionLog, maxTimeAllowedForSnapshotInDag, - pruneCompactionDagDaemonRunInterval, maxDbUpdatesSizeThreshold); + pruneCompactionDagDaemonRunInterval, maxDbUpdatesSizeThreshold, + createCheckpointDirs); } finally { tableConfigs.forEach(TableConfig::close); } @@ -290,6 +292,10 @@ public DBStoreBuilder setEnableCompactionLog(boolean enableCompactionLog) { return this; } + public DBStoreBuilder setCreateCheckpointDirs(boolean createCheckpointDirs) { + this.createCheckpointDirs = createCheckpointDirs; + return this; + } /** * Set the {@link ManagedDBOptions} and default * {@link ManagedColumnFamilyOptions} based on {@code prof}. diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index aa95bd32e5b4..92e95fc3a4fe 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@ -21,6 +21,8 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collection; @@ -84,7 +86,7 @@ public RDBStore(File dbFile, ManagedDBOptions options, this(dbFile, options, new ManagedWriteOptions(), families, new CodecRegistry(), false, 1000, null, false, TimeUnit.DAYS.toMillis(1), TimeUnit.HOURS.toMillis(1), - maxDbUpdatesSizeThreshold); + maxDbUpdatesSizeThreshold, true); } @SuppressWarnings("parameternumber") @@ -94,7 +96,9 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, String dbJmxBeanNameName, boolean enableCompactionLog, long maxTimeAllowedForSnapshotInDag, long compactionDagDaemonInterval, - long maxDbUpdatesSizeThreshold) + long maxDbUpdatesSizeThreshold, + boolean createCheckpointDirs) + throws IOException { Preconditions.checkNotNull(dbFile, "DB file location cannot be null"); Preconditions.checkNotNull(families); @@ -133,29 +137,22 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, } //create checkpoints directory if not exists. - checkpointsParentDir = - dbLocation.getParent() + OM_KEY_PREFIX + OM_CHECKPOINT_DIR; - File checkpointsDir = new File(checkpointsParentDir); - if (!checkpointsDir.exists()) { - boolean success = checkpointsDir.mkdir(); - if (!success) { - throw new IOException( - "Unable to create RocksDB checkpoint directory: " + - checkpointsParentDir); - } + if (!createCheckpointDirs) { + checkpointsParentDir = null; + } else { + Path checkpointsParentDirPath = + Paths.get(dbLocation.getParent(), OM_CHECKPOINT_DIR); + checkpointsParentDir = checkpointsParentDirPath.toString(); + Files.createDirectories(checkpointsParentDirPath); } - //create snapshot checkpoint directory if does not exist. - snapshotsParentDir = Paths.get(dbLocation.getParent(), - OM_SNAPSHOT_CHECKPOINT_DIR).toString(); - File snapshotsDir = new File(snapshotsParentDir); - if (!snapshotsDir.exists()) { - boolean success = snapshotsDir.mkdirs(); - if (!success) { - throw new IOException( - "Unable to create RocksDB snapshot checkpoint directory: " + - snapshotsParentDir); - } + if (!createCheckpointDirs) { + snapshotsParentDir = null; + } else { + Path snapshotsParentDirPath = + Paths.get(dbLocation.getParent(), OM_SNAPSHOT_CHECKPOINT_DIR); + snapshotsParentDir = snapshotsParentDirPath.toString(); + Files.createDirectories(snapshotsParentDirPath); } if (enableCompactionLog) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 1816d4c8723b..5d4d28d94d31 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -382,7 +382,7 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) RDBCheckpointUtils.waitForCheckpointDirectoryExist(checkpoint); } setStore(loadDB(conf, metaDir, dbName, false, - java.util.Optional.of(Boolean.TRUE))); + java.util.Optional.of(Boolean.TRUE), false)); initializeOmTables(false); } @@ -520,7 +520,7 @@ public void start(OzoneConfiguration configuration) throws IOException { public static DBStore loadDB(OzoneConfiguration configuration, File metaDir) throws IOException { return loadDB(configuration, metaDir, OM_DB_NAME, false, - java.util.Optional.empty()); + java.util.Optional.empty(), true); } public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, @@ -528,6 +528,16 @@ public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, java.util.Optional disableAutoCompaction) throws IOException { + return loadDB(configuration, metaDir, dbName, readOnly, + disableAutoCompaction, true); + } + + public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, + String dbName, boolean readOnly, + java.util.Optional + disableAutoCompaction, + boolean createCheckpointDirs) + throws IOException { final int maxFSSnapshots = configuration.getInt( OZONE_OM_FS_SNAPSHOT_MAX_LIMIT, OZONE_OM_FS_SNAPSHOT_MAX_LIMIT_DEFAULT); RocksDBConfiguration rocksDBConfiguration = @@ -537,7 +547,8 @@ public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, .setOpenReadOnly(readOnly) .setPath(Paths.get(metaDir.getPath())) .setMaxFSSnapshots(maxFSSnapshots) - .setEnableCompactionLog(true); + .setEnableCompactionLog(true) + .setCreateCheckpointDirs(createCheckpointDirs); disableAutoCompaction.ifPresent( dbStoreBuilder::disableDefaultCFAutoCompaction); return addOMTablesAndCodecs(dbStoreBuilder).build(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java index cdc473646b7f..5d3b988bc678 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java @@ -150,7 +150,7 @@ public BackgroundTaskResult call() throws Exception { try (RDBStore rdbStore = (RDBStore) OmMetadataManagerImpl .loadDB(ozoneManager.getConfiguration(), new File(snapshotCheckpointDir), - dbName, true, Optional.of(Boolean.TRUE))) { + dbName, true, Optional.of(Boolean.TRUE), false)) { RocksDatabase db = rdbStore.getDb(); db.deleteFilesNotMatchingPrefix(prefixPairs, filterFunction); }