org.bouncycastle
bcprov-jdk15on
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java
index b84cc5d1cc91..12a8017d0397 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java
@@ -17,13 +17,15 @@
*/
package org.apache.hadoop.ozone.om;
+import java.io.File;
import java.io.IOException;
import java.util.Properties;
import java.util.UUID;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
-import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.hdds.server.ServerUtils;
import org.apache.hadoop.ozone.common.Storage;
import static org.apache.hadoop.ozone.OzoneConsts.SCM_ID;
@@ -43,7 +45,7 @@ public class OMStorage extends Storage {
* @throws IOException if any directories are inaccessible.
*/
public OMStorage(OzoneConfiguration conf) throws IOException {
- super(NodeType.OM, OmUtils.getOmDbDir(conf), STORAGE_DIR);
+ super(NodeType.OM, getOmDbDir(conf), STORAGE_DIR);
}
public void setScmId(String scmId) throws IOException {
@@ -104,4 +106,15 @@ protected Properties getNodeProperties() {
}
return omProperties;
}
+
+ /**
+ * Get the location where OM should store its metadata directories.
+ * Fall back to OZONE_METADATA_DIRS if not defined.
+ *
+ * @param conf - Config
+ * @return File path, after creating all the required Directories.
+ */
+ public static File getOmDbDir(Configuration conf) {
+ return ServerUtils.getDBPath(conf, OMConfigKeys.OZONE_OM_DB_DIRS);
+ }
}
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index 12f17741556a..97b5d8cffd37 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -40,7 +40,6 @@
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.hdds.utils.db.cache.TableCacheImpl;
-import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.om.codec.OmBucketInfoCodec;
@@ -243,7 +242,7 @@ public void start(OzoneConfiguration configuration) throws IOException {
// We need to create the DB here, as when during restart, stop closes the
// db, so we need to create the store object and initialize DB.
if (store == null) {
- File metaDir = OmUtils.getOmDbDir(configuration);
+ File metaDir = OMStorage.getOmDbDir(configuration);
RocksDBConfiguration rocksDBConfiguration =
configuration.getObject(RocksDBConfiguration.class);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index fa25a8c5cb16..7c2c49e7dce6 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -345,7 +345,7 @@ private OzoneManager(OzoneConfiguration conf) throws IOException,
// Read configuration and set values.
ozAdmins = conf.getTrimmedStringCollection(OZONE_ADMINISTRATORS);
- omMetaDir = OmUtils.getOmDbDir(configuration);
+ omMetaDir = OMStorage.getOmDbDir(configuration);
this.isAclEnabled = conf.getBoolean(OZONE_ACL_ENABLED,
OZONE_ACL_ENABLED_DEFAULT);
this.scmBlockSize = (long) conf.getStorageSize(OZONE_SCM_BLOCK_SIZE,
@@ -418,7 +418,8 @@ private OzoneManager(OzoneConfiguration conf) throws IOException,
if (isRatisEnabled) {
// Create Ratis storage dir
- String omRatisDirectory = OmUtils.getOMRatisDirectory(configuration);
+ String omRatisDirectory =
+ OzoneManagerRatisServer.getOMRatisDirectory(configuration);
if (omRatisDirectory == null || omRatisDirectory.isEmpty()) {
throw new IllegalArgumentException(HddsConfigKeys.OZONE_METADATA_DIRS +
" must be defined.");
@@ -426,7 +427,7 @@ private OzoneManager(OzoneConfiguration conf) throws IOException,
OmUtils.createOMDir(omRatisDirectory);
// Create Ratis snapshot dir
omRatisSnapshotDir = OmUtils.createOMDir(
- OmUtils.getOMRatisSnapshotDirectory(configuration));
+ OzoneManagerRatisServer.getOMRatisSnapshotDirectory(configuration));
if (peerNodes != null && !peerNodes.isEmpty()) {
this.omSnapshotProvider = new OzoneManagerSnapshotProvider(
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java
index 7cab9d2738ab..5735b6f951dc 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java
@@ -23,6 +23,7 @@
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -34,11 +35,12 @@
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import com.google.common.base.Strings;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.ServiceException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.StorageUnit;
-import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.hdds.server.ServerUtils;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.ha.OMNodeDetails;
import org.apache.hadoop.ozone.om.OzoneManager;
@@ -364,7 +366,7 @@ private RaftProperties newRaftProperties(Configuration conf) {
}
// Set Ratis storage directory
- String storageDir = OmUtils.getOMRatisDirectory(conf);
+ String storageDir = OzoneManagerRatisServer.getOMRatisDirectory(conf);
RaftServerConfigKeys.setStorageDirs(properties,
Collections.singletonList(new File(storageDir)));
@@ -645,4 +647,26 @@ private UUID getRaftGroupIdFromOmServiceId(String omServiceId) {
public long getStateMachineLastAppliedIndex() {
return omStateMachine.getLastAppliedIndex();
}
+
+ /**
+ * Get the local directory where ratis logs will be stored.
+ */
+ public static String getOMRatisDirectory(Configuration conf) {
+ String storageDir = conf.get(OMConfigKeys.OZONE_OM_RATIS_STORAGE_DIR);
+
+ if (Strings.isNullOrEmpty(storageDir)) {
+ storageDir = ServerUtils.getDefaultRatisDirectory(conf);
+ }
+ return storageDir;
+ }
+
+ public static String getOMRatisSnapshotDirectory(Configuration conf) {
+ String snapshotDir = conf.get(OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_DIR);
+
+ if (Strings.isNullOrEmpty(snapshotDir)) {
+ snapshotDir = Paths.get(getOMRatisDirectory(conf),
+ "snapshot").toString();
+ }
+ return snapshotDir;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMStorage.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMStorage.java
new file mode 100644
index 000000000000..d752ec1d3794
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMStorage.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om;
+
+import java.io.File;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.test.GenericTestUtils;
+
+import org.apache.commons.io.FileUtils;
+import static org.junit.Assert.*;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+/**
+ * Testing OMStorage class.
+ */
+public class TestOMStorage {
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ /**
+ * Test {@link OMStorage#getOmDbDir}.
+ */
+ @Test
+ public void testGetOmDbDir() {
+ final File testDir = createTestDir();
+ final File dbDir = new File(testDir, "omDbDir");
+ final File metaDir = new File(testDir, "metaDir"); // should be ignored.
+ final Configuration conf = new OzoneConfiguration();
+ conf.set(OMConfigKeys.OZONE_OM_DB_DIRS, dbDir.getPath());
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.getPath());
+
+ try {
+ assertEquals(dbDir, OMStorage.getOmDbDir(conf));
+ assertTrue(dbDir.exists()); // should have been created.
+ } finally {
+ FileUtils.deleteQuietly(dbDir);
+ }
+ }
+
+ /**
+ * Test {@link OMStorage#getOmDbDir} with fallback to OZONE_METADATA_DIRS
+ * when OZONE_OM_DB_DIRS is undefined.
+ */
+ @Test
+ public void testGetOmDbDirWithFallback() {
+ final File testDir = createTestDir();
+ final File metaDir = new File(testDir, "metaDir");
+ final Configuration conf = new OzoneConfiguration();
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.getPath());
+
+ try {
+ assertEquals(metaDir, OMStorage.getOmDbDir(conf));
+ assertTrue(metaDir.exists()); // should have been created.
+ } finally {
+ FileUtils.deleteQuietly(metaDir);
+ }
+ }
+
+ @Test
+ public void testNoOmDbDirConfigured() {
+ thrown.expect(IllegalArgumentException.class);
+ OMStorage.getOmDbDir(new OzoneConfiguration());
+ }
+
+ public File createTestDir() {
+ File dir = new File(GenericTestUtils.getRandomizedTestDir(),
+ TestOMStorage.class.getSimpleName());
+ dir.mkdirs();
+ return dir;
+ }
+}
diff --git a/hadoop-ozone/ozonefs-lib-current/pom.xml b/hadoop-ozone/ozonefs-lib-current/pom.xml
index 9b5e601031af..1645ccc82b24 100644
--- a/hadoop-ozone/ozonefs-lib-current/pom.xml
+++ b/hadoop-ozone/ozonefs-lib-current/pom.xml
@@ -74,21 +74,6 @@
shade
-
-
- org.apache.hadoop:hadoop-hdds-server-framework
-
- webapps/datanode/**
- webapps/static/**
-
-
-
- org.apache.hadoop:hadoop-hdds-container-service
-
- webapps/hddsDatanode/**
-
-
-
diff --git a/hadoop-ozone/s3gateway/pom.xml b/hadoop-ozone/s3gateway/pom.xml
index 32c95874540b..15b09a2abf1c 100644
--- a/hadoop-ozone/s3gateway/pom.xml
+++ b/hadoop-ozone/s3gateway/pom.xml
@@ -36,6 +36,10 @@
hadoop-ozone-common
compile
+