diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 319fc539ffa7..fadbf4b2c3bc 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -114,14 +114,9 @@ public final class OzoneConsts { */ public static final String CONTAINER_DB_SUFFIX = "container.db"; public static final String PIPELINE_DB_SUFFIX = "pipeline.db"; - public static final String SCM_CONTAINER_DB = "scm-" + CONTAINER_DB_SUFFIX; - public static final String SCM_PIPELINE_DB = "scm-" + PIPELINE_DB_SUFFIX; public static final String DN_CONTAINER_DB = "-dn-"+ CONTAINER_DB_SUFFIX; - public static final String DELETED_BLOCK_DB = "deletedBlock.db"; public static final String OM_DB_NAME = "om.db"; public static final String OM_DB_BACKUP_PREFIX = "om.db.backup."; - public static final String OZONE_MANAGER_TOKEN_DB_NAME = "om-token.db"; - public static final String SCM_DB_NAME = "scm.db"; public static final String STORAGE_DIR_CHUNKS = "chunks"; public static final String OZONE_DB_CHECKPOINT_REQUEST_FLUSH = diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/BatchOperationHandler.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/BatchOperationHandler.java new file mode 100644 index 000000000000..eea483ccb764 --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/BatchOperationHandler.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.hdds.utils.db; + +import java.io.IOException; + +/** + * Create and commit batch operation for one DB. + */ +public interface BatchOperationHandler { + + /** + * Initialize an atomic batch operation which can hold multiple PUT/DELETE + * operations and committed later in one step. + * + * @return BatchOperation holder which can be used to add or commit batch + * operations. + */ + BatchOperation initBatchOperation(); + + /** + * Commit the batch operations. + * + * @param operation which contains all the required batch operation. + * @throws IOException on Failure. + */ + void commitBatchOperation(BatchOperation operation) throws IOException; +} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBColumnFamilyDefinition.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBColumnFamilyDefinition.java new file mode 100644 index 000000000000..e1c416370d6a --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBColumnFamilyDefinition.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.hdds.utils.db; + +import java.io.IOException; + +/** + * Class represents one single column table with the required codecs and types. + * + * @param the type of the key. + * @param they type of the value. + */ +public class DBColumnFamilyDefinition { + + private final String tableName; + + private final Class keyType; + + private final Codec keyCodec; + + private final Class valueType; + + private final Codec valueCodec; + + public DBColumnFamilyDefinition( + String tableName, + Class keyType, + Codec keyCodec, + Class valueType, + Codec valueCodec) { + this.tableName = tableName; + this.keyType = keyType; + this.keyCodec = keyCodec; + this.valueType = valueType; + this.valueCodec = valueCodec; + } + + public Table getTable(DBStore db) throws IOException { + return db.getTable(tableName, keyType, valueType); + } + + public String getName() { + return tableName; + } + + public String getTableName() { + return tableName; + } + + public Class getKeyType() { + return keyType; + } + + public Codec getKeyCodec() { + return keyCodec; + } + + public Class getValueType() { + return valueType; + } + + public Codec getValueCodec() { + return valueCodec; + } +} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBDefinition.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBDefinition.java new file mode 100644 index 000000000000..305826108363 --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBDefinition.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.hdds.utils.db; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Simple interface to provide information to create a DBStore.. + */ +public interface DBDefinition { + + Logger LOG = LoggerFactory.getLogger(DBDefinition.class); + + /** + * Logical name of the DB. + */ + String getName(); + + /** + * Configuration key defines the location of the DB. + */ + String getLocationConfigKey(); + + /** + * Create a new DB store instance based on the configuration. + */ + DBColumnFamilyDefinition[] getColumnFamilies(); + +} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java index ed64b745b5d4..8567d038fff6 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java @@ -35,7 +35,7 @@ * */ @InterfaceStability.Evolving -public interface DBStore extends AutoCloseable { +public interface DBStore extends AutoCloseable, BatchOperationHandler { /** * Gets an existing TableStore. @@ -141,22 +141,6 @@ void move(KEY sourceKey, KEY destKey, VALUE value, */ long getEstimatedKeyCount() throws IOException; - /** - * Initialize an atomic batch operation which can hold multiple PUT/DELETE - * operations and committed later in one step. - * - * @return BatchOperation holder which can be used to add or commit batch - * operations. - */ - BatchOperation initBatchOperation(); - - /** - * Commit the batch operations. - * - * @param operation which contains all the required batch operation. - * @throws IOException on Failure. - */ - void commitBatchOperation(BatchOperation operation) throws IOException; /** * Get current snapshot of DB store as an artifact stored on diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java index 88d509acf8f2..2e185301bf42 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java @@ -29,12 +29,15 @@ import java.util.List; import java.util.Set; -import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.StringUtils; +import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import com.google.common.base.Preconditions; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE; +import static org.apache.hadoop.hdds.server.ServerUtils.getDirectoryFromConfig; +import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF; @@ -78,10 +81,11 @@ public final class DBStoreBuilder { private String rocksDbStat; private RocksDBConfiguration rocksDBConfiguration; - private DBStoreBuilder(OzoneConfiguration configuration) { + private DBStoreBuilder(ConfigurationSource configuration) { this(configuration, configuration.getObject(RocksDBConfiguration.class)); } - private DBStoreBuilder(OzoneConfiguration configuration, + + private DBStoreBuilder(ConfigurationSource configuration, RocksDBConfiguration rocksDBConfiguration) { tables = new HashSet<>(); tableNames = new LinkedList<>(); @@ -93,8 +97,7 @@ private DBStoreBuilder(OzoneConfiguration configuration, this.rocksDBConfiguration = rocksDBConfiguration; } - - public static DBStoreBuilder newBuilder(OzoneConfiguration configuration) { + public static DBStoreBuilder newBuilder(ConfigurationSource configuration) { return new DBStoreBuilder(configuration); } @@ -263,4 +266,45 @@ private File getDBFile() throws IOException { return Paths.get(dbPath.toString(), dbname).toFile(); } + private static DBStoreBuilder createDBStoreBuilder( + ConfigurationSource configuration, DBDefinition definition) { + + File metadataDir = getDirectoryFromConfig(configuration, + definition.getLocationConfigKey(), definition.getName()); + + if (metadataDir == null) { + + LOG.warn("{} is not configured. We recommend adding this setting. " + + "Falling back to {} instead.", + definition.getLocationConfigKey(), + HddsConfigKeys.OZONE_METADATA_DIRS); + metadataDir = getOzoneMetaDirPath(configuration); + } + + return DBStoreBuilder.newBuilder(configuration) + .setName(definition.getName()) + .setPath(Paths.get(metadataDir.getPath())); + } + + /** + * Create DBStoreBuilder from a generic DBDefinition. + */ + public static DBStore createDBStore(ConfigurationSource configuration, + DBDefinition definition) + throws IOException { + DBStoreBuilder builder = createDBStoreBuilder(configuration, definition); + for (DBColumnFamilyDefinition columnTableDefinition : definition + .getColumnFamilies()) { + builder.registerTable(columnTableDefinition); + } + return builder.build(); + } + + private void registerTable( + DBColumnFamilyDefinition definition) { + addTable(definition.getName()) + .addCodec(definition.getKeyType(), definition.getKeyCodec()) + .addCodec(definition.getValueType(), definition.getValueCodec()); + } + } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java index 5ca75d2a116a..0980369c3379 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java @@ -16,38 +16,35 @@ */ package org.apache.hadoop.hdds.scm.block; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction; import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.node.NodeManager; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction; import org.apache.hadoop.hdds.server.events.EventPublisher; -import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; -import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand; -import org.apache.hadoop.util.Time; import org.apache.hadoop.hdds.utils.BackgroundService; import org.apache.hadoop.hdds.utils.BackgroundTask; import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult; +import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; +import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand; +import org.apache.hadoop.util.Time; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - -import static org.apache.hadoop.ozone.OzoneConfigKeys - .OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL; -import static org.apache.hadoop.ozone.OzoneConfigKeys - .OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT; - /** * A background service running in SCM to delete blocks. This service scans * block deletion log in certain interval and caches block deletion commands diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java index 973026de487f..43c1cedf6189 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java @@ -16,15 +16,15 @@ */ package org.apache.hadoop.hdds.scm.container; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.scm.pipeline.Pipeline; - import java.io.Closeable; import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; + // TODO: Write extensive java doc. // This is the main interface of ContainerManager. /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java index 38c3d1189ee0..9f47608f30ee 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java @@ -16,7 +16,6 @@ */ package org.apache.hadoop.hdds.scm.container; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -32,7 +31,6 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ContainerInfoProto; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; @@ -42,18 +40,13 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException; -import org.apache.hadoop.hdds.server.ServerUtils; -import org.apache.hadoop.hdds.utils.BatchOperation; -import org.apache.hadoop.hdds.utils.MetadataStore; -import org.apache.hadoop.hdds.utils.MetadataStoreBuilder; -import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.hdds.utils.db.BatchOperationHandler; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.Table.KeyValue; +import org.apache.hadoop.hdds.utils.db.TableIterator; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.google.common.primitives.Longs; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_DEFAULT; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_MB; -import static org.apache.hadoop.ozone.OzoneConsts.SCM_CONTAINER_DB; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,38 +60,40 @@ public class SCMContainerManager implements ContainerManager { SCMContainerManager.class); private final Lock lock; - private final MetadataStore containerStore; + private final PipelineManager pipelineManager; + private final ContainerStateManager containerStateManager; + private final int numContainerPerOwnerInPipeline; private final SCMContainerManagerMetrics scmContainerManagerMetrics; + private Table containerStore; + + private BatchOperationHandler batchHandler; + /** * Constructs a mapping class that creates mapping between container names * and pipelines. - * + *

* passed to LevelDB and this memory is allocated in Native code space. * CacheSize is specified * in MB. - * @param conf - {@link ConfigurationSource} + * + * @param conf - {@link ConfigurationSource} * @param pipelineManager - {@link PipelineManager} * @throws IOException on Failure. */ - public SCMContainerManager(final ConfigurationSource conf, + public SCMContainerManager( + final ConfigurationSource conf, + Table containerStore, + BatchOperationHandler batchHandler, PipelineManager pipelineManager) throws IOException { - final File containerDBPath = getContainerDBPath(conf); - final int cacheSize = conf.getInt(OZONE_SCM_DB_CACHE_SIZE_MB, - OZONE_SCM_DB_CACHE_SIZE_DEFAULT); - - this.containerStore = MetadataStoreBuilder.newBuilder() - .setConf(conf) - .setDbFile(containerDBPath) - .setCacheSize(cacheSize * OzoneConsts.MB) - .build(); - + this.batchHandler = batchHandler; + this.containerStore = containerStore; this.lock = new ReentrantLock(); this.pipelineManager = pipelineManager; this.containerStateManager = new ContainerStateManager(conf); @@ -112,11 +107,12 @@ public SCMContainerManager(final ConfigurationSource conf, } private void loadExistingContainers() throws IOException { - List> range = containerStore - .getSequentialRangeKVs(null, Integer.MAX_VALUE, null); - for (Map.Entry entry : range) { - ContainerInfo container = ContainerInfo.fromProtobuf( - ContainerInfoProto.PARSER.parseFrom(entry.getValue())); + + TableIterator> + iterator = containerStore.iterator(); + + while (iterator.hasNext()) { + ContainerInfo container = iterator.next().getValue(); Preconditions.checkNotNull(container); containerStateManager.loadContainer(container); try { @@ -304,10 +300,8 @@ public void deleteContainer(ContainerID containerID) throws IOException { lock.lock(); try { containerStateManager.removeContainer(containerID); - final byte[] dbKey = Longs.toByteArray(containerID.getId()); - final byte[] containerBytes = containerStore.get(dbKey); - if (containerBytes != null) { - containerStore.delete(dbKey); + if (containerStore.get(containerID) != null) { + containerStore.delete(containerID); } else { // Where did the container go? o_O LOG.warn("Unable to remove the container {} from container store," + @@ -358,8 +352,7 @@ private HddsProtos.LifeCycleState updateContainerState( containerID); } } - final byte[] dbKey = Longs.toByteArray(containerID.getId()); - containerStore.put(dbKey, container.getProtobuf().toByteArray()); + containerStore.put(containerID, container); return newState; } catch (ContainerNotFoundException cnfe) { throw new SCMException( @@ -372,38 +365,40 @@ private HddsProtos.LifeCycleState updateContainerState( } } - /** - * Update deleteTransactionId according to deleteTransactionMap. - * - * @param deleteTransactionMap Maps the containerId to latest delete - * transaction id for the container. - * @throws IOException - */ + /** + * Update deleteTransactionId according to deleteTransactionMap. + * + * @param deleteTransactionMap Maps the containerId to latest delete + * transaction id for the container. + * @throws IOException + */ public void updateDeleteTransactionId(Map deleteTransactionMap) throws IOException { + if (deleteTransactionMap == null) { return; } - + org.apache.hadoop.hdds.utils.db.BatchOperation batchOperation = + batchHandler.initBatchOperation(); lock.lock(); try { - BatchOperation batch = new BatchOperation(); for (Map.Entry entry : deleteTransactionMap.entrySet()) { long containerID = entry.getKey(); - byte[] dbKey = Longs.toByteArray(containerID); - byte[] containerBytes = containerStore.get(dbKey); - if (containerBytes == null) { + + ContainerID containerIdObject = new ContainerID(containerID); + ContainerInfo containerInfo = + containerStore.get(containerIdObject); + if (containerInfo == null) { throw new SCMException( "Failed to increment number of deleted blocks for container " + containerID + ", reason : " + "container doesn't exist.", SCMException.ResultCodes.FAILED_TO_FIND_CONTAINER); } - ContainerInfo containerInfo = ContainerInfo.fromProtobuf( - HddsProtos.ContainerInfoProto.parseFrom(containerBytes)); containerInfo.updateDeleteTransactionId(entry.getValue()); - batch.put(dbKey, containerInfo.getProtobuf().toByteArray()); + containerStore + .putWithBatch(batchOperation, containerIdObject, containerInfo); } - containerStore.writeBatch(batch); + batchHandler.commitBatchOperation(batchOperation); containerStateManager .updateDeleteTransactionId(deleteTransactionMap); } finally { @@ -477,10 +472,8 @@ public ContainerInfo getMatchingContainer(final long sizeRequired, protected void addContainerToDB(ContainerInfo containerInfo) throws IOException { try { - final byte[] containerIDBytes = Longs.toByteArray( - containerInfo.getContainerID()); - containerStore.put(containerIDBytes, - containerInfo.getProtobuf().toByteArray()); + containerStore + .put(new ContainerID(containerInfo.getContainerID()), containerInfo); // Incrementing here, as allocateBlock to create a container calls // getMatchingContainer() and finally calls this API to add newly // created container to DB. @@ -586,9 +579,6 @@ public void close() throws IOException { if (containerStateManager != null) { containerStateManager.close(); } - if (containerStore != null) { - containerStore.close(); - } if (scmContainerManagerMetrics != null) { this.scmContainerManagerMetrics.unRegister(); @@ -612,11 +602,6 @@ public void notifyContainerReportProcessing(boolean isFullReport, } } - protected File getContainerDBPath(ConfigurationSource conf) { - File metaDir = ServerUtils.getScmDbDir(conf); - return new File(metaDir, SCM_CONTAINER_DB); - } - protected PipelineManager getPipelineManager() { return pipelineManager; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/ContainerIDCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/ContainerIDCodec.java new file mode 100644 index 000000000000..87c9e9172698 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/ContainerIDCodec.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.hdds.scm.metadata; + +import java.io.IOException; + +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.utils.db.Codec; +import org.apache.hadoop.hdds.utils.db.LongCodec; + +/** + * Codec to serialize / deserialize ContainerID. + */ +public class ContainerIDCodec implements Codec { + + private Codec longCodec = new LongCodec(); + + @Override + public byte[] toPersistedFormat(ContainerID container) throws IOException { + return longCodec.toPersistedFormat(container.getId()); + } + + @Override + public ContainerID fromPersistedFormat(byte[] rawData) throws IOException { + return new ContainerID(longCodec.fromPersistedFormat(rawData)); + } + + @Override + public ContainerID copyObject(ContainerID object) { + return new ContainerID(object.getId()); + } +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/ContainerInfoCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/ContainerInfoCodec.java new file mode 100644 index 000000000000..6b262150c6f6 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/ContainerInfoCodec.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.hdds.scm.metadata; + +import java.io.IOException; + +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ContainerInfoProto; +import org.apache.hadoop.hdds.scm.container.ContainerInfo; +import org.apache.hadoop.hdds.utils.db.Codec; + +/** + * Codec to serialize / deserialize ContainerInfo. + */ +public class ContainerInfoCodec implements Codec { + + @Override + public byte[] toPersistedFormat(ContainerInfo container) throws IOException { + return container.getProtobuf().toByteArray(); + } + + @Override + public ContainerInfo fromPersistedFormat(byte[] rawData) throws IOException { + return ContainerInfo.fromProtobuf( + ContainerInfoProto.PARSER.parseFrom(rawData)); + } + + @Override + public ContainerInfo copyObject(ContainerInfo object) { + throw new UnsupportedOperationException(); + } +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/PipelineCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/PipelineCodec.java new file mode 100644 index 000000000000..25a1e44651ea --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/PipelineCodec.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.hdds.scm.metadata; + +import java.io.IOException; +import java.time.Instant; + +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.utils.db.Codec; + +import com.google.common.base.Preconditions; + +/** + * Codec to serialize / deserialize Pipeline. + */ +public class PipelineCodec implements Codec { + + @Override + public byte[] toPersistedFormat(Pipeline object) throws IOException { + return object.getProtobufMessage().toByteArray(); + } + + @Override + public Pipeline fromPersistedFormat(byte[] rawData) throws IOException { + HddsProtos.Pipeline.Builder pipelineBuilder = HddsProtos.Pipeline + .newBuilder(HddsProtos.Pipeline.PARSER.parseFrom(rawData)); + Pipeline pipeline = Pipeline.getFromProtobuf(pipelineBuilder.setState( + HddsProtos.PipelineState.PIPELINE_ALLOCATED).build()); + // When SCM is restarted, set Creation time with current time. + pipeline.setCreationTimestamp(Instant.now()); + Preconditions.checkNotNull(pipeline); + return pipeline; + } + + @Override + public Pipeline copyObject(Pipeline object) { + throw new UnsupportedOperationException(); + } +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/PipelineIDCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/PipelineIDCodec.java new file mode 100644 index 000000000000..d661e3467b2c --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/PipelineIDCodec.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.hdds.scm.metadata; + +import java.io.IOException; + +import org.apache.hadoop.hdds.scm.pipeline.PipelineID; +import org.apache.hadoop.hdds.utils.db.Codec; + +/** + * Codec to serialize / deserialize PipelineID. + */ +public class PipelineIDCodec implements Codec { + + @Override + public byte[] toPersistedFormat(PipelineID object) throws IOException { + return object.getProtobuf().toByteArray(); + } + + @Override + public PipelineID fromPersistedFormat(byte[] rawData) throws IOException { + return null; + } + + @Override + public PipelineID copyObject(PipelineID object) { + throw new UnsupportedOperationException(); + } +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java new file mode 100644 index 000000000000..fcddcdd274d8 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements.  See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership.  The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License.  You may obtain a copy of the License at + * + *      http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.metadata; + +import java.math.BigInteger; +import java.security.cert.X509Certificate; + +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerInfo; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.scm.pipeline.PipelineID; +import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition; +import org.apache.hadoop.hdds.utils.db.DBDefinition; +import org.apache.hadoop.hdds.utils.db.LongCodec; + +/** + * Class defines the structure and types of the scm.db. + */ +public class SCMDBDefinition implements DBDefinition { + + public static final DBColumnFamilyDefinition + DELETED_BLOCKS = + new DBColumnFamilyDefinition<>( + "deletedBlocks", + Long.class, + new LongCodec(), + DeletedBlocksTransaction.class, + new DeletedBlocksTransactionCodec()); + + public static final DBColumnFamilyDefinition + VALID_CERTS = + new DBColumnFamilyDefinition<>( + "validCerts", + BigInteger.class, + new BigIntegerCodec(), + X509Certificate.class, + new X509CertificateCodec()); + + public static final DBColumnFamilyDefinition + REVOKED_CERTS = + new DBColumnFamilyDefinition<>( + "revokedCerts", + BigInteger.class, + new BigIntegerCodec(), + X509Certificate.class, + new X509CertificateCodec()); + + public static final DBColumnFamilyDefinition + PIPELINES = + new DBColumnFamilyDefinition<>( + "pipelines", + PipelineID.class, + new PipelineIDCodec(), + Pipeline.class, + new PipelineCodec()); + + public static final DBColumnFamilyDefinition + CONTAINERS = + new DBColumnFamilyDefinition( + "containers", + ContainerID.class, + new ContainerIDCodec(), + ContainerInfo.class, + new ContainerInfoCodec()); + + @Override + public String getName() { + return "scm.db"; + } + + @Override + public String getLocationConfigKey() { + return ScmConfigKeys.OZONE_SCM_DB_DIRS; + } + + @Override + public DBColumnFamilyDefinition[] getColumnFamilies() { + return new DBColumnFamilyDefinition[] {DELETED_BLOCKS, VALID_CERTS, + REVOKED_CERTS, PIPELINES, CONTAINERS}; + } +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStore.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStore.java index 11503169ce24..0452c05ae2a6 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStore.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStore.java @@ -17,18 +17,24 @@ */ package org.apache.hadoop.hdds.scm.metadata; +import java.io.IOException; import java.math.BigInteger; import java.security.cert.X509Certificate; + import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import java.io.IOException; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerInfo; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore; +import org.apache.hadoop.hdds.utils.db.BatchOperationHandler; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction; import org.apache.hadoop.hdds.utils.db.TableIterator; +import com.google.common.annotations.VisibleForTesting; + /** * Generic interface for data stores for SCM. * This is similar to the OMMetadataStore class, @@ -99,5 +105,18 @@ public interface SCMMetadataStore { */ TableIterator getAllCerts(CertificateStore.CertType certType); + /** + * A Table that maintains all the pipeline information. + */ + Table getPipelineTable(); + + /** + * Helper to create and write batch transactions. + */ + BatchOperationHandler getBatchHandler(); + /** + * Table that maintains all the container information. + */ + Table getContainerTable(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreRDBImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreRDBImpl.java index 72818a3c6e01..3823fd838071 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreRDBImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreRDBImpl.java @@ -17,53 +17,48 @@ */ package org.apache.hadoop.hdds.scm.metadata; -import java.io.File; +import java.io.IOException; import java.math.BigInteger; -import java.nio.file.Paths; import java.security.cert.X509Certificate; import java.util.concurrent.atomic.AtomicLong; + import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import java.io.IOException; -import org.apache.hadoop.hdds.security.x509.certificate.authority - .CertificateStore; -import org.apache.hadoop.hdds.server.ServerUtils; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerInfo; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.scm.pipeline.PipelineID; +import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore; +import org.apache.hadoop.hdds.utils.db.BatchOperationHandler; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction; +import org.apache.hadoop.hdds.utils.db.Table.KeyValue; import org.apache.hadoop.hdds.utils.db.TableIterator; + +import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.CONTAINERS; +import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.DELETED_BLOCKS; +import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.PIPELINES; +import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.REVOKED_CERTS; +import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.VALID_CERTS; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.hadoop.ozone.OzoneConsts.SCM_DB_NAME; - /** * A RocksDB based implementation of SCM Metadata Store. - *

- *

- * +---------------+------------------+-------------------------+ - * | Column Family | Key | Value | - * +---------------+------------------+-------------------------+ - * | DeletedBlocks | TXID(Long) | DeletedBlockTransaction | - * +---------------+------------------+-------------------------+ - * | ValidCerts | Serial (BigInt) | X509Certificate | - * +---------------+------------------+-------------------------+ - * |RevokedCerts | Serial (BigInt) | X509Certificate | - * +---------------+------------------+-------------------------+ + * */ public class SCMMetadataStoreRDBImpl implements SCMMetadataStore { - private static final String DELETED_BLOCKS_TABLE = "deletedBlocks"; - private Table deletedBlocksTable; + private Table deletedBlocksTable; - private static final String VALID_CERTS_TABLE = "validCerts"; - private Table validCertsTable; + private Table validCertsTable; - private static final String REVOKED_CERTS_TABLE = "revokedCerts"; - private Table revokedCertsTable; + private Table revokedCertsTable; + private Table containerTable; + private Table pipelineTable; private static final Logger LOG = LoggerFactory.getLogger(SCMMetadataStoreRDBImpl.class); @@ -88,31 +83,26 @@ public SCMMetadataStoreRDBImpl(OzoneConfiguration config) public void start(OzoneConfiguration config) throws IOException { if (this.store == null) { - File metaDir = ServerUtils.getScmDbDir(configuration); - - this.store = DBStoreBuilder.newBuilder(configuration) - .setName(SCM_DB_NAME) - .setPath(Paths.get(metaDir.getPath())) - .addTable(DELETED_BLOCKS_TABLE) - .addTable(VALID_CERTS_TABLE) - .addTable(REVOKED_CERTS_TABLE) - .addCodec(DeletedBlocksTransaction.class, - new DeletedBlocksTransactionCodec()) - .addCodec(BigInteger.class, new BigIntegerCodec()) - .addCodec(X509Certificate.class, new X509CertificateCodec()) - .build(); - - deletedBlocksTable = this.store.getTable(DELETED_BLOCKS_TABLE, - Long.class, DeletedBlocksTransaction.class); - checkTableStatus(deletedBlocksTable, DELETED_BLOCKS_TABLE); - - validCertsTable = this.store.getTable(VALID_CERTS_TABLE, - BigInteger.class, X509Certificate.class); - checkTableStatus(validCertsTable, VALID_CERTS_TABLE); - - revokedCertsTable = this.store.getTable(REVOKED_CERTS_TABLE, - BigInteger.class, X509Certificate.class); - checkTableStatus(revokedCertsTable, REVOKED_CERTS_TABLE); + + this.store = DBStoreBuilder.createDBStore(config, new SCMDBDefinition()); + + deletedBlocksTable = + DELETED_BLOCKS.getTable(this.store); + + checkTableStatus(deletedBlocksTable, + DELETED_BLOCKS.getName()); + + validCertsTable = VALID_CERTS.getTable(store); + + checkTableStatus(validCertsTable, VALID_CERTS.getName()); + + revokedCertsTable = REVOKED_CERTS.getTable(store); + + checkTableStatus(revokedCertsTable, REVOKED_CERTS.getName()); + + pipelineTable = PIPELINES.getTable(store); + + containerTable = CONTAINERS.getTable(store); } } @@ -162,6 +152,21 @@ public TableIterator getAllCerts(CertificateStore.CertType certType) { return null; } + @Override + public Table getPipelineTable() { + return pipelineTable; + } + + @Override + public BatchOperationHandler getBatchHandler() { + return this.store; + } + + @Override + public Table getContainerTable() { + return containerTable; + } + @Override public Long getCurrentTXID() { return this.txID.get(); @@ -174,8 +179,8 @@ public Long getCurrentTXID() { * @throws IOException */ private Long getLargestRecordedTXID() throws IOException { - try (TableIterator txIter = - deletedBlocksTable.iterator()) { + try (TableIterator> + txIter = deletedBlocksTable.iterator()) { txIter.seekToLast(); Long txid = txIter.key(); if (txid != null) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java index 200b35820eba..38edbacdb34a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hdds.scm.pipeline; import javax.management.ObjectName; -import java.io.File; import java.io.IOException; import java.time.Duration; import java.time.Instant; @@ -38,7 +37,6 @@ import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; import org.apache.hadoop.hdds.scm.ScmConfigKeys; @@ -46,19 +44,15 @@ import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager; -import org.apache.hadoop.hdds.server.ServerUtils; import org.apache.hadoop.hdds.server.events.EventPublisher; -import org.apache.hadoop.hdds.utils.MetadataKeyFilters; -import org.apache.hadoop.hdds.utils.MetadataStore; -import org.apache.hadoop.hdds.utils.MetadataStoreBuilder; import org.apache.hadoop.hdds.utils.Scheduler; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.Table.KeyValue; +import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.metrics2.util.MBeans; -import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.util.Time; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import static org.apache.hadoop.ozone.OzoneConsts.SCM_PIPELINE_DB; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,7 +71,6 @@ public class SCMPipelineManager implements PipelineManager { private PipelineStateManager stateManager; private final BackgroundPipelineCreator backgroundPipelineCreator; private Scheduler scheduler; - private MetadataStore pipelineStore; private final EventPublisher eventPublisher; private final NodeManager nodeManager; @@ -87,28 +80,35 @@ public class SCMPipelineManager implements PipelineManager { // Pipeline Manager MXBean private ObjectName pmInfoBean; + private Table pipelineStore; + private final AtomicBoolean isInSafeMode; // Used to track if the safemode pre-checks have completed. This is designed // to prevent pipelines being created until sufficient nodes have registered. private final AtomicBoolean pipelineCreationAllowed; - public SCMPipelineManager(ConfigurationSource conf, NodeManager nodeManager, + public SCMPipelineManager(ConfigurationSource conf, + NodeManager nodeManager, + Table pipelineStore, EventPublisher eventPublisher) throws IOException { - this(conf, nodeManager, eventPublisher, null, null); + this(conf, nodeManager, pipelineStore, eventPublisher, null, null); this.stateManager = new PipelineStateManager(); this.pipelineFactory = new PipelineFactory(nodeManager, stateManager, conf, eventPublisher); + this.pipelineStore = pipelineStore; initializePipelineState(); } protected SCMPipelineManager(ConfigurationSource conf, NodeManager nodeManager, - EventPublisher eventPublisher, - PipelineStateManager pipelineStateManager, - PipelineFactory pipelineFactory) + Table pipelineStore, + EventPublisher eventPublisher, + PipelineStateManager pipelineStateManager, + PipelineFactory pipelineFactory) throws IOException { this.lock = new ReentrantReadWriteLock(); + this.pipelineStore = pipelineStore; this.conf = conf; this.pipelineFactory = pipelineFactory; this.stateManager = pipelineStateManager; @@ -116,16 +116,6 @@ protected SCMPipelineManager(ConfigurationSource conf, scheduler = new Scheduler("RatisPipelineUtilsThread", false, 1); this.backgroundPipelineCreator = new BackgroundPipelineCreator(this, scheduler, conf); - int cacheSize = conf.getInt(ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_MB, - ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_DEFAULT); - final File pipelineDBPath = getPipelineDBPath(conf); - this.pipelineStore = - MetadataStoreBuilder.newBuilder() - .setCreateIfMissing(true) - .setConf(conf) - .setDbFile(pipelineDBPath) - .setCacheSize(cacheSize * OzoneConsts.MB) - .build(); this.eventPublisher = eventPublisher; this.nodeManager = nodeManager; this.metrics = SCMPipelineMetrics.create(); @@ -168,18 +158,10 @@ protected void initializePipelineState() throws IOException { LOG.info("No pipeline exists in current db"); return; } - List> pipelines = - pipelineStore.getSequentialRangeKVs(null, Integer.MAX_VALUE, - (MetadataKeyFilters.MetadataKeyFilter[])null); - - for (Map.Entry entry : pipelines) { - HddsProtos.Pipeline.Builder pipelineBuilder = HddsProtos.Pipeline - .newBuilder(HddsProtos.Pipeline.PARSER.parseFrom(entry.getValue())); - Pipeline pipeline = Pipeline.getFromProtobuf(pipelineBuilder.setState( - HddsProtos.PipelineState.PIPELINE_ALLOCATED).build()); - // When SCM is restarted, set Creation time with current time. - pipeline.setCreationTimestamp(Instant.now()); - Preconditions.checkNotNull(pipeline); + TableIterator> + iterator = pipelineStore.iterator(); + while (iterator.hasNext()) { + Pipeline pipeline = iterator.next().getValue(); stateManager.addPipeline(pipeline); nodeManager.addPipeline(pipeline); } @@ -231,8 +213,7 @@ public synchronized Pipeline createPipeline(ReplicationType type, lock.writeLock().lock(); try { Pipeline pipeline = pipelineFactory.create(type, factor); - pipelineStore.put(pipeline.getId().getProtobuf().toByteArray(), - pipeline.getProtobufMessage().toByteArray()); + pipelineStore.put(pipeline.getId(), pipeline); stateManager.addPipeline(pipeline); nodeManager.addPipeline(pipeline); recordMetricsForPipeline(pipeline); @@ -588,11 +569,10 @@ protected void destroyPipeline(Pipeline pipeline) throws IOException { * @throws IOException */ protected void removePipeline(PipelineID pipelineId) throws IOException { - byte[] key = pipelineId.getProtobuf().toByteArray(); lock.writeLock().lock(); try { if (pipelineStore != null) { - pipelineStore.delete(key); + pipelineStore.delete(pipelineId); Pipeline pipeline = stateManager.removePipeline(pipelineId); nodeManager.removePipeline(pipeline); metrics.incNumPipelineDestroyed(); @@ -617,16 +597,6 @@ public void close() throws IOException { scheduler = null; } - lock.writeLock().lock(); - try { - if (pipelineStore != null) { - pipelineStore.close(); - pipelineStore = null; - } - } finally { - lock.writeLock().unlock(); - } - if(pmInfoBean != null) { MBeans.unregister(this.pmInfoBean); pmInfoBean = null; @@ -638,11 +608,6 @@ public void close() throws IOException { pipelineFactory.shutdown(); } - protected File getPipelineDBPath(ConfigurationSource configuration) { - File metaDir = ServerUtils.getScmDbDir(configuration); - return new File(metaDir, SCM_PIPELINE_DB); - } - protected ReadWriteLock getLock() { return lock; } @@ -652,10 +617,6 @@ public PipelineFactory getPipelineFactory() { return pipelineFactory; } - protected MetadataStore getPipelineStore() { - return pipelineStore; - } - protected NodeManager getNodeManager() { return nodeManager; } @@ -665,6 +626,10 @@ public boolean getSafeModeStatus() { return this.isInSafeMode.get(); } + public Table getPipelineStore() { + return pipelineStore; + } + @Override public synchronized void handleSafeModeTransition( SCMSafeModeManager.SafeModeStatus status) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 1f2305f07915..8498a25a4b0b 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -406,13 +406,19 @@ private void initializeSystemManagers(OzoneConfiguration conf, pipelineManager = configurator.getPipelineManager(); } else { pipelineManager = - new SCMPipelineManager(conf, scmNodeManager, eventQueue); + new SCMPipelineManager(conf, scmNodeManager, + scmMetadataStore.getPipelineTable(), + eventQueue); } if (configurator.getContainerManager() != null) { containerManager = configurator.getContainerManager(); } else { - containerManager = new SCMContainerManager(conf, pipelineManager); + containerManager = + new SCMContainerManager(conf, + scmMetadataStore.getContainerTable(), + scmMetadataStore.getBatchHandler(), + pipelineManager); } if (configurator.getScmBlockManager() != null) { diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java index 6a19ff9e12ab..c9b5fde158b2 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java @@ -29,8 +29,7 @@ import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.SCMCommandProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler; @@ -40,9 +39,11 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock; import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreRDBImpl; +import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; -import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager; import org.apache.hadoop.hdds.scm.server.SCMConfigurator; @@ -55,6 +56,9 @@ import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; import org.apache.hadoop.ozone.protocol.commands.CreatePipelineCommand; import org.apache.hadoop.test.GenericTestUtils; + +import static org.apache.hadoop.ozone.OzoneConsts.GB; +import static org.apache.hadoop.ozone.OzoneConsts.MB; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -63,10 +67,6 @@ import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; -import static org.apache.hadoop.ozone.OzoneConsts.GB; -import static org.apache.hadoop.ozone.OzoneConsts.MB; - - /** * Tests for SCM Block Manager. */ @@ -88,6 +88,7 @@ public class TestBlockManager { @Rule public TemporaryFolder folder= new TemporaryFolder(); + private SCMMetadataStore scmMetadataStore; @Before public void setUp() throws Exception { @@ -105,16 +106,25 @@ public void setUp() throws Exception { // Override the default Node Manager in SCM with this Mock Node Manager. nodeManager = new MockNodeManager(true, 10); eventQueue = new EventQueue(); + + scmMetadataStore = new SCMMetadataStoreRDBImpl(conf); + scmMetadataStore.start(conf); pipelineManager = - new SCMPipelineManager(conf, nodeManager, eventQueue); + new SCMPipelineManager(conf, nodeManager, + scmMetadataStore.getPipelineTable(), + eventQueue); pipelineManager.allowPipelineCreation(); + PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, pipelineManager.getStateManager(), conf, eventQueue); pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider); SCMContainerManager containerManager = - new SCMContainerManager(conf, pipelineManager); + new SCMContainerManager(conf, + scmMetadataStore.getContainerTable(), + scmMetadataStore.getStore(), + pipelineManager); SCMSafeModeManager safeModeManager = new SCMSafeModeManager(conf, containerManager.getContainers(), pipelineManager, eventQueue) { @Override @@ -127,6 +137,7 @@ public void emitSafeModeStatus() { configurator.setPipelineManager(pipelineManager); configurator.setContainerManager(containerManager); configurator.setScmSafeModeManager(safeModeManager); + configurator.setMetadataStore(scmMetadataStore); scm = TestUtils.getScm(conf, configurator); // Initialize these fields so that the tests can pass. @@ -145,10 +156,11 @@ public void emitSafeModeStatus() { } @After - public void cleanup() { + public void cleanup() throws Exception { scm.stop(); scm.join(); eventQueue.close(); + scmMetadataStore.stop(); } @Test diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java index f5675006cc9e..09b41a5a47fb 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java @@ -28,10 +28,13 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.TestUtils; +import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.SCMTestUtils; import org.apache.hadoop.test.GenericTestUtils; @@ -58,6 +61,7 @@ public class TestCloseContainerEventHandler { private static long size; private static File testDir; private static EventQueue eventQueue; + private static DBStore dbStore; @BeforeClass public static void setUp() throws Exception { @@ -71,15 +75,19 @@ public static void setUp() throws Exception { configuration.setInt(ScmConfigKeys.OZONE_SCM_RATIS_PIPELINE_LIMIT, 16); nodeManager = new MockNodeManager(true, 10); eventQueue = new EventQueue(); + dbStore = + DBStoreBuilder.createDBStore(configuration, new SCMDBDefinition()); pipelineManager = - new SCMPipelineManager(configuration, nodeManager, eventQueue); + new SCMPipelineManager(configuration, nodeManager, + SCMDBDefinition.PIPELINES.getTable(dbStore), eventQueue); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, pipelineManager.getStateManager(), configuration, eventQueue); pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider); - containerManager = new SCMContainerManager(configuration, pipelineManager); + containerManager = new SCMContainerManager(configuration, + SCMDBDefinition.CONTAINERS.getTable(dbStore), dbStore, pipelineManager); pipelineManager.triggerPipelineCreation(); eventQueue.addHandler(CLOSE_CONTAINER, new CloseContainerEventHandler(pipelineManager, containerManager)); @@ -97,6 +105,9 @@ public static void tearDown() throws Exception { if (pipelineManager != null) { pipelineManager.close(); } + if (dbStore != null) { + dbStore.close(); + } FileUtil.fullyDelete(testDir); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java index 75d27124ee8d..1821e92bb691 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java @@ -44,9 +44,12 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.XceiverClientManager; +import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.SCMTestUtils; import org.apache.hadoop.test.GenericTestUtils; @@ -59,6 +62,7 @@ import org.junit.Test; import org.junit.rules.ExpectedException; + /** * Tests for Container ContainerManager. */ @@ -76,6 +80,7 @@ public class TestSCMContainerManager { @Rule public ExpectedException thrown = ExpectedException.none(); + @BeforeClass public static void setUp() throws Exception { OzoneConfiguration conf = SCMTestUtils.getConf(); @@ -93,10 +98,15 @@ public static void setUp() throws Exception { throw new IOException("Unable to create test directory path"); } nodeManager = new MockNodeManager(true, 10); + DBStore dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition()); pipelineManager = - new SCMPipelineManager(conf, nodeManager, new EventQueue()); + new SCMPipelineManager(conf, nodeManager, + SCMDBDefinition.PIPELINES.getTable(dbStore), new EventQueue()); pipelineManager.allowPipelineCreation(); - containerManager = new SCMContainerManager(conf, pipelineManager); + containerManager = new SCMContainerManager(conf, + SCMDBDefinition.CONTAINERS.getTable(dbStore), + dbStore, + pipelineManager); xceiverClientManager = new XceiverClientManager(conf); replicationFactor = SCMTestUtils.getReplicationFactor(conf); replicationType = SCMTestUtils.getReplicationType(conf); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java index de027ed1fb40..6ce66a298108 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java @@ -36,19 +36,24 @@ import org.apache.hadoop.hdds.scm.container.SCMContainerManager; import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity; import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.SCMTestUtils; import org.apache.hadoop.test.PathUtils; import org.apache.commons.io.IOUtils; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_DEFAULT; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_MB; +import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.CONTAINERS; +import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.PIPELINES; +import org.junit.After; import static org.junit.Assert.assertEquals; +import org.junit.Before; import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; @@ -59,9 +64,21 @@ * Test for different container placement policy. */ public class TestContainerPlacement { + @Rule public ExpectedException thrown = ExpectedException.none(); + private DBStore dbStore; + + @Before + public void createDbStore() throws IOException { + dbStore = + DBStoreBuilder.createDBStore(getConf(), new SCMDBDefinition()); + } + @After + public void destroyDBStore() throws Exception { + dbStore.close(); + } /** * Returns a new copy of Configuration. * @@ -100,11 +117,13 @@ SCMNodeManager createNodeManager(OzoneConfiguration config) SCMContainerManager createContainerManager(ConfigurationSource config, NodeManager scmNodeManager) throws IOException { EventQueue eventQueue = new EventQueue(); - final int cacheSize = config.getInt(OZONE_SCM_DB_CACHE_SIZE_MB, - OZONE_SCM_DB_CACHE_SIZE_DEFAULT); + PipelineManager pipelineManager = - new SCMPipelineManager(config, scmNodeManager, eventQueue); - return new SCMContainerManager(config, pipelineManager); + new SCMPipelineManager(config, scmNodeManager, + PIPELINES.getTable(dbStore), eventQueue); + return new SCMContainerManager(config, CONTAINERS.getTable(dbStore), + dbStore, + pipelineManager); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java index fcb1c9486488..007f071c510b 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java @@ -37,9 +37,12 @@ import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.MockNodeManager; import org.apache.hadoop.hdds.scm.exceptions.SCMException; +import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager; import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode; import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.test.GenericTestUtils; @@ -55,6 +58,7 @@ import org.junit.Before; import org.junit.Test; + /** * Test cases to verify PipelineManager. */ @@ -62,6 +66,7 @@ public class TestSCMPipelineManager { private static MockNodeManager nodeManager; private static File testDir; private static OzoneConfiguration conf; + private DBStore store; @Before public void setUp() throws Exception { @@ -76,17 +81,24 @@ public void setUp() throws Exception { throw new IOException("Unable to create test directory path"); } nodeManager = new MockNodeManager(true, 20); + + store = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition()); + } @After - public void cleanup() { + public void cleanup() throws Exception { + store.close(); FileUtil.fullyDelete(testDir); } @Test public void testPipelineReload() throws IOException { SCMPipelineManager pipelineManager = - new SCMPipelineManager(conf, nodeManager, new EventQueue()); + new SCMPipelineManager(conf, + nodeManager, + SCMDBDefinition.PIPELINES.getTable(store), + new EventQueue()); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, @@ -106,7 +118,8 @@ public void testPipelineReload() throws IOException { // new pipeline manager should be able to load the pipelines from the db pipelineManager = - new SCMPipelineManager(conf, nodeManager, new EventQueue()); + new SCMPipelineManager(conf, nodeManager, + SCMDBDefinition.PIPELINES.getTable(store), new EventQueue()); pipelineManager.allowPipelineCreation(); mockRatisProvider = new MockRatisPipelineProvider(nodeManager, @@ -137,7 +150,8 @@ public void testPipelineReload() throws IOException { @Test public void testRemovePipeline() throws IOException { SCMPipelineManager pipelineManager = - new SCMPipelineManager(conf, nodeManager, new EventQueue()); + new SCMPipelineManager(conf, nodeManager, + SCMDBDefinition.PIPELINES.getTable(store), new EventQueue()); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, @@ -156,7 +170,8 @@ public void testRemovePipeline() throws IOException { // new pipeline manager should not be able to load removed pipelines pipelineManager = - new SCMPipelineManager(conf, nodeManager, new EventQueue()); + new SCMPipelineManager(conf, nodeManager, + SCMDBDefinition.PIPELINES.getTable(store), new EventQueue()); try { pipelineManager.getPipeline(pipeline.getId()); fail("Pipeline should not have been retrieved"); @@ -172,7 +187,8 @@ public void testRemovePipeline() throws IOException { public void testPipelineReport() throws IOException { EventQueue eventQueue = new EventQueue(); SCMPipelineManager pipelineManager = - new SCMPipelineManager(conf, nodeManager, eventQueue); + new SCMPipelineManager(conf, nodeManager, + SCMDBDefinition.PIPELINES.getTable(store), eventQueue); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, @@ -238,7 +254,8 @@ public void testPipelineCreationFailedMetric() throws Exception { MockNodeManager nodeManagerMock = new MockNodeManager(true, 20); SCMPipelineManager pipelineManager = - new SCMPipelineManager(conf, nodeManagerMock, new EventQueue()); + new SCMPipelineManager(conf, nodeManagerMock, + SCMDBDefinition.PIPELINES.getTable(store), new EventQueue()); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManagerMock, @@ -297,7 +314,8 @@ public void testPipelineCreationFailedMetric() throws Exception { @Test public void testActivateDeactivatePipeline() throws IOException { final SCMPipelineManager pipelineManager = - new SCMPipelineManager(conf, nodeManager, new EventQueue()); + new SCMPipelineManager(conf, nodeManager, + SCMDBDefinition.PIPELINES.getTable(store), new EventQueue()); pipelineManager.allowPipelineCreation(); final PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, @@ -345,7 +363,8 @@ public void testActivateDeactivatePipeline() throws IOException { public void testPipelineOpenOnlyWhenLeaderReported() throws Exception { EventQueue eventQueue = new EventQueue(); SCMPipelineManager pipelineManager = - new SCMPipelineManager(conf, nodeManager, eventQueue); + new SCMPipelineManager(conf, nodeManager, + SCMDBDefinition.PIPELINES.getTable(store), eventQueue); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, @@ -361,7 +380,8 @@ public void testPipelineOpenOnlyWhenLeaderReported() throws Exception { pipelineManager.close(); // new pipeline manager loads the pipelines from the db in ALLOCATED state pipelineManager = - new SCMPipelineManager(conf, nodeManager, eventQueue); + new SCMPipelineManager(conf, nodeManager, + SCMDBDefinition.PIPELINES.getTable(store), eventQueue); mockRatisProvider = new MockRatisPipelineProvider(nodeManager, pipelineManager.getStateManager(), conf); @@ -406,7 +426,8 @@ public void testScrubPipeline() throws IOException { EventQueue eventQueue = new EventQueue(); final SCMPipelineManager pipelineManager = - new SCMPipelineManager(conf, nodeManager, eventQueue); + new SCMPipelineManager(conf, nodeManager, + SCMDBDefinition.PIPELINES.getTable(store), eventQueue); pipelineManager.allowPipelineCreation(); final PipelineProvider ratisProvider = new MockRatisPipelineProvider( nodeManager, pipelineManager.getStateManager(), conf, eventQueue, @@ -448,8 +469,9 @@ public void testPipelineNotCreatedUntilSafeModePrecheck() TimeUnit.MILLISECONDS); EventQueue eventQueue = new EventQueue(); - final SCMPipelineManager pipelineManager = - new SCMPipelineManager(conf, nodeManager, eventQueue); + SCMPipelineManager pipelineManager = + new SCMPipelineManager(conf, nodeManager, + SCMDBDefinition.PIPELINES.getTable(store), eventQueue); final PipelineProvider ratisProvider = new MockRatisPipelineProvider( nodeManager, pipelineManager.getStateManager(), conf, eventQueue, false); @@ -484,7 +506,6 @@ public Boolean get() { pipelineManager.close(); } - @Test public void testSafeModeUpdatedOnSafemodeExit() throws IOException, TimeoutException, InterruptedException { @@ -494,8 +515,9 @@ public void testSafeModeUpdatedOnSafemodeExit() TimeUnit.MILLISECONDS); EventQueue eventQueue = new EventQueue(); - final SCMPipelineManager pipelineManager = - new SCMPipelineManager(conf, nodeManager, eventQueue); + SCMPipelineManager pipelineManager = + new SCMPipelineManager(conf, nodeManager, + SCMDBDefinition.PIPELINES.getTable(store), eventQueue); final PipelineProvider ratisProvider = new MockRatisPipelineProvider( nodeManager, pipelineManager.getStateManager(), conf, eventQueue, false); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java index bda6f846b1a6..700479d40645 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java @@ -18,6 +18,11 @@ package org.apache.hadoop.hdds.scm.safemode; +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -26,21 +31,20 @@ import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.MockNodeManager; import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; -import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.test.GenericTestUtils; + import org.junit.Assert; import org.junit.Test; import org.slf4j.LoggerFactory; -import java.io.File; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - /** * This class tests HealthyPipelineSafeMode rule. */ @@ -49,7 +53,7 @@ public class TestHealthyPipelineSafeModeRule { @Test public void testHealthyPipelineSafeModeRuleWithNoPipelines() throws Exception { - + DBStore store = null; String storageDir = GenericTestUtils.getTempPath( TestHealthyPipelineSafeModeRule.class.getName() + UUID.randomUUID()); try { @@ -65,9 +69,9 @@ public void testHealthyPipelineSafeModeRuleWithNoPipelines() HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); config.setBoolean( HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false); - + store = DBStoreBuilder.createDBStore(config, new SCMDBDefinition()); SCMPipelineManager pipelineManager = new SCMPipelineManager(config, - nodeManager, eventQueue); + nodeManager, SCMDBDefinition.PIPELINES.getTable(store), eventQueue); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, pipelineManager.getStateManager(), config); @@ -82,16 +86,16 @@ public void testHealthyPipelineSafeModeRuleWithNoPipelines() // This should be immediately satisfied, as no pipelines are there yet. Assert.assertTrue(healthyPipelineSafeModeRule.validate()); } finally { + store.close(); FileUtil.fullyDelete(new File(storageDir)); } } @Test public void testHealthyPipelineSafeModeRuleWithPipelines() throws Exception { - String storageDir = GenericTestUtils.getTempPath( TestHealthyPipelineSafeModeRule.class.getName() + UUID.randomUUID()); - + DBStore store = null; try { EventQueue eventQueue = new EventQueue(); List containers = @@ -109,8 +113,9 @@ public void testHealthyPipelineSafeModeRuleWithPipelines() throws Exception { config.setBoolean( HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false); + store = DBStoreBuilder.createDBStore(config, new SCMDBDefinition()); SCMPipelineManager pipelineManager = new SCMPipelineManager(config, - nodeManager, eventQueue); + nodeManager, SCMDBDefinition.PIPELINES.getTable(store), eventQueue); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = @@ -153,6 +158,7 @@ public void testHealthyPipelineSafeModeRuleWithPipelines() throws Exception { GenericTestUtils.waitFor(() -> healthyPipelineSafeModeRule.validate(), 1000, 5000); } finally { + store.close(); FileUtil.fullyDelete(new File(storageDir)); } } @@ -164,6 +170,7 @@ public void testHealthyPipelineSafeModeRuleWithMixedPipelines() String storageDir = GenericTestUtils.getTempPath( TestHealthyPipelineSafeModeRule.class.getName() + UUID.randomUUID()); + DBStore store = null; try { EventQueue eventQueue = new EventQueue(); @@ -183,8 +190,10 @@ public void testHealthyPipelineSafeModeRuleWithMixedPipelines() config.setBoolean( HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false); + store = DBStoreBuilder.createDBStore(config, new SCMDBDefinition()); SCMPipelineManager pipelineManager = new SCMPipelineManager(config, - nodeManager, eventQueue); + nodeManager, SCMDBDefinition.PIPELINES.getTable(store), eventQueue); + pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, @@ -235,6 +244,7 @@ public void testHealthyPipelineSafeModeRuleWithMixedPipelines() 1000, 5000); } finally { + store.close(); FileUtil.fullyDelete(new File(storageDir)); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java index dd8d301b9b0a..c1f09faa3cb0 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java @@ -17,28 +17,32 @@ package org.apache.hadoop.hdds.scm.safemode; +import java.util.ArrayList; +import java.util.List; + import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.HddsTestUtils; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.MockNodeManager; -import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.test.GenericTestUtils; + import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.List; - /** * This class tests OneReplicaPipelineSafeModeRule. */ @@ -50,7 +54,6 @@ public class TestOneReplicaPipelineSafeModeRule { private SCMPipelineManager pipelineManager; private EventQueue eventQueue; - private void setup(int nodes, int pipelineFactorThreeCount, int pipelineFactorOneCount) throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); @@ -66,8 +69,13 @@ private void setup(int nodes, int pipelineFactorThreeCount, MockNodeManager mockNodeManager = new MockNodeManager(true, nodes); eventQueue = new EventQueue(); + + DBStore dbStore = + DBStoreBuilder.createDBStore(ozoneConfiguration, new SCMDBDefinition()); + pipelineManager = new SCMPipelineManager(ozoneConfiguration, mockNodeManager, + SCMDBDefinition.PIPELINES.getTable(dbStore), eventQueue); pipelineManager.allowPipelineCreation(); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java index 0620883520bf..9d2230492290 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdds.scm.safemode; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -33,6 +34,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.MockNodeManager; import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; @@ -41,8 +43,11 @@ import org.apache.hadoop.hdds.server.events.EventHandler; import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.test.GenericTestUtils; +import org.junit.After; import org.junit.Assert; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -69,6 +74,8 @@ public class TestSCMSafeModeManager { @Rule public final TemporaryFolder tempDir = new TemporaryFolder(); + private DBStore dbStore; + @Before public void setUp() { queue = new EventQueue(); @@ -77,6 +84,20 @@ public void setUp() { false); } + @Before + public void initDbStore() throws IOException { + config.set(HddsConfigKeys.OZONE_METADATA_DIRS, + tempDir.newFolder().getAbsolutePath()); + dbStore = DBStoreBuilder.createDBStore(config, new SCMDBDefinition()); + } + + @After + public void destroyDbStore() throws Exception { + if (dbStore != null) { + dbStore.close(); + } + } + @Test public void testSafeModeState() throws Exception { // Test 1: test for 0 containers @@ -184,12 +205,32 @@ private OzoneConfiguration createConf(double healthyPercent, } @Test - public void testSafeModeExitRuleWithPipelineAvailabilityCheck() - throws Exception{ + public void testSafeModeExitRuleWithPipelineAvailabilityCheck1() + throws Exception { testSafeModeExitRuleWithPipelineAvailabilityCheck(100, 30, 8, 0.90, 1); + } + + @Test + public void testSafeModeExitRuleWithPipelineAvailabilityCheck2() + throws Exception { testSafeModeExitRuleWithPipelineAvailabilityCheck(100, 90, 22, 0.10, 0.9); + } + + @Test + public void testSafeModeExitRuleWithPipelineAvailabilityCheck3() + throws Exception { testSafeModeExitRuleWithPipelineAvailabilityCheck(100, 30, 8, 0, 0.9); + } + + @Test + public void testSafeModeExitRuleWithPipelineAvailabilityCheck4() + throws Exception { testSafeModeExitRuleWithPipelineAvailabilityCheck(100, 90, 22, 0, 0); + } + + @Test + public void testSafeModeExitRuleWithPipelineAvailabilityCheck5() + throws Exception { testSafeModeExitRuleWithPipelineAvailabilityCheck(100, 90, 22, 0, 0.5); } @@ -201,7 +242,7 @@ public void testFailWithIncorrectValueForHealthyPipelinePercent() 0.9); MockNodeManager mockNodeManager = new MockNodeManager(true, 10); PipelineManager pipelineManager = new SCMPipelineManager(conf, - mockNodeManager, queue); + mockNodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue); scmSafeModeManager = new SCMSafeModeManager( conf, containers, pipelineManager, queue); fail("testFailWithIncorrectValueForHealthyPipelinePercent"); @@ -219,7 +260,7 @@ public void testFailWithIncorrectValueForOneReplicaPipelinePercent() 200); MockNodeManager mockNodeManager = new MockNodeManager(true, 10); PipelineManager pipelineManager = new SCMPipelineManager(conf, - mockNodeManager, queue); + mockNodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue); scmSafeModeManager = new SCMSafeModeManager( conf, containers, pipelineManager, queue); fail("testFailWithIncorrectValueForOneReplicaPipelinePercent"); @@ -236,7 +277,7 @@ public void testFailWithIncorrectValueForSafeModePercent() throws Exception { conf.setDouble(HddsConfigKeys.HDDS_SCM_SAFEMODE_THRESHOLD_PCT, -1.0); MockNodeManager mockNodeManager = new MockNodeManager(true, 10); PipelineManager pipelineManager = new SCMPipelineManager(conf, - mockNodeManager, queue); + mockNodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue); scmSafeModeManager = new SCMSafeModeManager( conf, containers, pipelineManager, queue); fail("testFailWithIncorrectValueForSafeModePercent"); @@ -260,7 +301,7 @@ public void testSafeModeExitRuleWithPipelineAvailabilityCheck( MockNodeManager mockNodeManager = new MockNodeManager(true, nodeCount); SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, - mockNodeManager, queue); + mockNodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(mockNodeManager, pipelineManager.getStateManager(), config, true); @@ -477,7 +518,7 @@ public void testSafeModePipelineExitRule() throws Exception { HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); SCMPipelineManager pipelineManager = new SCMPipelineManager(config, - nodeManager, queue); + nodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, @@ -531,7 +572,8 @@ public void testPipelinesNotCreatedUntilPreCheckPasses() HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); SCMPipelineManager pipelineManager = new SCMPipelineManager(config, - nodeManager, queue); + nodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue); + PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml index 8be5c5b9364f..e892053596ed 100644 --- a/hadoop-ozone/dist/pom.xml +++ b/hadoop-ozone/dist/pom.xml @@ -23,7 +23,7 @@ hadoop-ozone-dist Apache Hadoop Ozone Distribution - pom + jar 0.6.0-SNAPSHOT UTF-8 diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java index 20afa096a433..aa524b02f00c 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.recon.scm; -import java.io.File; import java.io.IOException; import org.apache.hadoop.hdds.conf.ConfigurationSource; @@ -30,11 +29,11 @@ import org.apache.hadoop.hdds.scm.container.SCMContainerManager; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; -import org.apache.hadoop.ozone.recon.ReconUtils; +import org.apache.hadoop.hdds.utils.db.BatchOperationHandler; +import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.recon.persistence.ContainerSchemaManager; import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; -import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_SCM_CONTAINER_DB; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,33 +55,29 @@ public class ReconContainerManager extends SCMContainerManager { * CacheSize is specified * in MB. * - * @param conf - {@link ConfigurationSource} - * @param pipelineManager - {@link PipelineManager} * @throws IOException on Failure. */ public ReconContainerManager( - ConfigurationSource conf, PipelineManager pipelineManager, + ConfigurationSource conf, + Table containerStore, + BatchOperationHandler batchHandler, + PipelineManager pipelineManager, StorageContainerServiceProvider scm, ContainerSchemaManager containerSchemaManager) throws IOException { - super(conf, pipelineManager); + super(conf, containerStore, batchHandler, pipelineManager); this.scmClient = scm; this.containerSchemaManager = containerSchemaManager; } - @Override - protected File getContainerDBPath(ConfigurationSource conf) { - File metaDir = ReconUtils.getReconScmDbDir(conf); - return new File(metaDir, RECON_SCM_CONTAINER_DB); - } - /** * Check and add new container if not already present in Recon. - * @param containerID containerID to check. + * + * @param containerID containerID to check. * @param datanodeDetails Datanode from where we got this container. * @throws IOException on Error. */ public void checkAndAddNewContainer(ContainerID containerID, - DatanodeDetails datanodeDetails) + DatanodeDetails datanodeDetails) throws IOException { if (!exists(containerID)) { LOG.info("New container {} got from {}.", containerID, @@ -143,7 +138,7 @@ public void addNewContainer(long containerId, */ @Override public void updateContainerReplica(ContainerID containerID, - ContainerReplica replica) + ContainerReplica replica) throws ContainerNotFoundException { super.updateContainerReplica(containerID, replica); // Update container_history table diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconDBDefinition.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconDBDefinition.java new file mode 100644 index 000000000000..bcfe060e7baf --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconDBDefinition.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.ozone.recon.scm; + +import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.ozone.recon.ReconServerConfigKeys; + +/** + * SCM db file for ozone. + */ +public class ReconDBDefinition extends SCMDBDefinition { + + @Override + public String getName() { + return "recon-scm.db"; + } + + @Override + public String getLocationConfigKey() { + return ReconServerConfigKeys.OZONE_RECON_SCM_DB_DIR; + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineManager.java index 20f77c77b937..a8dd3c990ca3 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineManager.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.recon.scm; -import java.io.File; import java.io.IOException; import java.util.List; import java.util.stream.Collectors; @@ -31,14 +30,15 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineStateManager; import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; import org.apache.hadoop.hdds.server.events.EventPublisher; -import org.apache.hadoop.ozone.recon.ReconUtils; +import org.apache.hadoop.hdds.utils.db.Table; import com.google.common.annotations.VisibleForTesting; import static org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState.CLOSED; -import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_SCM_PIPELINE_DB; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + + /** * Recon's overriding implementation of SCM's Pipeline Manager. */ @@ -48,20 +48,16 @@ public class ReconPipelineManager extends SCMPipelineManager { LoggerFactory.getLogger(ReconPipelineManager.class); public ReconPipelineManager(ConfigurationSource conf, - NodeManager nodeManager, - EventPublisher eventPublisher) + NodeManager nodeManager, + Table pipelineStore, + EventPublisher eventPublisher) throws IOException { - super(conf, nodeManager, eventPublisher, new PipelineStateManager(), + super(conf, nodeManager, pipelineStore, eventPublisher, + new PipelineStateManager(), new ReconPipelineFactory()); initializePipelineState(); } - - @Override - protected File getPipelineDBPath(ConfigurationSource conf) { - File metaDir = ReconUtils.getReconScmDbDir(conf); - return new File(metaDir, RECON_SCM_PIPELINE_DB); - } - + @Override public void triggerPipelineCreation() { // Don't do anything in Recon. @@ -147,8 +143,7 @@ public void removeInvalidPipelines(List pipelinesFromScm) { void addPipeline(Pipeline pipeline) throws IOException { getLock().writeLock().lock(); try { - getPipelineStore().put(pipeline.getId().getProtobuf().toByteArray(), - pipeline.getProtobufMessage().toByteArray()); + getPipelineStore().put(pipeline.getId(), pipeline); getStateManager().addPipeline(pipeline); getNodeManager().addPipeline(pipeline); } finally { diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java index 61de428d1e75..7800abb7dae0 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java @@ -18,9 +18,6 @@ package org.apache.hadoop.ozone.recon.scm; -import static org.apache.hadoop.hdds.recon.ReconConfigKeys.RECON_SCM_CONFIG_PREFIX; -import static org.apache.hadoop.hdds.scm.server.StorageContainerManager.buildRpcServerStartMessage; - import java.io.IOException; import java.net.InetSocketAddress; import java.util.HashSet; @@ -28,8 +25,6 @@ import java.util.Map; import java.util.Set; - -import com.google.inject.Inject; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.block.BlockManager; import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler; @@ -49,13 +44,19 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineActionHandler; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.scm.safemode.SafeModeManager; -import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ozone.recon.fsck.MissingContainerTask; import org.apache.hadoop.ozone.recon.persistence.ContainerSchemaManager; import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; + +import com.google.inject.Inject; +import static org.apache.hadoop.hdds.recon.ReconConfigKeys.RECON_SCM_CONFIG_PREFIX; +import static org.apache.hadoop.hdds.scm.server.StorageContainerManager.buildRpcServerStartMessage; import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,6 +74,7 @@ public class ReconStorageContainerManagerFacade private final ReconDatanodeProtocolServer datanodeProtocolServer; private final EventQueue eventQueue; private final SCMStorageConfig scmStorageConfig; + private final DBStore dbStore; private ReconNodeManager nodeManager; private ReconPipelineManager pipelineManager; @@ -83,23 +85,34 @@ public class ReconStorageContainerManagerFacade @Inject public ReconStorageContainerManagerFacade(OzoneConfiguration conf, - StorageContainerServiceProvider scmServiceProvider, - ReconTaskStatusDao reconTaskStatusDao, - ContainerSchemaManager containerSchemaManager) + StorageContainerServiceProvider scmServiceProvider, + ReconTaskStatusDao reconTaskStatusDao, + ContainerSchemaManager containerSchemaManager) throws IOException { this.eventQueue = new EventQueue(); eventQueue.setSilent(true); this.ozoneConfiguration = getReconScmConfiguration(conf); this.scmStorageConfig = new ReconStorageConfig(conf); this.clusterMap = new NetworkTopologyImpl(conf); + dbStore = DBStoreBuilder + .createDBStore(ozoneConfiguration, new ReconDBDefinition()); + this.nodeManager = new ReconNodeManager(conf, scmStorageConfig, eventQueue, clusterMap); this.datanodeProtocolServer = new ReconDatanodeProtocolServer( conf, this, eventQueue); this.pipelineManager = - new ReconPipelineManager(conf, nodeManager, eventQueue); - this.containerManager = new ReconContainerManager(conf, pipelineManager, - scmServiceProvider, containerSchemaManager); + + new ReconPipelineManager(conf, + nodeManager, + ReconDBDefinition.PIPELINES.getTable(dbStore), + eventQueue); + this.containerManager = new ReconContainerManager(conf, + ReconDBDefinition.CONTAINERS.getTable(dbStore), + dbStore, + pipelineManager, + scmServiceProvider, + containerSchemaManager); this.scmServiceProvider = scmServiceProvider; NodeReportHandler nodeReportHandler = @@ -214,6 +227,11 @@ public void stop() { IOUtils.cleanupWithLogger(LOG, nodeManager); IOUtils.cleanupWithLogger(LOG, containerManager); IOUtils.cleanupWithLogger(LOG, pipelineManager); + try { + dbStore.close(); + } catch (Exception e) { + LOG.error("Can't close dbStore ", e); + } } public ReconDatanodeProtocolServer getDatanodeProtocolServer() { diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java index 7f87806d5079..04010e512f3d 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java @@ -18,15 +18,6 @@ package org.apache.hadoop.ozone.recon.scm; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.OPEN; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.STAND_ALONE; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - import java.io.IOException; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -40,12 +31,23 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.ozone.recon.persistence.ContainerSchemaManager; import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; + +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.OPEN; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.STAND_ALONE; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline; import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.rules.TemporaryFolder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** * Abstract class for Recon Container Manager related tests. @@ -59,6 +61,7 @@ public class AbstractReconContainerManagerTest { private SCMStorageConfig scmStorageConfig; private ReconPipelineManager pipelineManager; private ReconContainerManager containerManager; + private DBStore store; @Before public void setUp() throws Exception { @@ -66,20 +69,28 @@ public void setUp() throws Exception { conf.set(OZONE_METADATA_DIRS, temporaryFolder.newFolder().getAbsolutePath()); conf.set(OZONE_SCM_NAMES, "localhost"); + store = DBStoreBuilder.createDBStore(conf, new ReconDBDefinition()); scmStorageConfig = new ReconStorageConfig(conf); NetworkTopology clusterMap = new NetworkTopologyImpl(conf); EventQueue eventQueue = new EventQueue(); NodeManager nodeManager = new SCMNodeManager(conf, scmStorageConfig, eventQueue, clusterMap); - pipelineManager = new ReconPipelineManager(conf, nodeManager, eventQueue); - containerManager = new ReconContainerManager(conf, pipelineManager, - getScmServiceProvider(), mock(ContainerSchemaManager.class)); + pipelineManager = new ReconPipelineManager(conf, nodeManager, + ReconDBDefinition.PIPELINES.getTable(store), eventQueue); + containerManager = new ReconContainerManager( + conf, + ReconDBDefinition.CONTAINERS.getTable(store), + store, + pipelineManager, + getScmServiceProvider(), + mock(ContainerSchemaManager.class)); } @After - public void tearDown() throws IOException { + public void tearDown() throws Exception { containerManager.close(); pipelineManager.close(); + store.close(); } protected OzoneConfiguration getConf() { diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineManager.java index 3d4d239a5778..c891f3321725 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineManager.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineManager.java @@ -18,15 +18,6 @@ package org.apache.hadoop.ozone.recon.scm; -import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; - import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -45,11 +36,23 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.ozone.recon.scm.ReconPipelineFactory.ReconPipelineProvider; + +import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline; +import org.junit.After; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import static org.mockito.Mockito.mock; /** * Class to test Recon Pipeline Manager. @@ -61,6 +64,7 @@ public class TestReconPipelineManager { private OzoneConfiguration conf; private SCMStorageConfig scmStorageConfig; + private DBStore store; @Before public void setup() throws IOException { @@ -69,6 +73,12 @@ public void setup() throws IOException { temporaryFolder.newFolder().getAbsolutePath()); conf.set(OZONE_SCM_NAMES, "localhost"); scmStorageConfig = new ReconStorageConfig(conf); + store = DBStoreBuilder.createDBStore(conf, new ReconDBDefinition()); + } + + @After + public void tearDown() throws Exception { + store.close(); } @Test @@ -103,7 +113,8 @@ public void testInitialize() throws IOException { new SCMNodeManager(conf, scmStorageConfig, eventQueue, clusterMap); try (ReconPipelineManager reconPipelineManager = - new ReconPipelineManager(conf, nodeManager, eventQueue)) { + new ReconPipelineManager(conf, nodeManager, + ReconDBDefinition.PIPELINES.getTable(store), eventQueue)) { reconPipelineManager.addPipeline(validPipeline); reconPipelineManager.addPipeline(invalidPipeline); @@ -138,7 +149,8 @@ public void testAddPipeline() throws IOException { new SCMNodeManager(conf, scmStorageConfig, eventQueue, clusterMap); ReconPipelineManager reconPipelineManager = - new ReconPipelineManager(conf, nodeManager, eventQueue); + new ReconPipelineManager(conf, nodeManager, + ReconDBDefinition.PIPELINES.getTable(store), eventQueue); assertFalse(reconPipelineManager.containsPipeline(pipeline.getId())); reconPipelineManager.addPipeline(pipeline); assertTrue(reconPipelineManager.containsPipeline(pipeline.getId())); @@ -150,7 +162,8 @@ public void testStubbedReconPipelineFactory() throws IOException { NodeManager nodeManagerMock = mock(NodeManager.class); ReconPipelineManager reconPipelineManager = new ReconPipelineManager( - conf, nodeManagerMock, new EventQueue()); + conf, nodeManagerMock, ReconDBDefinition.PIPELINES.getTable(store), + new EventQueue()); PipelineFactory pipelineFactory = reconPipelineManager.getPipelineFactory(); assertTrue(pipelineFactory instanceof ReconPipelineFactory); ReconPipelineFactory reconPipelineFactory = diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkOzoneManager.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkOzoneManager.java index 6e531f3e1c84..19b5e8e642bd 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkOzoneManager.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkOzoneManager.java @@ -18,8 +18,6 @@ package org.apache.hadoop.ozone.genesis; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT; - import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -27,7 +25,6 @@ import java.util.UUID; import java.util.concurrent.locks.ReentrantLock; -import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -47,7 +44,9 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.client.AuthenticationException; + +import org.apache.commons.lang3.RandomStringUtils; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.Level; import org.openjdk.jmh.annotations.Scope; @@ -77,7 +76,7 @@ public class BenchMarkOzoneManager { @Setup(Level.Trial) public static void initialize() - throws IOException, AuthenticationException, InterruptedException { + throws Exception { try { lock.lock(); if (scm == null) { diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkSCM.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkSCM.java index 0839ea53acf9..64e2f4d0adec 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkSCM.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkSCM.java @@ -18,13 +18,10 @@ package org.apache.hadoop.ozone.genesis; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT; - import java.io.File; import java.io.IOException; import java.util.concurrent.locks.ReentrantLock; -import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -38,7 +35,9 @@ import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager; import org.apache.hadoop.hdds.scm.server.SCMConfigurator; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; -import org.apache.hadoop.security.authentication.client.AuthenticationException; + +import org.apache.commons.lang3.RandomStringUtils; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.Level; import org.openjdk.jmh.annotations.Param; @@ -67,7 +66,7 @@ public class BenchMarkSCM { @Setup(Level.Trial) public static void initialize() - throws IOException, AuthenticationException, InterruptedException { + throws Exception { try { lock.lock(); if (scm == null) { diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java index e98cabca6b73..797c8051bc78 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.ozone.genesis; -import java.io.File; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; @@ -32,15 +31,17 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.server.SCMConfigurator; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; -import org.apache.hadoop.hdds.server.ServerUtils; import org.apache.hadoop.hdds.utils.MetadataStore; import org.apache.hadoop.hdds.utils.MetadataStoreBuilder; -import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; +import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.common.Storage; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMStorage; @@ -48,9 +49,7 @@ import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.commons.lang3.RandomStringUtils; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_DEFAULT; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_MB; -import static org.apache.hadoop.ozone.OzoneConsts.SCM_PIPELINE_DB; + /** * Utility class for benchmark test cases. @@ -150,16 +149,11 @@ static void configureSCM(OzoneConfiguration conf, int numHandlers) { } static void addPipelines(HddsProtos.ReplicationFactor factor, - int numPipelines, ConfigurationSource conf) throws IOException { - final File metaDir = ServerUtils.getScmDbDir(conf); - final File pipelineDBPath = new File(metaDir, SCM_PIPELINE_DB); - int cacheSize = conf.getInt(OZONE_SCM_DB_CACHE_SIZE_MB, - OZONE_SCM_DB_CACHE_SIZE_DEFAULT); - MetadataStore pipelineStore = - MetadataStoreBuilder.newBuilder().setCreateIfMissing(true) - .setConf(conf).setDbFile(pipelineDBPath) - .setCacheSize(cacheSize * OzoneConsts.MB).build(); + int numPipelines, ConfigurationSource conf) throws Exception { + DBStore dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition()); + Table pipelineTable = + SCMDBDefinition.PIPELINES.getTable(dbStore); List nodes = new ArrayList<>(); for (int i = 0; i < factor.getNumber(); i++) { nodes @@ -174,11 +168,11 @@ static void addPipelines(HddsProtos.ReplicationFactor factor, .setFactor(factor) .setNodes(nodes) .build(); - pipelineStore.put(pipeline.getId().getProtobuf().toByteArray(), - pipeline.getProtobufMessage().toByteArray()); + pipelineTable.put(pipeline.getId(), + pipeline); } - pipelineStore.close(); + dbStore.close(); } static OzoneManager getOm(OzoneConfiguration conf)