diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java index 8e4ec6a00f8f..b8f0fb623fdf 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java @@ -111,10 +111,12 @@ private void createPipelines() { continue; } - try { - pipelineManager.scrubPipeline(type, factor); - } catch (IOException e) { - LOG.error("Error while scrubbing pipelines {}", e); + if (!pipelineManager.getSafeModeStatus()) { + try { + pipelineManager.scrubPipeline(type, factor); + } catch (IOException e) { + LOG.error("Error while scrubbing pipelines {}", e); + } } while (true) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java index 635e032c4764..68d66ff10d0a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java @@ -112,4 +112,17 @@ void scrubPipeline(ReplicationType type, ReplicationFactor factor) default void waitPipelineReady(PipelineID pipelineID, long timeout) throws IOException { } + + /** + * Set SafeMode status. + * + * @param safeModeStatus + */ + void setSafeModeStatus(boolean safeModeStatus); + + /** + * Get SafeMode status. + * @return boolean + */ + boolean getSafeModeStatus(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java index eb2e0d6d0a22..0989d34fc21d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java @@ -54,6 +54,7 @@ import java.util.Set; import java.util.Collection; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; @@ -85,6 +86,8 @@ public class SCMPipelineManager implements PipelineManager { // Pipeline Manager MXBean private ObjectName pmInfoBean; + private final AtomicBoolean isInSafeMode; + public SCMPipelineManager(Configuration conf, NodeManager nodeManager, EventPublisher eventPublisher) throws IOException { @@ -127,6 +130,9 @@ protected SCMPipelineManager(Configuration conf, NodeManager nodeManager, HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL, HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); + this.isInSafeMode = new AtomicBoolean(conf.getBoolean( + HddsConfigKeys.HDDS_SCM_SAFEMODE_ENABLED, + HddsConfigKeys.HDDS_SCM_SAFEMODE_ENABLED_DEFAULT)); } public PipelineStateManager getStateManager() { @@ -414,7 +420,7 @@ public void scrubPipeline(ReplicationType type, ReplicationFactor factor) .toEpochMilli() >= pipelineScrubTimeoutInMills) .collect(Collectors.toList()); for (Pipeline p : needToSrubPipelines) { - LOG.info("srubbing pipeline: id: " + p.getId().toString() + + LOG.info("Scrubbing pipeline: id: " + p.getId().toString() + " since it stays at ALLOCATED stage for " + Duration.between(currentTime, p.getCreationTimestamp()).toMinutes() + " mins."); @@ -618,4 +624,15 @@ protected MetadataStore getPipelineStore() { protected NodeManager getNodeManager() { return nodeManager; } + + @Override + public void setSafeModeStatus(boolean safeModeStatus) { + this.isInSafeMode.set(safeModeStatus); + } + + @Override + public boolean getSafeModeStatus() { + return this.isInSafeMode.get(); + } + } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SCMSafeModeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SCMSafeModeManager.java index 90fce3f63042..ca484c834641 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SCMSafeModeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SCMSafeModeManager.java @@ -134,7 +134,6 @@ public SCMSafeModeManager(Configuration conf, exitRules.put(ATLEAST_ONE_DATANODE_REPORTED_PIPELINE_EXIT_RULE, oneReplicaPipelineSafeModeRule); } - emitSafeModeStatus(); boolean createPipelineInSafemode = conf.getBoolean( HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION_DEFAULT); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeHandler.java index 2fbe89361412..50095abbf979 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeHandler.java @@ -21,7 +21,6 @@ import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.scm.block.BlockManager; import org.apache.hadoop.hdds.scm.container.ReplicationManager; -import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.scm.server.SCMClientProtocolServer; import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager.SafeModeStatus; @@ -30,8 +29,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.List; import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -94,8 +91,8 @@ public SafeModeHandler(Configuration configuration, * Set SafeMode status based on * {@link org.apache.hadoop.hdds.scm.events.SCMEvents#SAFE_MODE_STATUS}. * - * Inform BlockManager, ScmClientProtocolServer and replicationAcitivity - * status about safeMode status. + * Inform BlockManager, ScmClientProtocolServer, ScmPipeline Manager and + * Replication Manager status about safeMode status. * * @param safeModeStatus * @param publisher @@ -114,8 +111,9 @@ public void onMessage(SafeModeStatus safeModeStatus, } catch (InterruptedException e) { Thread.currentThread().interrupt(); } + scmPipelineManager.setSafeModeStatus(isInSafeMode.get()); replicationManager.start(); - cleanupPipelines(); + scmPipelineManager.triggerPipelineCreation(); }); safeModeExitThread.setDaemon(true); @@ -124,21 +122,6 @@ public void onMessage(SafeModeStatus safeModeStatus, } - private void cleanupPipelines() { - List pipelineList = scmPipelineManager.getPipelines(); - pipelineList.forEach((pipeline) -> { - try { - if (pipeline.getPipelineState() == Pipeline.PipelineState.ALLOCATED && - pipeline.isAllocationTimeout()) { - scmPipelineManager.finalizeAndDestroyPipeline(pipeline, false); - } - } catch (IOException ex) { - LOG.error("Finalize and destroy pipeline failed for pipeline " - + pipeline.toString(), ex); - } - }); - } - public boolean getSafeModeStatus() { return isInSafeMode.get(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 6063a6248da5..133429338a2a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -358,6 +358,9 @@ public StorageContainerManager(OzoneConfiguration conf, eventQueue.addHandler(SCMEvents.PIPELINE_ACTIONS, pipelineActionHandler); eventQueue.addHandler(SCMEvents.PIPELINE_REPORT, pipelineReportHandler); eventQueue.addHandler(SCMEvents.SAFE_MODE_STATUS, safeModeHandler); + + // Emit initial safe mode status, as now handlers are registered. + scmSafeModeManager.emitSafeModeStatus(); registerMXBean(); registerMetricsSource(this); }