HDDS-7853. Add support for RemoveSCM in SCMRatisServer.

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/RemoveSCMRequest.java

@@ -0,0 +1,112 @@
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+
+/**
+ * Request class using which SCM can be removed form the HA Ring.
+ */
+public class RemoveSCMRequest {
+
+  private final String clusterId;
+  private final String scmId;
+  private final String ratisAddr;
+
+  public RemoveSCMRequest(String clusterId, String scmId, String addr) {
+    this.clusterId = clusterId;
+    this.scmId = scmId;
+    this.ratisAddr = addr;
+  }
+
+  public static RemoveSCMRequest getFromProtobuf(
+      HddsProtos.RemoveScmRequestProto proto) {
+    return new Builder().setClusterId(proto.getClusterId())
+        .setScmId(proto.getScmId()).setRatisAddr(proto.getRatisAddr()).build();
+  }
+
+  public HddsProtos.RemoveScmRequestProto getProtobuf() {
+    return HddsProtos.RemoveScmRequestProto.newBuilder().setClusterId(clusterId)
+        .setScmId(scmId).setRatisAddr(ratisAddr).build();
+  }
+  /**
+   * Builder for RemoveSCMRequest.
+   */
+  public static class Builder {
+    private String clusterId;
+    private String scmId;
+    private String ratisAddr;
+
+
+    /**
+     * sets the cluster id.
+     * @param cid clusterId to be set
+     * @return Builder for RemoveSCMRequest
+     */
+    public RemoveSCMRequest.Builder setClusterId(String cid) {
+      this.clusterId = cid;
+      return this;
+    }
+
+    /**
+     * sets the scmId.
+     * @param id scmId
+     * @return Builder for RemoveSCMRequest
+     */
+    public RemoveSCMRequest.Builder setScmId(String id) {
+      this.scmId = id;
+      return this;
+    }
+
+    /**
+     * Set ratis address in Scm HA.
+     * @param   addr  address in the format of [ip|hostname]:port
+     * @return  Builder for RemoveSCMRequest
+     */
+    public RemoveSCMRequest.Builder setRatisAddr(String addr) {
+      this.ratisAddr = addr;
+      return this;
+    }
+
+    public RemoveSCMRequest build() {
+      return new RemoveSCMRequest(clusterId, scmId, ratisAddr);
+    }
+  }
+
+  /**
+   * Gets the clusterId from the Version file.
+   * @return ClusterId
+   */
+  public String getClusterId() {
+    return clusterId;
+  }
+
+  /**
+   * Gets the SCM Id from the Version file.
+   * @return SCM Id
+   */
+  public String getScmId() {
+    return scmId;
+  }
+
+  public String getRatisAddr() {
+    return ratisAddr;
+  }
+
+}

hadoop-hdds/interface-client/src/main/proto/hdds.proto

@@ -265,6 +265,17 @@ message AddScmResponseProto {
     optional string scmId = 2;
 }
 
+message RemoveScmRequestProto {
+    required string clusterId = 1;
+    required string scmId = 2;
+    required string ratisAddr = 3;
+}
+
+message RemoveScmResponseProto {
+    required bool success = 1;
+    optional string scmId = 2;
+}
+
 enum ReplicationType {
     RATIS = 1;
     STAND_ALONE = 2;

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManager.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.hdds.scm.ha;
 
 import org.apache.hadoop.hdds.scm.AddSCMRequest;
+import org.apache.hadoop.hdds.scm.RemoveSCMRequest;
 import org.apache.hadoop.hdds.scm.metadata.DBTransactionBuffer;
 import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
 import org.apache.ratis.server.protocol.TermIndex;
@@ -62,13 +63,21 @@ public interface SCMHAManager {
   void stop() throws IOException;
 
   /**
-   * Adds the SC M instance to the SCM HA group.
+   * Adds the SCM instance to the SCM HA Ring.
    * @param request AddSCM request
    * @return status signying whether the AddSCM request succeeded or not.
    * @throws IOException
    */
   boolean addSCM(AddSCMRequest request) throws IOException;
 
+  /** Remove the SCM instance from the SCM HA Ring.
+   * @param request RemoveSCM request
+   *
+   * @return status signaling whether the RemoveSCM request succeeded or not.
+   * @throws IOException
+   */
+  boolean removeSCM(RemoveSCMRequest request) throws IOException;
+
   /**
    * Download the latest checkpoint from leader SCM.
    *

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java

@@ -21,6 +21,7 @@
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.scm.AddSCMRequest;
+import org.apache.hadoop.hdds.scm.RemoveSCMRequest;
 import org.apache.hadoop.hdds.scm.metadata.DBTransactionBuffer;
 import org.apache.hadoop.hdds.scm.metadata.SCMDBTransactionBufferImpl;
 import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore;
@@ -355,6 +356,26 @@ public boolean addSCM(AddSCMRequest request) throws IOException {
     return getRatisServer().addSCM(request);
   }
 
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public boolean removeSCM(RemoveSCMRequest request) throws IOException {
+    // Currently we don't support decommission of Primordial SCM
+    // The caller should make sure that the requested node is not Primordial SCM
+
+    String clusterId = scm.getClusterId();
+    if (!request.getClusterId().equals(scm.getClusterId())) {
+      throw new IOException("SCM " + request.getScmId() +
+          " with address " + request.getRatisAddr() +
+          " has cluster Id " + request.getClusterId() +
+          " but leader SCM cluster id is " + clusterId);
+    }
+    Preconditions.checkNotNull(ratisServer.getDivision().getGroup());
+    return ratisServer.removeSCM(request);
+  }
+
+
   void stopServices() throws Exception {
 
     // just stop the SCMMetaData store. All other background

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerStub.java

@@ -28,6 +28,7 @@
 import com.google.protobuf.InvalidProtocolBufferException;
 import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.RequestType;
 import org.apache.hadoop.hdds.scm.AddSCMRequest;
+import org.apache.hadoop.hdds.scm.RemoveSCMRequest;
 import org.apache.hadoop.hdds.scm.metadata.DBTransactionBuffer;
 import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
 import org.apache.hadoop.hdds.utils.db.DBStore;
@@ -131,6 +132,11 @@ public boolean addSCM(AddSCMRequest request) throws IOException {
     return false;
   }
 
+  @Override
+  public boolean removeSCM(RemoveSCMRequest request) throws IOException {
+    return false;
+  }
+
   @Override
   public DBCheckpoint downloadCheckpointFromLeader(String leaderId) {
     return null;
@@ -251,6 +257,11 @@ public boolean addSCM(AddSCMRequest request) throws IOException {
       return false;
     }
 
+    @Override
+    public boolean removeSCM(RemoveSCMRequest request) throws IOException {
+      return false;
+    }
+
     @Override
     public GrpcTlsConfig getGrpcTlsConfig() {
       return null;

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServer.java

@@ -19,6 +19,7 @@
 
 import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.RequestType;
 import org.apache.hadoop.hdds.scm.AddSCMRequest;
+import org.apache.hadoop.hdds.scm.RemoveSCMRequest;
 import org.apache.ratis.grpc.GrpcTlsConfig;
 import org.apache.ratis.protocol.exceptions.NotLeaderException;
 import org.apache.ratis.server.RaftServer;
@@ -59,6 +60,8 @@ SCMRatisResponse submitRequest(SCMRatisRequest request)
 
   boolean addSCM(AddSCMRequest request) throws IOException;
 
+  boolean removeSCM(RemoveSCMRequest request) throws  IOException;
+
   SCMStateMachine getSCMStateMachine();
 
   GrpcTlsConfig getGrpcTlsConfig();

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java

@@ -35,6 +35,7 @@
 import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.RequestType;
 import org.apache.hadoop.hdds.ratis.RatisHelper;
 import org.apache.hadoop.hdds.scm.AddSCMRequest;
+import org.apache.hadoop.hdds.scm.RemoveSCMRequest;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
 import org.apache.hadoop.hdds.security.x509.SecurityConfig;
@@ -320,6 +321,45 @@ public boolean addSCM(AddSCMRequest request) throws IOException {
     }
   }
 
+  @Override
+  public boolean removeSCM(RemoveSCMRequest request) throws IOException {
+    final List<RaftPeer> newRaftPeerList =
+        new ArrayList<>(division.getGroup().getPeers());
+    // remove the SCM node from the raft peer list
+
+    final RaftPeer raftPeer = RaftPeer.newBuilder().setId(request.getScmId())
+        .setAddress(request.getRatisAddr()).build();
+
+    newRaftPeerList.remove(raftPeer);
+
+    LOG.info("{}: Submitting SetConfiguration request to Ratis server with" +
+            " updated SCM peers list: {}", request.getScmId(),
+        newRaftPeerList);
+    final SetConfigurationRequest configRequest =
+        new SetConfigurationRequest(clientId, division.getPeer().getId(),
+            division.getGroup().getGroupId(), nextCallId(), newRaftPeerList);
+
+    try {
+      RaftClientReply raftClientReply = server.setConfiguration(configRequest);
+      if (raftClientReply.isSuccess()) {
+        LOG.info("Successfully removed SCM: {}.", request.getScmId());
+      } else {
+        LOG.error("Failed to remove SCM: {}. Ratis reply: {}" +
+            request.getScmId(), raftClientReply);
+        throw new IOException(raftClientReply.getException());
+      }
+      return raftClientReply.isSuccess();
+    } catch (IOException e) {
+      if (e instanceof NotLeaderException) {
+        LOG.debug("Cannot remove peer: {}", request.getScmId(), e);
+      } else {
+        LOG.error("Failed to update Ratis configuration and remove peer. " +
+            "Cannot remove SCM: {}.", request.getScmId(), e);
+      }
+      throw e;
+    }
+  }
+
   private static RaftGroup buildRaftGroup(SCMNodeDetails details,
       String scmId, String clusterId) {
     Preconditions.checkNotNull(scmId);

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java

@@ -41,6 +41,7 @@
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
 import org.apache.hadoop.hdds.scm.PipelineChoosePolicy;
 import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.RemoveSCMRequest;
 import org.apache.hadoop.hdds.scm.ScmUtils;
 import org.apache.hadoop.hdds.scm.container.ContainerManager;
 import org.apache.hadoop.hdds.scm.container.ContainerManagerImpl;
@@ -2070,4 +2071,27 @@ private String reconfOzoneAdmins(String newVal) {
         newVal, admins);
     return String.valueOf(newVal);
   }
+
+  /**
+   * This will remove the given SCM node from HA Ring by removing it from
+   * Ratis Ring and deleting the related certificates from certificate store.
+   *
+   * @return true if remove was successful, else false.
+   */
+  public boolean removePeerFromHARing(RemoveSCMRequest request)
+      throws IOException {
+    // We cannot remove a node if it's currently leader.
+    if (scmContext.isLeader() && request.getScmId().equals(getScmId())) {
+      throw new IOException("Cannot remove current leader.");
+    }
+
+    // Currently we don't support removal of primordial node.
+    if (request.getScmId().equals(primaryScmNodeId)) {
+      throw new IOException("Removal of primordial node is not supported.");
+    }
+
+    // TODO: Remove the certificate from certificate store.
+    return scmHAManager.removeSCM(request);
+
+  }
 }

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestReplicationAnnotation.java

@@ -20,6 +20,7 @@
 import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol;
 import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.RequestType;
 import org.apache.hadoop.hdds.scm.AddSCMRequest;
+import org.apache.hadoop.hdds.scm.RemoveSCMRequest;
 import org.apache.hadoop.hdds.scm.container.ContainerStateManager;
 import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore;
 import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
@@ -92,6 +93,11 @@ public boolean addSCM(AddSCMRequest request)
         return false;
       }
 
+      @Override
+      public boolean removeSCM(RemoveSCMRequest request) throws IOException {
+        return false;
+      }
+
       @Override
       public SCMStateMachine getSCMStateMachine() {
         return null;