HDDS-2240. Command line tool for OM HA.

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java

@@ -42,6 +42,7 @@
 
 import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
 import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo;
 import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
 import org.apache.hadoop.ozone.security.acl.OzoneObj;
 import org.apache.hadoop.security.KerberosInfo;
@@ -645,4 +646,12 @@ List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
    * */
   List<OzoneAcl> getAcl(OzoneObj obj) throws IOException;
 
+  /**
+   * Get a list of all OMs and their Ratis server roles.
+   *
+   * @return list of OM server roles
+   * @throws IOException
+   */
+  List<OMRoleInfo> getOMServerRoles() throws IOException;
+
 }

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -72,6 +72,7 @@
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.XceiverClientManager;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo;
 import org.apache.hadoop.ozone.security.GDPRSymmetricKey;
 import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
 import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType;
@@ -1071,6 +1072,11 @@ public List<OzoneAcl> getAcl(OzoneObj obj) throws IOException {
     return ozoneManagerClient.getAcl(obj);
   }
 
+  @Override
+  public List<OMRoleInfo> getOMServerRoles() throws IOException {
+    return ozoneManagerClient.getOMServerRoles();
+  }
+
   private OzoneInputStream createInputStream(OmKeyInfo keyInfo)
       throws IOException {
     LengthInputStream lengthInputStream = KeyInputStream

hadoop-ozone/common/src/main/bin/ozone

@@ -55,6 +55,7 @@ function hadoop_usage
   hadoop_add_subcommand "version" client "print the version"
   hadoop_add_subcommand "dtutil" client "operations related to delegation tokens"
   hadoop_add_subcommand "upgrade" client "HDFS to Ozone in-place upgrade tool"
+  hadoop_add_subcommand "admin" client "Ozone admin tool"
 
   hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" false
 }
@@ -207,6 +208,10 @@ function ozonecmd_case
       HADOOP_CLASSNAME=org.apache.hadoop.ozone.upgrade.InPlaceUpgrade
       OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-upgrade"
     ;;
+    admin)
+      HADOOP_CLASSNAME=org.apache.hadoop.ozone.admin.OzoneAdmin
+      OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
+    ;;
     *)
       HADOOP_CLASSNAME="${subcmd}"
       if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java

@@ -227,6 +227,7 @@ public static boolean isReadOnly(
     case InfoS3Bucket:
     case ListS3Buckets:
     case ServiceList:
+    case OMServerRoles:
     case ListMultiPartUploadParts:
     case GetFileStatus:
     case LookupFile:

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java

@@ -41,7 +41,8 @@
 import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
 import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
 import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
 
 import java.io.Closeable;
@@ -291,6 +292,8 @@ List<OmKeyInfo> listKeys(String volumeName,
 
   ServiceInfoEx getServiceInfo() throws IOException;
 
+  List<OMRoleInfo> getOMServerRoles() throws IOException;
+
   /*
    * S3 Specific functionality that is supported by Ozone Manager.
    */
@@ -523,8 +526,7 @@ List<OzoneFileStatus> listStatus(OmKeyArgs keyArgs, boolean recursive,
    * @return Wrapper containing the updates.
    * @throws SequenceNumberNotFoundException if db is unable to read the data.
    */
-  DBUpdatesWrapper getDBUpdates(
-      OzoneManagerProtocolProtos.DBUpdatesRequest dbUpdatesRequest)
+  DBUpdatesWrapper getDBUpdates(DBUpdatesRequest dbUpdatesRequest)
       throws IOException;
 
 }

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java

@@ -79,6 +79,8 @@
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateDirectoryRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetFileStatusResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetFileStatusRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMServerRolesResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMServerRolesRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketArgs;
@@ -120,6 +122,7 @@
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartUploadListPartsResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RemoveAclRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RemoveAclResponse;
@@ -1230,6 +1233,20 @@ public ServiceInfoEx getServiceInfo() throws IOException {
         resp.getCaCertificate());
   }
 
+  @Override
+  public List<OMRoleInfo> getOMServerRoles() throws IOException {
+    OMServerRolesRequest req = OMServerRolesRequest.newBuilder().build();
+
+    OMRequest omRequest = createOMRequest(Type.OMServerRoles)
+        .setOmServerRolesRequest(req)
+        .build();
+
+    final OMServerRolesResponse resp = handleError(submitRequest(omRequest))
+        .getOmServerRolesResponse();
+
+    return resp.getRoleInfosList();
+  }
+
   /**
    * Get a valid Delegation Token.
    *

hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto

@@ -73,6 +73,7 @@ enum Type {
   ListMultiPartUploadParts = 50;
 
   ServiceList = 51;
+  OMServerRoles = 52;
   DBUpdates = 53;
 
   GetDelegationToken = 61;
@@ -139,7 +140,8 @@ message OMRequest {
   optional MultipartUploadListPartsRequest  listMultipartUploadPartsRequest = 50;
 
   optional ServiceListRequest               serviceListRequest             = 51;
-  optional DBUpdatesRequest                  dbUpdatesRequest              = 53;
+  optional OMServerRolesRequest             omServerRolesRequest           = 52;
+  optional DBUpdatesRequest                 dbUpdatesRequest               = 53;
 
   optional hadoop.common.GetDelegationTokenRequestProto getDelegationTokenRequest = 61;
   optional hadoop.common.RenewDelegationTokenRequestProto renewDelegationTokenRequest= 62;
@@ -211,7 +213,8 @@ message OMResponse {
   optional MultipartUploadListPartsResponse listMultipartUploadPartsResponse = 50;
 
   optional ServiceListResponse               ServiceListResponse           = 51;
-  optional DBUpdatesResponse                 dbUpdatesResponse             = 52;
+  optional OMServerRolesResponse             omServerRolesResponse         = 52;
+  optional DBUpdatesResponse                 dbUpdatesResponse             = 53;
 
   optional GetDelegationTokenResponseProto getDelegationTokenResponse = 61;
   optional RenewDelegationTokenResponseProto renewDelegationTokenResponse = 62;
@@ -903,6 +906,18 @@ message ServicePort {
     required uint32 value = 2;
 }
 
+message OMServerRolesRequest {
+}
+
+message OMServerRolesResponse {
+    repeated OMRoleInfo roleInfos = 1;
+}
+
+message OMRoleInfo {
+    required string nodeId = 1;
+    required string serverRole = 2;
+}
+
 message ServiceInfo {
     required hadoop.hdds.NodeType nodeType = 1;
     required string hostname = 2;

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java

@@ -86,6 +86,7 @@
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
     .KeyArgs;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo;
 import org.apache.hadoop.ozone.protocolPB.ProtocolMessageMetrics;
 import org.apache.hadoop.ozone.security.OzoneSecurityException;
 import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
@@ -154,6 +155,7 @@
 import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
 import org.apache.hadoop.hdds.utils.db.DBStore;
 
+import org.apache.ratis.proto.RaftProtos.RaftPeerRole;
 import org.apache.ratis.server.protocol.TermIndex;
 import org.apache.ratis.util.FileUtils;
 import org.apache.ratis.util.LifeCycle;
@@ -2430,6 +2432,47 @@ public ServiceInfoEx getServiceInfo() throws IOException {
     return new ServiceInfoEx(getServiceList(), caCertPem);
   }
 
+  @Override
+  public List<OMRoleInfo> getOMServerRoles() throws IOException {
+    List<OMRoleInfo> roleInfos = new ArrayList<>();
+    if (isRatisEnabled) {
+      Map<String, String> serverRoles = omRatisServer.getServerRoles();
+
+      String nodeId = getOMNodeId();
+      String serverRole = serverRoles.get(nodeId);
+      if (serverRole == null) {
+        serverRole = RaftPeerRole.UNRECOGNIZED.name();
+      }
+      OMRoleInfo selfOMRoleInfo = OMRoleInfo.newBuilder()
+          .setNodeId(nodeId)
+          .setServerRole(serverRole)
+          .build();
+      roleInfos.add(selfOMRoleInfo);
+
+      for (OMNodeDetails peerNode : peerNodes) {
+
+        String peerNodeId = peerNode.getOMNodeId();
+        String peerServerRole = serverRoles.get(peerNodeId);
+        if (peerServerRole == null) {
+          peerServerRole = RaftPeerRole.UNRECOGNIZED.name();
+        }
+        OMRoleInfo peerOMRoleInfo = OMRoleInfo.newBuilder()
+            .setNodeId(peerNodeId)
+            .setServerRole(peerServerRole)
+            .build();
+        roleInfos.add(peerOMRoleInfo);
+      }
+    } else {
+      // There is only OM in the service. It can be assumed to be Leader.
+      OMRoleInfo selfOMRoleInfo = OMRoleInfo.newBuilder()
+          .setNodeId(getOMNodeId())
+          .setServerRole(RaftPeerRole.LEADER.name())
+          .build();
+      roleInfos.add(selfOMRoleInfo);
+    }
+    return roleInfos;
+  }
+
   @Override
   /**
    * {@inheritDoc}

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java

@@ -25,7 +25,9 @@
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.UUID;
 import java.util.concurrent.Executors;
@@ -55,6 +57,7 @@
 import org.apache.ratis.netty.NettyConfigKeys;
 import org.apache.ratis.proto.RaftProtos.RoleInfoProto;
 import org.apache.ratis.proto.RaftProtos.RaftPeerRole;
+import org.apache.ratis.proto.RaftProtos.ServerRpcProto;
 import org.apache.ratis.protocol.ClientId;
 import org.apache.ratis.protocol.GroupInfoReply;
 import org.apache.ratis.protocol.GroupInfoRequest;
@@ -617,13 +620,45 @@ private void setServerRole(RaftPeerRole currentRole,
     }
   }
 
-  private GroupInfoReply getGroupInfo() throws IOException {
+  public GroupInfoReply getGroupInfo() throws IOException {
     GroupInfoRequest groupInfoRequest = new GroupInfoRequest(clientId,
         raftPeerId, raftGroupId, nextCallId());
     GroupInfoReply groupInfo = server.getGroupInfo(groupInfoRequest);
     return groupInfo;
   }
 
+  public Map<String, String> getServerRoles() throws IOException {
+    GroupInfoReply groupInfo = getGroupInfo();
+    if (groupInfo == null) {
+      throw new IOException(ozoneManager.getOMNodeId() +
+          ": Failed to get GroupInfo.");
+    } else {
+      Map<String, String> serverRoles = new HashMap<>();
+
+      RoleInfoProto roleInfoProto = groupInfo.getRoleInfoProto();
+      String selfNodeId = new String(
+          roleInfoProto.getSelf().getId().toByteArray());
+      String selfRole = roleInfoProto.getRole().name();
+      serverRoles.put(selfNodeId, selfRole);
+
+      if (roleInfoProto.hasLeaderInfo()) {
+        for (ServerRpcProto followerInfo :
+            roleInfoProto.getLeaderInfo().getFollowerInfoList()) {
+          String nodeId = new String(
+              followerInfo.getId().getId().toByteArray());
+          serverRoles.put(nodeId, RaftPeerRole.FOLLOWER.name());
+        }
+      }
+      if (roleInfoProto.hasFollowerInfo()) {
+        String nodeId = new String(
+            roleInfoProto.getFollowerInfo().getLeaderInfo().getId().getId()
+                .toByteArray());
+        serverRoles.put(nodeId, RaftPeerRole.FOLLOWER.name());
+      }
+      return serverRoles;
+    }
+  }
+
   public int getServerPort() {
     return port;
   }

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java

@@ -310,6 +310,11 @@ public OMResponse handle(OMRequest request) {
             request.getServiceListRequest());
         responseBuilder.setServiceListResponse(serviceListResponse);
         break;
+      case OMServerRoles:
+        OMServerRolesResponse omServerRolesResponse = getOMServerRoles(
+            request.getOmServerRolesRequest());
+        responseBuilder.setOmServerRolesResponse(omServerRolesResponse);
+        break;
       case DBUpdates:
         DBUpdatesResponse dbUpdatesResponse = getOMDBUpdates(
             request.getDbUpdatesRequest());
@@ -772,6 +777,14 @@ private ServiceListResponse getServiceList(ServiceListRequest request)
     return resp.build();
   }
 
+  private OMServerRolesResponse getOMServerRoles(OMServerRolesRequest request)
+    throws IOException {
+    OMServerRolesResponse resp = OMServerRolesResponse.newBuilder()
+        .addAllRoleInfos(impl.getOMServerRoles())
+        .build();
+    return resp;
+  }
+
   private S3CreateBucketResponse createS3Bucket(S3CreateBucketRequest request)
       throws IOException {
     S3CreateBucketResponse.Builder resp = S3CreateBucketResponse.newBuilder();