Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@

import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.apache.hadoop.security.KerberosInfo;
Expand All @@ -58,6 +59,13 @@
@KerberosInfo(serverPrincipal = OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY)
public interface ClientProtocol {

/**
* List of OM node Ids and their Ratis server roles.
* @return List of OM server roles
* @throws IOException
*/
List<OMRoleInfo> getOmRoleInfos() throws IOException;

/**
* Creates a new Volume.
* @param volumeName Name of the Volume
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@
import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.om.protocolPB
Expand All @@ -72,6 +73,7 @@
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo;
import org.apache.hadoop.ozone.security.GDPRSymmetricKey;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType;
Expand Down Expand Up @@ -223,6 +225,23 @@ public RpcClient(Configuration conf, String omServiceId) throws IOException {
OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_DEFAULT);
}

@Override
public List<OMRoleInfo> getOmRoleInfos() throws IOException {

List<ServiceInfo> serviceList = ozoneManagerClient.getServiceList();
List<OMRoleInfo> roleInfos = new ArrayList<>();

for (ServiceInfo serviceInfo : serviceList) {
if (serviceInfo.getNodeType().equals(HddsProtos.NodeType.OM)) {
OMRoleInfo omRoleInfo = serviceInfo.getOmRoleInfo();
if (omRoleInfo != null) {
roleInfos.add(omRoleInfo);
}
}
}
return roleInfos;
}

@Override
public void createVolume(String volumeName) throws IOException {
createVolume(volumeName, VolumeArgs.newBuilder().build());
Expand Down
5 changes: 5 additions & 0 deletions hadoop-ozone/common/src/main/bin/ozone
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ function hadoop_usage
hadoop_add_subcommand "version" client "print the version"
hadoop_add_subcommand "dtutil" client "operations related to delegation tokens"
hadoop_add_subcommand "upgrade" client "HDFS to Ozone in-place upgrade tool"
hadoop_add_subcommand "admin" client "Ozone admin tool"

hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" false
}
Expand Down Expand Up @@ -207,6 +208,10 @@ function ozonecmd_case
HADOOP_CLASSNAME=org.apache.hadoop.ozone.upgrade.InPlaceUpgrade
OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-upgrade"
;;
admin)
HADOOP_CLASSNAME=org.apache.hadoop.ozone.admin.OzoneAdmin
OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
;;
*)
HADOOP_CLASSNAME="${subcmd}"
if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.base.Preconditions;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.OMRoleInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.ServicePort;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
Expand Down Expand Up @@ -58,6 +60,8 @@ public final class ServiceInfo {
* List of ports the service listens to.
*/
private Map<ServicePort.Type, Integer> ports;

private OMRoleInfo omRoleInfo;

/**
* Default constructor for JSON deserialization.
Expand All @@ -70,8 +74,8 @@ public ServiceInfo() {}
* @param hostname hostname of the service
* @param portList list of ports the service listens to
*/
private ServiceInfo(
NodeType nodeType, String hostname, List<ServicePort> portList) {
private ServiceInfo(NodeType nodeType, String hostname,
List<ServicePort> portList, OMRoleInfo omRole) {
Preconditions.checkNotNull(nodeType);
Preconditions.checkNotNull(hostname);
this.nodeType = nodeType;
Expand All @@ -80,6 +84,7 @@ private ServiceInfo(
for (ServicePort port : portList) {
ports.put(port.getType(), port.getValue());
}
this.omRoleInfo = omRole;
}

/**
Expand Down Expand Up @@ -128,6 +133,15 @@ public String getServiceAddress(ServicePort.Type portType) {
return hostname + ":" + getPort(portType);
}

/**
* Returns the OM role info - node id and ratis server role.
* @return OmRoleInfo
*/
@JsonIgnore
public OMRoleInfo getOmRoleInfo() {
return omRoleInfo;
}

/**
* Converts {@link ServiceInfo} to OzoneManagerProtocolProtos.ServiceInfo.
*
Expand All @@ -147,6 +161,9 @@ public OzoneManagerProtocolProtos.ServiceInfo getProtobuf() {
.setType(entry.getKey())
.setValue(entry.getValue()).build())
.collect(Collectors.toList()));
if (nodeType == NodeType.OM && omRoleInfo != null) {
builder.setOmRole(omRoleInfo);
}
return builder.build();
}

Expand All @@ -160,7 +177,8 @@ public static ServiceInfo getFromProtobuf(
OzoneManagerProtocolProtos.ServiceInfo serviceInfo) {
return new ServiceInfo(serviceInfo.getNodeType(),
serviceInfo.getHostname(),
serviceInfo.getServicePortsList());
serviceInfo.getServicePortsList(),
serviceInfo.hasOmRole() ? serviceInfo.getOmRole() : null);
}

/**
Expand All @@ -179,7 +197,7 @@ public static class Builder {
private NodeType node;
private String host;
private List<ServicePort> portList = new ArrayList<>();

private OMRoleInfo omRoleInfo;

/**
* Sets the node/service type.
Expand Down Expand Up @@ -211,13 +229,17 @@ public Builder addServicePort(ServicePort servicePort) {
return this;
}

public Builder setOmRoleInfo(OMRoleInfo omRole) {
omRoleInfo = omRole;
return this;
}

/**
* Builds and returns {@link ServiceInfo} with the set values.
* @return {@link ServiceInfo}
*/
public ServiceInfo build() {
return new ServiceInfo(node, host, portList);
return new ServiceInfo(node, host, portList, omRoleInfo);
}
}

Expand Down
6 changes: 6 additions & 0 deletions hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
Original file line number Diff line number Diff line change
Expand Up @@ -903,10 +903,16 @@ message ServicePort {
required uint32 value = 2;
}

message OMRoleInfo {
required string nodeId = 1;
required string serverRole = 2;
}

message ServiceInfo {
required hadoop.hdds.NodeType nodeType = 1;
required string hostname = 2;
repeated ServicePort servicePorts = 3;
optional OMRoleInfo omRole = 4;
}

message S3CreateBucketRequest {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.KeyArgs;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo;
import org.apache.hadoop.ozone.protocolPB.ProtocolMessageMetrics;
import org.apache.hadoop.ozone.security.OzoneSecurityException;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
Expand Down Expand Up @@ -154,6 +155,7 @@
import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
import org.apache.hadoop.hdds.utils.db.DBStore;

import org.apache.ratis.proto.RaftProtos.RaftPeerRole;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.util.FileUtils;
import org.apache.ratis.util.LifeCycle;
Expand Down Expand Up @@ -2384,6 +2386,43 @@ public List<ServiceInfo> getServiceList() throws IOException {
.setValue(httpServer.getHttpsAddress().getPort())
.build());
}

// Since this OM is processing the request, we can assume it to be the
// leader OM

OMRoleInfo omRole = OMRoleInfo.newBuilder()
.setNodeId(getOMNodeId())
.setServerRole(RaftPeerRole.LEADER.name())
.build();
omServiceInfoBuilder.setOmRoleInfo(omRole);

if (isRatisEnabled) {
if (omRatisServer != null) {
omServiceInfoBuilder.addServicePort(ServicePort.newBuilder()
.setType(ServicePort.Type.RATIS)
.setValue(omNodeDetails.getRatisPort())
.build());
}

for (OMNodeDetails peerNode : peerNodes) {
ServiceInfo.Builder peerOmServiceInfoBuilder = ServiceInfo.newBuilder()
.setNodeType(HddsProtos.NodeType.OM)
.setHostname(peerNode.getAddress().getHostName())
.addServicePort(ServicePort.newBuilder()
.setType(ServicePort.Type.RPC)
.setValue(peerNode.getRpcPort())
.build());

OMRoleInfo peerOmRole = OMRoleInfo.newBuilder()
.setNodeId(peerNode.getOMNodeId())
.setServerRole(RaftPeerRole.FOLLOWER.name())
.build();
peerOmServiceInfoBuilder.setOmRoleInfo(peerOmRole);

services.add(peerOmServiceInfoBuilder.build());
}
}

services.add(omServiceInfoBuilder.build());

// For client we have to return SCM with container protocol port,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.admin;

import org.apache.hadoop.hdds.cli.GenericCli;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.admin.om.OMAdmin;
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.log4j.PatternLayout;
import picocli.CommandLine;

/**
* Ozone Admin Command line tool.
*/
@CommandLine.Command(name = "ozone admin",
hidden = true,
description = "Developer tools for Ozone Admin operations",
versionProvider = HddsVersionProvider.class,
subcommands = {
OMAdmin.class
},
mixinStandardHelpOptions = true)
public class OzoneAdmin extends GenericCli {
private OzoneConfiguration ozoneConf;

public OzoneConfiguration getOzoneConf() {
if (ozoneConf == null) {
ozoneConf = createOzoneConfiguration();
}
return ozoneConf;
}

/**
* Main for the Ozone Admin shell Command handling.
*
* @param argv - System Args Strings[]
* @throws Exception
*/
public static void main(String[] argv) throws Exception {
LogManager.resetConfiguration();
Logger.getRootLogger().setLevel(Level.WARN);
Logger.getRootLogger()
.addAppender(new ConsoleAppender(new PatternLayout("%m%n")));
Logger.getLogger(NativeCodeLoader.class).setLevel(Level.ERROR);

new OzoneAdmin().run(argv);
}
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

New line missing, for newly added files.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry did not understand this.

Copy link
Contributor

@dineshchitlangia dineshchitlangia Oct 29, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From the latest CI run, this doesn't seem to be an issue.
@hanishakoneru He meant that one of the checkstyle rule is that any source file must have an empty new line as its last line.

Copy link
Contributor

@bharatviswa504 bharatviswa504 Oct 29, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. Newline at end of the file. I think there is no checkstyle rule for this.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@bharatviswa504 I just downloaded the patch and confirmed the new line at EOF is there. So no changes are needed.

Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.hadoop.ozone.admin.om;

import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo;
import picocli.CommandLine;

import java.util.List;
import java.util.concurrent.Callable;

/**
* Handler of om get-service-roles command.
*/
@CommandLine.Command(
name = "getserviceroles",
description = "List all OMs and their respective Ratis server roles",
mixinStandardHelpOptions = true,
versionProvider = HddsVersionProvider.class)
public class GetServiceRolesSubcommand implements Callable<Void> {

@CommandLine.ParentCommand
private OMAdmin parent;

@CommandLine.Option(names = {"-id", "--service-id"},
description = "OM Service ID",
required = true)
private String omServiceId;

@Override
public Void call() throws Exception {
ClientProtocol client = parent.createClient(omServiceId);
getOmServerRoles(client.getOmRoleInfos());
return null;
}

private void getOmServerRoles(List<OMRoleInfo> roleInfos) {
for (OMRoleInfo roleInfo : roleInfos) {
System.out.println(
roleInfo.getNodeId() + " : " + roleInfo.getServerRole());
}
}
}
Loading