diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmInfo.java index 6236febb7b12..b9d823e8d817 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmInfo.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmInfo.java @@ -18,6 +18,9 @@ package org.apache.hadoop.hdds.scm; +import java.util.ArrayList; +import java.util.List; + /** * ScmInfo wraps the result returned from SCM#getScmInfo which * contains clusterId and the SCM Id. @@ -25,6 +28,7 @@ public final class ScmInfo { private String clusterId; private String scmId; + private List peerRoles; /** * Builder for ScmInfo. @@ -32,6 +36,11 @@ public final class ScmInfo { public static class Builder { private String clusterId; private String scmId; + private List peerRoles; + + public Builder() { + peerRoles = new ArrayList<>(); + } /** * sets the cluster id. @@ -53,14 +62,25 @@ public Builder setScmId(String id) { return this; } + /** + * Set peer address in Scm HA. + * @param roles ratis peer address in the format of [ip|hostname]:port + * @return Builder for scmInfo + */ + public Builder setRatisPeerRoles(List roles) { + peerRoles.addAll(roles); + return this; + } + public ScmInfo build() { - return new ScmInfo(clusterId, scmId); + return new ScmInfo(clusterId, scmId, peerRoles); } } - private ScmInfo(String clusterId, String scmId) { + private ScmInfo(String clusterId, String scmId, List peerRoles) { this.clusterId = clusterId; this.scmId = scmId; + this.peerRoles = peerRoles; } /** @@ -78,4 +98,12 @@ public String getClusterId() { public String getScmId() { return scmId; } + + /** + * Gets the list of peer roles (currently address) in Scm HA. + * @return List of peer address + */ + public List getRatisPeerRoles() { + return peerRoles; + } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java index e4369fa86272..7c3c94cb7ae1 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java @@ -246,5 +246,8 @@ Map> getSafeModeRuleStatuses() */ boolean getReplicationManagerStatus() throws IOException; - + /** + * returns the list of ratis peer roles. Currently only include peer address. + */ + List getScmRatisRoles() throws IOException; } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java index 0733940deb2c..cf888697313b 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java @@ -444,7 +444,9 @@ public ScmInfo getScmInfo() throws IOException { .getGetScmInfoResponse(); ScmInfo.Builder builder = new ScmInfo.Builder() .setClusterId(resp.getClusterId()) - .setScmId(resp.getScmId()); + .setScmId(resp.getScmId()) + .setRatisPeerRoles(resp.getPeerRolesList()); + return builder.build(); } diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto index de439079b713..acd2068aa26d 100644 --- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto +++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto @@ -201,6 +201,7 @@ message GetScmInfoRequestProto { message GetScmInfoResponseProto { required string clusterId = 1; required string scmId = 2; + repeated string peerRoles = 3; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManager.java index ade0ad965768..dc68b413536a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManager.java @@ -17,6 +17,7 @@ package org.apache.hadoop.hdds.scm.ha; +import java.util.List; import org.apache.ratis.protocol.NotLeaderException; import org.apache.ratis.protocol.RaftPeer; @@ -52,6 +53,11 @@ public interface SCMHAManager { */ void shutdown() throws IOException; + /** + * Returns roles of ratis peers. + */ + List getRatisRoles(); + /** * Returns NotLeaderException with useful info. */ diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java index 8bb94578b3b1..e2aa04f2f4e8 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hdds.scm.ha; import com.google.common.base.Preconditions; +import java.util.List; +import java.util.stream.Collectors; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.ratis.protocol.NotLeaderException; import org.apache.ratis.protocol.RaftGroupMemberId; @@ -145,6 +147,15 @@ public void shutdown() throws IOException { ratisServer.stop(); } + @Override + public List getRatisRoles() { + return getRatisServer() + .getRaftPeers() + .stream() + .map(peer -> peer.getAddress() == null ? "" : peer.getAddress()) + .collect(Collectors.toList()); + } + /** * {@inheritDoc} */ diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java index d07cc5635c99..198cd1bf2fff 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java @@ -441,8 +441,8 @@ public HddsProtos.GetScmInfoResponseProto getScmInfo( return HddsProtos.GetScmInfoResponseProto.newBuilder() .setClusterId(scmInfo.getClusterId()) .setScmId(scmInfo.getScmId()) + .addAllPeerRoles(scmInfo.getRatisPeerRoles()) .build(); - } public InSafeModeResponseProto inSafeMode( diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 594527a22e9a..3ad31d7913b3 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -488,7 +488,8 @@ public ScmInfo getScmInfo() throws IOException { ScmInfo.Builder builder = new ScmInfo.Builder() .setClusterId(scm.getScmStorageConfig().getClusterID()) - .setScmId(scm.getScmStorageConfig().getScmId()); + .setScmId(scm.getScmStorageConfig().getScmId()) + .setRatisPeerRoles(scm.getScmHAManager().getRatisRoles()); return builder.build(); } catch (Exception ex) { auditSuccess = false; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index f54e15cb3e64..768ca09f6b21 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -1168,4 +1168,8 @@ public Map getRuleStatusMetrics() { } return map; } + + public SCMHAManager getScmHAManager() { + return scmHAManager; + } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/MockSCMHAManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/MockSCMHAManager.java index ce48c1136a9f..e31e7e16bc5e 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/MockSCMHAManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/MockSCMHAManager.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; +import java.util.Arrays; import java.util.EnumMap; import java.util.List; import java.util.Map; @@ -107,6 +108,14 @@ public void shutdown() throws IOException { ratisServer.stop(); } + @Override + public List getRatisRoles() { + return Arrays.asList( + "180.3.14.5:9865", + "180.3.14.21:9865", + "180.3.14.145:9865"); + } + /** * {@inheritDoc} */ diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java index 96cd5307491d..038364227f1a 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java @@ -508,4 +508,8 @@ public boolean getReplicationManagerStatus() throws IOException { return storageContainerLocationClient.getReplicationManagerStatus(); } + @Override + public List getScmRatisRoles() throws IOException { + return storageContainerLocationClient.getScmInfo().getRatisPeerRoles(); + } } diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/scmha.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/scmha.robot new file mode 100644 index 000000000000..31a990f857d7 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/scmha.robot @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Smoketest ozone cluster startup +Library OperatingSystem +Library BuiltIn +Resource ../commonlib.robot +Test Timeout 5 minutes + +*** Variables *** + +*** Test Cases *** +Run scm roles + ${output} = Execute ozone admin scm roles + Should contain ${output} [] diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestScmAdminHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestScmAdminHA.java new file mode 100644 index 000000000000..63a8e7186c79 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestScmAdminHA.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.shell; + +import java.net.InetSocketAddress; +import java.util.UUID; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.admin.OzoneAdmin; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * This class tests ozone admin scm commands. + */ +public class TestScmAdminHA { + private static OzoneAdmin ozoneAdmin; + private static OzoneConfiguration conf; + private static String omServiceId; + private static int numOfOMs; + private static String clusterId; + private static String scmId; + private static MiniOzoneCluster cluster; + + @BeforeClass + public static void init() throws Exception { + ozoneAdmin = new OzoneAdmin(); + conf = new OzoneConfiguration(); + + // Init HA cluster + omServiceId = "om-service-test1"; + numOfOMs = 3; + clusterId = UUID.randomUUID().toString(); + scmId = UUID.randomUUID().toString(); + cluster = MiniOzoneCluster.newHABuilder(conf) + .setClusterId(clusterId) + .setScmId(scmId) + .setOMServiceId(omServiceId) + .setNumOfOzoneManagers(numOfOMs) + .build(); + conf.setQuietMode(false); + // enable ratis for Scm. + conf.setBoolean(ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY, true); + cluster.waitForClusterToBeReady(); + } + + @AfterClass + public static void shutdown() { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test + public void testGetRatisRoles() { + InetSocketAddress address = + cluster.getStorageContainerManager().getClientRpcAddress(); + String hostPort = address.getHostName() + ":" + address.getPort(); + String[] args = {"--scm", hostPort, "scm", "roles"}; + ozoneAdmin.execute(args); + } +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/OzoneAdmin.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/OzoneAdmin.java index 81f8f649b694..76ba42fea0ea 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/OzoneAdmin.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/OzoneAdmin.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hdds.scm.cli.pipeline.PipelineCommands; import org.apache.hadoop.hdds.scm.client.ScmClient; import org.apache.hadoop.ozone.admin.om.OMAdmin; +import org.apache.hadoop.ozone.admin.scm.ScmAdmin; import org.apache.hadoop.util.NativeCodeLoader; import org.apache.commons.lang3.StringUtils; @@ -56,6 +57,7 @@ versionProvider = HddsVersionProvider.class, subcommands = { OMAdmin.class, + ScmAdmin.class, SafeModeCommands.class, ContainerCommands.class, PipelineCommands.class, diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/GetScmRatisRolesSubcommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/GetScmRatisRolesSubcommand.java new file mode 100644 index 000000000000..cf2310c3cd5c --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/GetScmRatisRolesSubcommand.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.admin.scm; + +import java.util.List; +import java.util.concurrent.Callable; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.scm.client.ScmClient; +import picocli.CommandLine; + +/** + * Handler of scm status command. + */ +@CommandLine.Command( + name = "roles", + description = "List all SCMs and their respective Ratis server roles", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class) +public class GetScmRatisRolesSubcommand implements Callable { + + @CommandLine.ParentCommand + private ScmAdmin parent; + + @Override + public Void call() throws Exception { + ScmClient scmClient = parent.createScmClient(); + List roles = scmClient.getScmRatisRoles(); + System.out.println(roles); + return null; + } +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/ScmAdmin.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/ScmAdmin.java new file mode 100644 index 000000000000..2605a6d3b950 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/ScmAdmin.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.admin.scm; + +import org.apache.hadoop.hdds.cli.GenericCli; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.scm.client.ScmClient; +import org.apache.hadoop.ozone.admin.OzoneAdmin; +import picocli.CommandLine; +import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.Spec; + +/** + * Subcommand for admin operations related to SCM. + */ +@CommandLine.Command( + name = "scm", + description = "Ozone Storage Container Manager specific admin operations", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class, + subcommands = { + GetScmRatisRolesSubcommand.class + }) +public class ScmAdmin extends GenericCli { + + @CommandLine.ParentCommand + private OzoneAdmin parent; + + @Spec + private CommandSpec spec; + + public OzoneAdmin getParent() { + return parent; + } + + @Override + public Void call() throws Exception { + GenericCli.missingSubcommand(spec); + return null; + } + + public ScmClient createScmClient() { + return parent.createScmClient(); + } +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/package-info.java new file mode 100644 index 000000000000..ec15a3320069 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/package-info.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * SCM related Admin tools. + */ +package org.apache.hadoop.ozone.admin.scm;