HDDS-10479. Support Ozone to run ratis local related commands

hadoop-ozone/dist/src/main/license/bin/LICENSE.txt

@@ -407,6 +407,7 @@ Apache License 2.0
    org.apache.ratis:ratis-proto
    org.apache.ratis:ratis-server
    org.apache.ratis:ratis-server-api
+   org.apache.ratis:ratis-shell
    org.apache.ratis:ratis-thirdparty-misc
    org.apache.ratis:ratis-tools
    org.apache.thrift:libthrift
@@ -458,6 +459,7 @@ MIT
    org.kohsuke.metainf-services:metainf-services
    org.slf4j:slf4j-api
    org.slf4j:slf4j-reload4j
+   org.slf4j:slf4j-simple
 
 
 Public Domain

hadoop-ozone/dist/src/main/license/jar-report.txt

@@ -252,6 +252,7 @@ share/ozone/lib/ratis-netty.jar
 share/ozone/lib/ratis-proto.jar
 share/ozone/lib/ratis-server-api.jar
 share/ozone/lib/ratis-server.jar
+share/ozone/lib/ratis-shell.jar
 share/ozone/lib/ratis-thirdparty-misc.jar
 share/ozone/lib/ratis-tools.jar
 share/ozone/lib/re2j.jar
@@ -264,6 +265,7 @@ share/ozone/lib/simpleclient_dropwizard.jar
 share/ozone/lib/simpleclient.jar
 share/ozone/lib/slf4j-api.jar
 share/ozone/lib/slf4j-reload4j.jar
+share/ozone/lib/slf4j-simple.jar
 share/ozone/lib/snakeyaml.jar
 share/ozone/lib/snappy-java.jar
 share/ozone/lib/spring-beans.jar

hadoop-ozone/dist/src/shell/ozone/ozone

@@ -61,6 +61,7 @@ function ozone_usage
   ozone_add_subcommand "debug" client "Ozone debug tool"
   ozone_add_subcommand "repair" client "Ozone repair tool"
   ozone_add_subcommand "checknative" client "checks if native libraries are loaded"
+  ozone_add_subcommand "ratis" client "Ozone ratis tool"
 
   ozone_generate_usage "${OZONE_SHELL_EXECNAME}" false
 }
@@ -231,6 +232,10 @@ function ozonecmd_case
         OZONE_CLASSNAME=org.apache.hadoop.ozone.shell.checknative.CheckNative
         OZONE_RUN_ARTIFACT_NAME="ozone-tools"
     ;;
+    ratis)
+      OZONE_CLASSNAME=org.apache.hadoop.ozone.ratis.OzoneRatis
+      OZONE_RUN_ARTIFACT_NAME="ozone-tools"
+    ;;
     *)
       OZONE_CLASSNAME="${subcmd}"
       if ! ozone_validate_classname "${OZONE_CLASSNAME}"; then

hadoop-ozone/tools/pom.xml

@@ -160,6 +160,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <groupId>org.apache.ratis</groupId>
       <artifactId>ratis-tools</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.ratis</groupId>
+      <artifactId>ratis-shell</artifactId>
+    </dependency>
 
     <dependency>
       <groupId>info.picocli</groupId>

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneRatis.java

@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell;
+
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.tracing.TracingUtil;
+import org.apache.ratis.shell.cli.sh.RatisShell;
+
+import picocli.CommandLine;
+
+/**
+ * Ozone Ratis Command line tool.
+ */
+@CommandLine.Command(name = "ozone ratis",
+        description = "Shell for running Ratis commands",
+        versionProvider = HddsVersionProvider.class,
+        mixinStandardHelpOptions = true)
+public class OzoneRatis extends Shell {
+
+  public OzoneRatis() {
+    super(OzoneRatis.class);
+  }
+
+  /**
+   * Main for the OzoneRatis Command handling.
+   *
+   * @param argv - System Args Strings[]
+   */
+  public static void main(String[] argv) throws Exception {
+    new OzoneRatis().run(argv);
+  }
+
+  @Override
+  public int execute(String[] argv) {
+    TracingUtil.initTracing("shell", createOzoneConfiguration());
+    String spanName = "ozone ratis" + String.join(" ", argv);
+    return TracingUtil.executeInNewSpan(spanName, () -> {
+      // TODO: When Ozone has RATIS-2155, update this line to use the RatisShell.Builder
+      //       in order to setup TLS and other confs.
+      final RatisShell shell = new RatisShell(System.out);
+      return shell.run(argv);
+    });
+  }
+}

hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneRatis.java

@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell;
+
+import org.apache.ratis.proto.RaftProtos;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Tests for OzoneRatis.
+ */
+public class TestOzoneRatis {
+  private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name();
+  private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+  private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
+  private final PrintStream originalOut = System.out;
+  private final PrintStream originalErr = System.err;
+  private OzoneRatis ozoneRatis;
+
+  @BeforeEach
+  public void setUp() throws UnsupportedEncodingException {
+    System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING));
+    System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING));
+    ozoneRatis = new OzoneRatis();
+  }
+
+  @AfterEach
+  public void tearDown() {
+    System.setOut(originalOut);
+    System.setErr(originalErr);
+  }
+
+  /**
+   * Execute method to invoke the OzoneRatis class and capture output.
+   *
+   * @param args command line arguments to pass
+   * @return the output from OzoneRatis
+   */
+  private String execute(String[] args) throws IOException {
+    ozoneRatis.execute(args);
+    return outContent.toString(StandardCharsets.UTF_8.name());
+  }
+
+  @Test
+  public void testBasicOzoneRatisCommand() throws IOException {
+    String[] args = {""};
+    String output = execute(args);
+    assertTrue(output.contains("Usage: ratis sh [generic options]"));
+  }
+
+  @Test
+  public void testLocalRaftMetaConfSubcommand(@TempDir Path tempDir) throws IOException {
+    // Set up temporary directory and files
+    Path metadataDir = tempDir.resolve("data/metadata/ratis/test-cluster/current/");
+    Files.createDirectories(metadataDir);
+
+    // Create a dummy raft-meta.conf file using protobuf
+    Path raftMetaConfFile = metadataDir.resolve("raft-meta.conf");
+
+    // Create a LogEntryProto with a dummy index and peer
+    RaftProtos.RaftPeerProto raftPeerProto = RaftProtos.RaftPeerProto.newBuilder()
+            .setId(ByteString.copyFromUtf8("peer1"))
+            .setAddress("localhost:8000")
+            .setStartupRole(RaftProtos.RaftPeerRole.FOLLOWER)
+            .build();
+
+    RaftProtos.LogEntryProto logEntryProto = RaftProtos.LogEntryProto.newBuilder()
+            .setConfigurationEntry(RaftProtos.RaftConfigurationProto.newBuilder()
+                    .addPeers(raftPeerProto).build())
+            .setIndex(0)
+            .build();
+
+    // Write the logEntryProto to the raft-meta.conf file
+    try (OutputStream out = Files.newOutputStream(raftMetaConfFile)) {
+      logEntryProto.writeTo(out);
+    }
+
+
+    String[] args = {"local", "raftMetaConf", "-peers", "peer1|localhost:8080", "-path", metadataDir.toString()};
+    String output = execute(args);
+
+    assertTrue(output.contains("Index in the original file is: 0"));
+    assertTrue(output.contains("Generate new LogEntryProto info is:"));
+
+    // Verify that the new raft-meta.conf is generated
+    Path newRaftMetaConfFile = metadataDir.resolve("new-raft-meta.conf");
+    assertTrue(Files.exists(newRaftMetaConfFile), "New raft-meta.conf file should be created.");
+
+    // Verify content of the newly generated file
+    try (InputStream in = Files.newInputStream(newRaftMetaConfFile)) {
+      RaftProtos.LogEntryProto newLogEntryProto = RaftProtos.LogEntryProto.parseFrom(in);
+      assertEquals(1, newLogEntryProto.getIndex());
+      RaftProtos.RaftPeerProto peerProto = newLogEntryProto.getConfigurationEntry().getPeers(0);
+      assertEquals("peer1", peerProto.getId().toStringUtf8());
+      assertEquals("localhost:8080", peerProto.getAddress());
+      assertEquals(RaftProtos.RaftPeerRole.FOLLOWER, peerProto.getStartupRole());
+    }
+  }
+
+  @Test
+  public void testMissingRequiredArguments() throws IOException {
+    String[] args = {"local", "raftMetaConf"};
+    String output = execute(args);
+    assertTrue(output.contains("Failed to parse args for raftMetaConf: Missing required options: peers, path"));
+  }
+
+  @Test
+  public void testMissingPeerArgument() throws IOException {
+    String[] args = {"local", "raftMetaConf", "-path", "/path"};
+    String output = execute(args);
+    assertTrue(output.contains("Failed to parse args for raftMetaConf: Missing required option: peers"));
+  }
+
+  @Test
+  public void testMissingPathArgument() throws IOException {
+    String[] args = {"local", "raftMetaConf", "-peers", "localhost:8080"};
+    String output = execute(args);
+    assertTrue(output.contains("Failed to parse args for raftMetaConf: Missing required option: path"));
+  }
+
+  @Test
+  public void testInvalidPeersFormat() throws IOException {
+    String[] args = {"local", "raftMetaConf", "-peers", "localhost8080", "-path", "/path"};
+    String output = execute(args);
+    assertTrue(output.contains("Failed to parse the server address parameter \"localhost8080\"."));
+  }
+
+  @Test
+  public void testDuplicatePeersAddress() throws IOException {
+    String[] args = {"local", "raftMetaConf", "-peers", "localhost:8080,localhost:8080", "-path", "/path"};
+    String output = execute(args);
+    assertTrue(output.contains("Found duplicated address: localhost:8080."));
+  }
+
+  @Test
+  public void testDuplicatePeersId() throws IOException {
+    String[] args = {"local", "raftMetaConf", "-peers", "peer1|localhost:8080,peer1|localhost:8081", "-path", "/path"};
+    String output = execute(args);
+    assertTrue(output.contains("Found duplicated ID: peer1."));
+  }
+}

pom.xml

@@ -758,6 +758,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <artifactId>ratis-common</artifactId>
         <version>${ratis.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.ratis</groupId>
+        <artifactId>ratis-shell</artifactId>
+        <version>${ratis.version}</version>
+      </dependency>
 
       <dependency>
         <groupId>io.netty</groupId>