From a5d9de71a5317076c0a5003fc34743e91778d156 Mon Sep 17 00:00:00 2001
From: S O'Donnell <sodonnell@cloudera.com>
Date: Fri, 10 Mar 2023 11:58:22 +0000
Subject: [PATCH 1/3] HDDS-8133. Create ozone sh key checksum command

---
 .../ozone/shell/keys/ChecksumKeyHandler.java  | 89 +++++++++++++++++++
 .../hadoop/ozone/shell/keys/KeyCommands.java  |  3 +-
 2 files changed, 91 insertions(+), 1 deletion(-)
 create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ChecksumKeyHandler.java

diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ChecksumKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ChecksumKeyHandler.java
new file mode 100644
index 000000000000..a502de3f91c2
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ChecksumKeyHandler.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.keys;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.hdds.scm.OzoneClientConfig;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import picocli.CommandLine;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.ozone.OzoneClientUtils.getFileChecksumWithCombineMode;
+
+/**
+ * Class to display checksum information about an existing key.
+ */
+@CommandLine.Command(name = "checksum",
+    description = "returns checksum information about an existing key")
+public class ChecksumKeyHandler extends KeyHandler {
+
+  @CommandLine.Option(
+      names = {"-c", "--combineMode"},
+      description = "Method of combining the chunk checksums. Valid values are "
+          + "COMPOSITE_CRC and MD5MD5CRC. Defaults to COMPOSITE_CRC.")
+  private OzoneClientConfig.ChecksumCombineMode mode =
+      OzoneClientConfig.ChecksumCombineMode.COMPOSITE_CRC;
+
+  @Override
+  protected void execute(OzoneClient client, OzoneAddress address)
+      throws IOException, OzoneClientException {
+
+    ChecksumInfo checksumInfo = new ChecksumInfo(address, client, mode);
+    printObjectAsJson(checksumInfo);
+  }
+
+  /**
+   * Wrapper class to allow checksum information to be printed as JSON.
+   */
+  @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY)
+  private static class ChecksumInfo {
+
+    private final String volumeName;
+    private final String bucketName;
+    private final String name;
+    private final long dataSize;
+    private final String algorithm;
+    private final String checksum;
+
+    ChecksumInfo(OzoneAddress address, OzoneClient client,
+        OzoneClientConfig.ChecksumCombineMode mode) throws IOException {
+      volumeName = address.getVolumeName();
+      bucketName = address.getBucketName();
+      name = address.getKeyName();
+
+      OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
+      OzoneBucket bucket = vol.getBucket(bucketName);
+      OzoneKeyDetails key = bucket.getKey(name);
+      dataSize = key.getDataSize();
+
+      FileChecksum fileChecksum = getFileChecksumWithCombineMode(vol, bucket,
+          name, dataSize, mode, client.getObjectStore().getClientProxy());
+
+      this.algorithm = fileChecksum.getAlgorithmName();
+      this.checksum = javax.xml.bind.DatatypeConverter.printHexBinary(
+          fileChecksum.getBytes());
+    }
+  }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/KeyCommands.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/KeyCommands.java
index 28af82fab0fd..726ab8be4ae2 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/KeyCommands.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/KeyCommands.java
@@ -49,7 +49,8 @@
         AddAclKeyHandler.class,
         RemoveAclKeyHandler.class,
         SetAclKeyHandler.class,
-        GetAclKeyHandler.class
+        GetAclKeyHandler.class,
+        ChecksumKeyHandler.class
     },
     mixinStandardHelpOptions = true,
     versionProvider = HddsVersionProvider.class)

From 80545ad2c6826ea946cd5d71c11312b875235d7c Mon Sep 17 00:00:00 2001
From: S O'Donnell <sodonnell@cloudera.com>
Date: Fri, 10 Mar 2023 17:37:25 +0000
Subject: [PATCH 2/3] Added tests

---
 .../ozone/shell/keys/ChecksumKeyHandler.java  |  17 ++-
 .../shell/keys/TestChecksumKeyHandler.java    | 122 ++++++++++++++++++
 .../hadoop/ozone/shell/keys/package-info.java |  21 +++
 3 files changed, 157 insertions(+), 3 deletions(-)
 create mode 100644 hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/keys/TestChecksumKeyHandler.java
 create mode 100644 hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/keys/package-info.java

diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ChecksumKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ChecksumKeyHandler.java
index a502de3f91c2..dd31536d8487 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ChecksumKeyHandler.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ChecksumKeyHandler.java
@@ -25,6 +25,7 @@
 import org.apache.hadoop.ozone.client.OzoneClientException;
 import org.apache.hadoop.ozone.client.OzoneKeyDetails;
 import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
 import org.apache.hadoop.ozone.shell.OzoneAddress;
 import picocli.CommandLine;
 
@@ -54,11 +55,21 @@ protected void execute(OzoneClient client, OzoneAddress address)
     printObjectAsJson(checksumInfo);
   }
 
+  /**
+   * Wrapper to the checksum computer to allow it to be override in tests.
+   */
+  protected FileChecksum getFileChecksum(OzoneVolume vol, OzoneBucket bucket,
+      String keyName, long dataSize, ClientProtocol clientProxy)
+      throws IOException {
+    return getFileChecksumWithCombineMode(vol, bucket, keyName, dataSize, mode,
+        clientProxy);
+  }
+
   /**
    * Wrapper class to allow checksum information to be printed as JSON.
    */
   @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY)
-  private static class ChecksumInfo {
+  private class ChecksumInfo {
 
     private final String volumeName;
     private final String bucketName;
@@ -78,8 +89,8 @@ private static class ChecksumInfo {
       OzoneKeyDetails key = bucket.getKey(name);
       dataSize = key.getDataSize();
 
-      FileChecksum fileChecksum = getFileChecksumWithCombineMode(vol, bucket,
-          name, dataSize, mode, client.getObjectStore().getClientProxy());
+      FileChecksum fileChecksum = getFileChecksum(vol, bucket,
+          name, dataSize, client.getObjectStore().getClientProxy());
 
       this.algorithm = fileChecksum.getAlgorithmName();
       this.checksum = javax.xml.bind.DatatypeConverter.printHexBinary(
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/keys/TestChecksumKeyHandler.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/keys/TestChecksumKeyHandler.java
new file mode 100644
index 000000000000..1279adcc8a0f
--- /dev/null
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/keys/TestChecksumKeyHandler.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.keys;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hadoop.fs.CompositeCrcFileChecksum;
+import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.util.CrcUtil;
+import org.apache.hadoop.util.DataChecksum;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
+
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Tests for ChecksumKeyHandler.
+ */
+public class TestChecksumKeyHandler {
+
+  private ChecksumKeyHandler cmd;
+  private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+  private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
+  private final PrintStream originalOut = System.out;
+  private final PrintStream originalErr = System.err;
+  private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name();
+  private static final int CHECKSUM = 123456;
+
+  @BeforeEach
+  public void setup() throws UnsupportedEncodingException {
+
+    cmd = new ChecksumKeyHandler() {
+      // Just return a known checksum for testing, as it reduces the amount of
+      // mocking needed.
+      @Override
+      protected FileChecksum getFileChecksum(OzoneVolume vol,
+          OzoneBucket bucket, String keyName, long dataSize,
+          ClientProtocol clientProxy) {
+        return new CompositeCrcFileChecksum(
+            CHECKSUM, DataChecksum.Type.CRC32, 512);
+      }
+    };
+    System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING));
+    System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING));
+  }
+
+  @AfterEach
+  public void tearDown() {
+    System.setOut(originalOut);
+    System.setErr(originalErr);
+  }
+
+  @Test
+  public void testChecksumKeyHandler()
+      throws OzoneClientException, IOException {
+    OzoneAddress address = new OzoneAddress("o3://ozone1/volume/bucket/key");
+    long keySize = 1024L;
+
+    ObjectStore objectStore = mock(ObjectStore.class);
+    OzoneClient client = mock(OzoneClient.class);
+    Mockito.when(client.getObjectStore()).thenReturn(objectStore);
+
+    OzoneVolume volume = mock(OzoneVolume.class);
+    OzoneBucket bucket = mock(OzoneBucket.class);
+    OzoneKeyDetails key = mock(OzoneKeyDetails.class);
+
+    Mockito.when(volume.getBucket(anyString())).thenReturn(bucket);
+    Mockito.when(bucket.getKey(anyString()))
+        .thenReturn(key);
+    Mockito.when(objectStore.getVolume(anyString())).
+        thenReturn(volume);
+    Mockito.when(key.getDataSize()).thenReturn(keySize);
+
+    cmd.execute(client, address);
+
+    ObjectMapper mapper = new ObjectMapper();
+    JsonNode json = mapper.readTree(outContent.toString("UTF-8"));
+    Assertions.assertEquals("volume", json.get("volumeName").asText());
+    Assertions.assertEquals("bucket", json.get("bucketName").asText());
+    Assertions.assertEquals("key", json.get("name").asText());
+    Assertions.assertEquals(keySize, json.get("dataSize").asLong());
+    Assertions.assertEquals("COMPOSITE-CRC32", json.get("algorithm").asText());
+
+    String expectedChecksum = javax.xml.bind.DatatypeConverter.printHexBinary(
+        CrcUtil.intToBytes(Integer.valueOf(CHECKSUM)));
+    Assertions.assertEquals(expectedChecksum, json.get("checksum").asText());
+  }
+
+}
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/keys/package-info.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/keys/package-info.java
new file mode 100644
index 000000000000..40f30232bf4d
--- /dev/null
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/keys/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.keys;
+/**
+ * Tests for ozone shell key commands.
+ */

From 3bff5eec92e9f3d9990e3f9665a6ec592b745672 Mon Sep 17 00:00:00 2001
From: Stephen O'Donnell <stephen.odonnell@gmail.com>
Date: Sat, 11 Mar 2023 16:39:05 +0000
Subject: [PATCH 3/3] Update
 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ChecksumKeyHandler.java

Co-authored-by: Doroszlai, Attila <6454655+adoroszlai@users.noreply.github.com>
---
 .../org/apache/hadoop/ozone/shell/keys/ChecksumKeyHandler.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ChecksumKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ChecksumKeyHandler.java
index dd31536d8487..b8acf783d6f6 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ChecksumKeyHandler.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ChecksumKeyHandler.java
@@ -41,7 +41,7 @@
 public class ChecksumKeyHandler extends KeyHandler {
 
   @CommandLine.Option(
-      names = {"-c", "--combineMode"},
+      names = {"-c", "--combine-mode"},
       description = "Method of combining the chunk checksums. Valid values are "
           + "COMPOSITE_CRC and MD5MD5CRC. Defaults to COMPOSITE_CRC.")
   private OzoneClientConfig.ChecksumCombineMode mode =