HDDS-4808. Add Genesis benchmark for various CRC implementations

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferImpl.java

@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.common;
+
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
+import java.nio.ByteBuffer;
+import java.util.zip.Checksum;
+
+public class ChecksumByteBufferImpl implements ChecksumByteBuffer {
+
+  public static class Java9Crc32CFactory {
+    private static final MethodHandle NEW_CRC32C_MH;
+
+    static {
+      MethodHandle newCRC32C = null;
+      try {
+        newCRC32C = MethodHandles.publicLookup()
+            .findConstructor(
+                Class.forName("java.util.zip.CRC32C"),
+                MethodType.methodType(void.class)
+            );
+      } catch (ReflectiveOperationException e) {
+        // Should not reach here.
+        throw new RuntimeException(e);
+      }
+      NEW_CRC32C_MH = newCRC32C;
+    }
+
+    public static java.util.zip.Checksum createChecksum() {
+      try {
+        // Should throw nothing
+        return (Checksum) NEW_CRC32C_MH.invoke();
+      } catch (Throwable t) {
+        throw (t instanceof RuntimeException) ? (RuntimeException) t
+            : new RuntimeException(t);
+      }
+    }
+  };
+
+  private Checksum checksum;
+
+  public ChecksumByteBufferImpl(Checksum impl) {
+    this.checksum = impl;
+  }
+
+  @Override
+  public void update(ByteBuffer buffer) {
+    if (buffer.hasArray()) {
+      checksum.update(buffer.array(), buffer.position() + buffer.arrayOffset(),
+          buffer.remaining());
+    } else {
+      byte[] b = new byte[buffer.remaining()];
+      buffer.get(b);
+      checksum.update(b, 0, b.length);
+    }
+  }
+
+  @Override
+  public void update(byte[] b, int off, int len) {
+    checksum.update(b, off, len);
+  }
+
+  @Override
+  public void update(int i) {
+    checksum.update(i);
+  }
+
+  @Override
+  public long getValue() {
+    return checksum.getValue();
+  }
+
+  @Override
+  public void reset() {
+    checksum.reset();
+  }
+
+}

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/NativeCheckSumCRC32.java

@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.common;
+
+import org.apache.commons.lang3.NotImplementedException;
+import org.apache.hadoop.util.NativeCRC32Wrapper;
+
+import java.nio.ByteBuffer;
+
+/**
+ * This is a partial implementation to be used only in benchmarks.
+ *
+ * The Hadoop Native checksum libraries do not allow for updating a checksum
+ * as the java.util.zip.Checksum dictates in its update(...) method.
+ *
+ * This class allows the Native Hadoop CRC32 implementations to be called to
+ * generate checksums, provided only a single call is made to the update(...)
+ * method.
+ *
+ */
+public class NativeCheckSumCRC32 implements java.util.zip.Checksum {
+
+  // 1 for crc32, 2 for crc32c - see NativeCRC32Wrapper
+  private int checksumType;
+  private int bytesPerSum;
+
+  private ByteBuffer checksum = ByteBuffer.allocate(4);
+  private boolean needsReset = false;
+
+  public NativeCheckSumCRC32(int checksumType, int bytesPerSum) {
+    this.checksumType = checksumType;
+    this.bytesPerSum = bytesPerSum;
+  }
+
+  @Override
+  public void update(int b) {
+    throw new NotImplementedException("Update method is not implemented");
+  }
+
+  /**
+   * Calculate the checksum. Note the checksum is not updatable. You should
+   * make a single call to this method and then call getValue() to retrive the
+   * value.
+   * @param b A byte array whose contents will be used to calculate a CRC32(C)
+   * @param off The offset in the byte array to start reading.
+   * @param len The number of bytes in the byte array to read.
+   */
+  @Override
+  public void update(byte[] b, int off, int len) {
+    if (needsReset) {
+      throw new IllegalArgumentException(
+          "This checksum implementation is not updatable");
+    }
+    NativeCRC32Wrapper.calculateChunkedSumsByteArray(bytesPerSum, checksumType,
+        checksum.array(), 0, b, off, len);
+    needsReset = true;
+  }
+
+  @Override
+  public long getValue() {
+    checksum.position(0);
+    return checksum.getInt();
+  }
+
+  @Override
+  public void reset() {
+    checksum.clear();
+    needsReset = false;
+  }
+}

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/PureJavaCrc32ByteBuffer.java

@@ -21,7 +21,8 @@
  * Similar to {@link org.apache.hadoop.util.PureJavaCrc32}
  * except that this class implement {@link ChecksumByteBuffer}.
  */
-final class PureJavaCrc32ByteBuffer extends ChecksumByteBuffer.CrcIntTable {
+public final class PureJavaCrc32ByteBuffer extends
+    ChecksumByteBuffer.CrcIntTable {
   @Override
   int[] getTable() {
     return T;

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/PureJavaCrc32CByteBuffer.java

@@ -24,7 +24,8 @@
  * Similar to {@link org.apache.hadoop.util.PureJavaCrc32C}
  * except that this class implement {@link ChecksumByteBuffer}.
  */
-final class PureJavaCrc32CByteBuffer extends ChecksumByteBuffer.CrcIntTable {
+public final class PureJavaCrc32CByteBuffer extends
+    ChecksumByteBuffer.CrcIntTable {
   @Override
   int[] getTable() {
     return T;

hadoop-hdds/common/src/main/java/org/apache/hadoop/util/NativeCRC32Wrapper.java

@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.apache.hadoop.fs.ChecksumException;
+
+import java.nio.ByteBuffer;
+
+/**
+ * This class wraps the NativeCRC32 class in hadoop-common, because the class
+ * is package private there. The intention of making this class available
+ * in Ozone is to allow the native libraries to be benchmarked alongside other
+ * implementations. At the current time, the hadoop native CRC is not used
+ * anywhere in Ozone except for benchmarks.
+ */
+public final class NativeCRC32Wrapper {
+
+  public static final int CHECKSUM_CRC32 = NativeCrc32.CHECKSUM_CRC32;
+  public static final int CHECKSUM_CRC32C = NativeCrc32.CHECKSUM_CRC32C;
+
+  // Private constructor
+  private NativeCRC32Wrapper() {
+  }
+
+  public static boolean isAvailable() {
+    return NativeCrc32.isAvailable();
+  }
+
+  public static void verifyChunkedSums(int bytesPerSum, int checksumType,
+      ByteBuffer sums, ByteBuffer data, String fileName, long basePos)
+      throws ChecksumException {
+    NativeCrc32.verifyChunkedSums(bytesPerSum, checksumType, sums, data,
+        fileName, basePos);
+  }
+
+  @SuppressWarnings("checkstyle:parameternumber")
+  public static void verifyChunkedSumsByteArray(int bytesPerSum,
+      int checksumType, byte[] sums, int sumsOffset, byte[] data,
+      int dataOffset, int dataLength, String fileName, long basePos)
+      throws ChecksumException {
+    NativeCrc32.verifyChunkedSumsByteArray(bytesPerSum, checksumType, sums,
+        sumsOffset, data, dataOffset, dataLength, fileName, basePos);
+  }
+
+  public static void calculateChunkedSums(int bytesPerSum, int checksumType,
+                                          ByteBuffer sums, ByteBuffer data) {
+    NativeCrc32.calculateChunkedSums(bytesPerSum, checksumType, sums, data);
+  }
+
+  public static void calculateChunkedSumsByteArray(int bytesPerSum,
+      int checksumType, byte[] sums, int sumsOffset, byte[] data,
+      int dataOffset, int dataLength) {
+    NativeCrc32.calculateChunkedSumsByteArray(bytesPerSum, checksumType, sums,
+        sumsOffset, data, dataOffset, dataLength);
+  }
+
+}

hadoop-hdds/common/src/main/java/org/apache/hadoop/util/package-info.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains class that wrap private classes in hadoop-common
+ * util.
+ */
+package org.apache.hadoop.util;

hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumImplsComputeSameValues.java

@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.common;
+
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.util.NativeCRC32Wrapper;
+import org.apache.hadoop.util.PureJavaCrc32;
+import org.apache.hadoop.util.PureJavaCrc32C;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.zip.CRC32;
+
+import static junit.framework.TestCase.assertEquals;
+
+public class TestChecksumImplsComputeSameValues {
+
+  private int dataSize = 1024 * 1024 * 64;
+  private ByteBuffer data = ByteBuffer.allocate(dataSize);
+  private int[] bytesPerChecksum = {512, 1024, 2048, 4096, 32768, 1048576};
+
+  @Test
+  public void testCRC32ImplsMatch() {
+    data.clear();
+    data.put(RandomUtils.nextBytes(data.remaining()));
+    for (int bpc : bytesPerChecksum) {
+      List<ChecksumByteBuffer> impls = new ArrayList<>();
+      impls.add(new PureJavaCrc32ByteBuffer());
+      impls.add(new ChecksumByteBufferImpl(new PureJavaCrc32()));
+      impls.add(new ChecksumByteBufferImpl(new CRC32()));
+      if (NativeCRC32Wrapper.isAvailable()) {
+        impls.add(new ChecksumByteBufferImpl(new NativeCheckSumCRC32(1, bpc)));
+      }
+      assertEquals(true, validateImpls(data, impls, bpc));
+    }
+  }
+
+  @Test
+  public void testCRC32CImplsMatch() {
+    data.clear();
+    data.put(RandomUtils.nextBytes(data.remaining()));
+    for (int bpc : bytesPerChecksum) {
+      List<ChecksumByteBuffer> impls = new ArrayList<>();
+      impls.add(new PureJavaCrc32CByteBuffer());
+      impls.add(new ChecksumByteBufferImpl(new PureJavaCrc32C()));
+      // TODO - optional loaded java.util.zip.CRC32C if >= Java 9
+      // impls.add(new ChecksumByteBufferImpl(new CRC32C())));
+      if (NativeCRC32Wrapper.isAvailable()) {
+        impls.add(new ChecksumByteBufferImpl(new NativeCheckSumCRC32(2, bpc)));
+      }
+      assertEquals(true, validateImpls(data, impls, bpc));
+    }
+  }
+
+  private boolean validateImpls(ByteBuffer buf, List<ChecksumByteBuffer> impls,
+      int bpc) {
+    for (int i = 0; i < buf.capacity(); i += bpc) {
+      buf.position(i);
+      buf.limit(i + bpc);
+      impls.get(0).update(buf);
+      int res = (int) impls.get(0).getValue();
+      impls.get(0).reset();
+      for (int j = 1; j < impls.size(); j++) {
+        ChecksumByteBuffer csum = impls.get(j);
+        buf.position(i);
+        buf.limit(i + bpc);
+        csum.update(buf);
+        if ((int) csum.getValue() != res) {
+          return false;
+        }
+        csum.reset();
+      }
+    }
+    return true;
+  }
+
+}