diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferImpl.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferImpl.java new file mode 100644 index 000000000000..18651fe46242 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferImpl.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.common; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.nio.ByteBuffer; +import java.util.zip.Checksum; + +public class ChecksumByteBufferImpl implements ChecksumByteBuffer { + + public static class Java9Crc32CFactory { + private static final MethodHandle NEW_CRC32C_MH; + + static { + MethodHandle newCRC32C = null; + try { + newCRC32C = MethodHandles.publicLookup() + .findConstructor( + Class.forName("java.util.zip.CRC32C"), + MethodType.methodType(void.class) + ); + } catch (ReflectiveOperationException e) { + // Should not reach here. + throw new RuntimeException(e); + } + NEW_CRC32C_MH = newCRC32C; + } + + public static java.util.zip.Checksum createChecksum() { + try { + // Should throw nothing + return (Checksum) NEW_CRC32C_MH.invoke(); + } catch (Throwable t) { + throw (t instanceof RuntimeException) ? (RuntimeException) t + : new RuntimeException(t); + } + } + }; + + private Checksum checksum; + + public ChecksumByteBufferImpl(Checksum impl) { + this.checksum = impl; + } + + @Override + // TODO - when we eventually move to a minimum Java version >= 9 this method + // should be refactored to simply call checksum.update(buffer), as the + // Checksum interface has been enhanced to allow this since Java 9. + public void update(ByteBuffer buffer) { + if (buffer.hasArray()) { + checksum.update(buffer.array(), buffer.position() + buffer.arrayOffset(), + buffer.remaining()); + } else { + byte[] b = new byte[buffer.remaining()]; + buffer.get(b); + checksum.update(b, 0, b.length); + } + } + + @Override + public void update(byte[] b, int off, int len) { + checksum.update(b, off, len); + } + + @Override + public void update(int i) { + checksum.update(i); + } + + @Override + public long getValue() { + return checksum.getValue(); + } + + @Override + public void reset() { + checksum.reset(); + } + +} \ No newline at end of file diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/NativeCheckSumCRC32.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/NativeCheckSumCRC32.java new file mode 100644 index 000000000000..81061acbacf0 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/NativeCheckSumCRC32.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.common; + +import org.apache.commons.lang3.NotImplementedException; +import org.apache.hadoop.util.NativeCRC32Wrapper; + +import java.nio.ByteBuffer; + +/** + * This is a partial implementation to be used only in benchmarks. + * + * The Hadoop Native checksum libraries do not allow for updating a checksum + * as the java.util.zip.Checksum dictates in its update(...) method. + * + * This class allows the Native Hadoop CRC32 implementations to be called to + * generate checksums, provided only a single call is made to the update(...) + * method. + * + */ +public class NativeCheckSumCRC32 implements java.util.zip.Checksum { + + // 1 for crc32, 2 for crc32c - see NativeCRC32Wrapper + private int checksumType; + private int bytesPerSum; + + private ByteBuffer checksum = ByteBuffer.allocate(4); + private boolean needsReset = false; + + public NativeCheckSumCRC32(int checksumType, int bytesPerSum) { + this.checksumType = checksumType; + this.bytesPerSum = bytesPerSum; + } + + @Override + public void update(int b) { + throw new NotImplementedException("Update method is not implemented"); + } + + /** + * Calculate the checksum. Note the checksum is not updatable. You should + * make a single call to this method and then call getValue() to retrive the + * value. + * @param b A byte array whose contents will be used to calculate a CRC32(C) + * @param off The offset in the byte array to start reading. + * @param len The number of bytes in the byte array to read. + */ + @Override + public void update(byte[] b, int off, int len) { + if (needsReset) { + throw new IllegalArgumentException( + "This checksum implementation is not updatable"); + } + NativeCRC32Wrapper.calculateChunkedSumsByteArray(bytesPerSum, checksumType, + checksum.array(), 0, b, off, len); + needsReset = true; + } + + @Override + public long getValue() { + checksum.position(0); + return checksum.getInt(); + } + + @Override + public void reset() { + checksum.clear(); + needsReset = false; + } +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/PureJavaCrc32ByteBuffer.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/PureJavaCrc32ByteBuffer.java index 0d1f6307501a..001a6454a634 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/PureJavaCrc32ByteBuffer.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/PureJavaCrc32ByteBuffer.java @@ -21,7 +21,8 @@ * Similar to {@link org.apache.hadoop.util.PureJavaCrc32} * except that this class implement {@link ChecksumByteBuffer}. */ -final class PureJavaCrc32ByteBuffer extends ChecksumByteBuffer.CrcIntTable { +public final class PureJavaCrc32ByteBuffer extends + ChecksumByteBuffer.CrcIntTable { @Override int[] getTable() { return T; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/PureJavaCrc32CByteBuffer.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/PureJavaCrc32CByteBuffer.java index 1c443575f817..c101c30d289b 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/PureJavaCrc32CByteBuffer.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/PureJavaCrc32CByteBuffer.java @@ -24,7 +24,8 @@ * Similar to {@link org.apache.hadoop.util.PureJavaCrc32C} * except that this class implement {@link ChecksumByteBuffer}. */ -final class PureJavaCrc32CByteBuffer extends ChecksumByteBuffer.CrcIntTable { +public final class PureJavaCrc32CByteBuffer extends + ChecksumByteBuffer.CrcIntTable { @Override int[] getTable() { return T; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/NativeCRC32Wrapper.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/NativeCRC32Wrapper.java new file mode 100644 index 000000000000..d88cdbb16fdb --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/NativeCRC32Wrapper.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import org.apache.hadoop.fs.ChecksumException; + +import java.nio.ByteBuffer; + +/** + * This class wraps the NativeCRC32 class in hadoop-common, because the class + * is package private there. The intention of making this class available + * in Ozone is to allow the native libraries to be benchmarked alongside other + * implementations. At the current time, the hadoop native CRC is not used + * anywhere in Ozone except for benchmarks. + */ +public final class NativeCRC32Wrapper { + + public static final int CHECKSUM_CRC32 = NativeCrc32.CHECKSUM_CRC32; + public static final int CHECKSUM_CRC32C = NativeCrc32.CHECKSUM_CRC32C; + + // Private constructor + private NativeCRC32Wrapper() { + } + + public static boolean isAvailable() { + return NativeCrc32.isAvailable(); + } + + public static void verifyChunkedSums(int bytesPerSum, int checksumType, + ByteBuffer sums, ByteBuffer data, String fileName, long basePos) + throws ChecksumException { + NativeCrc32.verifyChunkedSums(bytesPerSum, checksumType, sums, data, + fileName, basePos); + } + + @SuppressWarnings("checkstyle:parameternumber") + public static void verifyChunkedSumsByteArray(int bytesPerSum, + int checksumType, byte[] sums, int sumsOffset, byte[] data, + int dataOffset, int dataLength, String fileName, long basePos) + throws ChecksumException { + NativeCrc32.verifyChunkedSumsByteArray(bytesPerSum, checksumType, sums, + sumsOffset, data, dataOffset, dataLength, fileName, basePos); + } + + public static void calculateChunkedSums(int bytesPerSum, int checksumType, + ByteBuffer sums, ByteBuffer data) { + NativeCrc32.calculateChunkedSums(bytesPerSum, checksumType, sums, data); + } + + public static void calculateChunkedSumsByteArray(int bytesPerSum, + int checksumType, byte[] sums, int sumsOffset, byte[] data, + int dataOffset, int dataLength) { + NativeCrc32.calculateChunkedSumsByteArray(bytesPerSum, checksumType, sums, + sumsOffset, data, dataOffset, dataLength); + } + +} \ No newline at end of file diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/package-info.java new file mode 100644 index 000000000000..3d1aba5d9dd4 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package contains class that wrap private classes in hadoop-common + * util. + */ +package org.apache.hadoop.util; diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumImplsComputeSameValues.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumImplsComputeSameValues.java new file mode 100644 index 000000000000..aa4b45462da3 --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumImplsComputeSameValues.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.common; + +import org.apache.commons.lang3.RandomUtils; +import org.apache.hadoop.util.NativeCRC32Wrapper; +import org.apache.hadoop.util.PureJavaCrc32; +import org.apache.hadoop.util.PureJavaCrc32C; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.zip.CRC32; + +import static junit.framework.TestCase.assertEquals; + +public class TestChecksumImplsComputeSameValues { + + private int dataSize = 1024 * 1024 * 64; + private ByteBuffer data = ByteBuffer.allocate(dataSize); + private int[] bytesPerChecksum = {512, 1024, 2048, 4096, 32768, 1048576}; + + @Test + public void testCRC32ImplsMatch() { + data.clear(); + data.put(RandomUtils.nextBytes(data.remaining())); + for (int bpc : bytesPerChecksum) { + List impls = new ArrayList<>(); + impls.add(new PureJavaCrc32ByteBuffer()); + impls.add(new ChecksumByteBufferImpl(new PureJavaCrc32())); + impls.add(new ChecksumByteBufferImpl(new CRC32())); + if (NativeCRC32Wrapper.isAvailable()) { + impls.add(new ChecksumByteBufferImpl(new NativeCheckSumCRC32(1, bpc))); + } + assertEquals(true, validateImpls(data, impls, bpc)); + } + } + + @Test + public void testCRC32CImplsMatch() { + data.clear(); + data.put(RandomUtils.nextBytes(data.remaining())); + for (int bpc : bytesPerChecksum) { + List impls = new ArrayList<>(); + impls.add(new PureJavaCrc32CByteBuffer()); + impls.add(new ChecksumByteBufferImpl(new PureJavaCrc32C())); + try { + impls.add(new ChecksumByteBufferImpl( + ChecksumByteBufferImpl.Java9Crc32CFactory.createChecksum())); + } catch (Throwable e) { + // NOOP + } + // impls.add(new ChecksumByteBufferImpl(new CRC32C()))); + if (NativeCRC32Wrapper.isAvailable()) { + impls.add(new ChecksumByteBufferImpl(new NativeCheckSumCRC32(2, bpc))); + } + assertEquals(true, validateImpls(data, impls, bpc)); + } + } + + private boolean validateImpls(ByteBuffer buf, List impls, + int bpc) { + for (int i = 0; i < buf.capacity(); i += bpc) { + buf.position(i); + buf.limit(i + bpc); + impls.get(0).update(buf); + int res = (int) impls.get(0).getValue(); + impls.get(0).reset(); + for (int j = 1; j < impls.size(); j++) { + ChecksumByteBuffer csum = impls.get(j); + buf.position(i); + buf.limit(i + bpc); + csum.update(buf); + if ((int) csum.getValue() != res) { + return false; + } + csum.reset(); + } + } + return true; + } + +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkCRCBatch.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkCRCBatch.java new file mode 100644 index 000000000000..b4875fc36968 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkCRCBatch.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.genesis; + +import java.nio.ByteBuffer; + +import org.apache.commons.lang3.RandomUtils; +import org.apache.hadoop.util.NativeCRC32Wrapper; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +/** + * Class to benchmark hadoop native CRC implementations in batch node. + * + * The hadoop native libraries must be available to run this test. libhadoop.so + * is not currently bundled with Ozone, so it needs to be obtained from a Hadoop + * build and the test needs to be executed on a compatible OS (ie Linux x86): + * + * ozone --jvmargs -Djava.library.path=/home/sodonnell/native genesis -b + * BenchmarkCRCBatch + */ +public class BenchMarkCRCBatch { + + private static int dataSize = 64 * 1024 * 1024; + + @State(Scope.Thread) + public static class BenchmarkState { + + private final ByteBuffer data = ByteBuffer.allocate(dataSize); + + @Param({"512", "1024", "2048", "4096", "32768", "1048576"}) + private int checksumSize; + + @Param({"nativeCRC32", "nativeCRC32C"}) + private String crcImpl; + + private byte[] checksumBuffer; + private int nativeChecksumType = 1; + + public ByteBuffer data() { + return data; + } + + public int checksumSize() { + return checksumSize; + } + + public String crcImpl() { + return crcImpl; + } + + @edu.umd.cs.findbugs.annotations.SuppressFBWarnings( + value="EI_EXPOSE_REP", + justification="The intent is to expose this variable") + public byte[] checksumBuffer() { + return checksumBuffer; + } + + public int nativeChecksumType() { + return nativeChecksumType; + } + + @Setup(Level.Trial) + public void setUp() { + switch (crcImpl) { + case "nativeCRC32": + if (NativeCRC32Wrapper.isAvailable()) { + nativeChecksumType = NativeCRC32Wrapper.CHECKSUM_CRC32; + checksumBuffer = new byte[4 * dataSize / checksumSize]; + } else { + throw new RuntimeException("Native library is not available"); + } + break; + case "nativeCRC32C": + if (NativeCRC32Wrapper.isAvailable()) { + nativeChecksumType = NativeCRC32Wrapper.CHECKSUM_CRC32C; + checksumBuffer = new byte[4 * dataSize / checksumSize]; + } else { + throw new RuntimeException("Native library is not available"); + } + break; + default: + } + data.put(RandomUtils.nextBytes(data.remaining())); + } + } + + @Benchmark + @Threads(1) + @Warmup(iterations = 3, time = 1000, timeUnit = MILLISECONDS) + @Fork(value = 1, warmups = 0) + @Measurement(iterations = 5, time = 2000, timeUnit = MILLISECONDS) + @BenchmarkMode(Mode.Throughput) + public void runCRCNativeBatch(Blackhole blackhole, BenchmarkState state) { + if (state.crcImpl.equals("nativeCRC32") + || state.crcImpl.equals("nativeCRC32C")) { + NativeCRC32Wrapper.calculateChunkedSumsByteArray( + state.checksumSize, state.nativeChecksumType, state.checksumBuffer, + 0, state.data.array(), 0, state.data.capacity()); + blackhole.consume(state.checksumBuffer); + } else { + throw new RuntimeException("Batch mode not available for " + + state.crcImpl); + } + } + + public static void main(String[] args) throws Exception { + org.openjdk.jmh.Main.main(args); + } +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkCRCStreaming.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkCRCStreaming.java new file mode 100644 index 000000000000..0d8ef2fc0b8b --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkCRCStreaming.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.genesis; + +import java.nio.ByteBuffer; + +import org.apache.commons.lang3.RandomUtils; +import org.apache.hadoop.ozone.common.ChecksumByteBuffer; +import org.apache.hadoop.ozone.common.ChecksumByteBufferImpl; +import org.apache.hadoop.ozone.common.NativeCheckSumCRC32; +import org.apache.hadoop.ozone.common.PureJavaCrc32ByteBuffer; +import org.apache.hadoop.ozone.common.PureJavaCrc32CByteBuffer; +import org.apache.hadoop.util.NativeCRC32Wrapper; +import org.apache.hadoop.util.PureJavaCrc32; +import org.apache.hadoop.util.PureJavaCrc32C; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.zip.CRC32; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +/** + * Class to benchmark various CRC implementations. This can be executed via + * + * ozone genesis -b BenchmarkCRC + * + * However there are some points to keep in mind. java.util.zip.CRC32C is not + * available until Java 9, therefore if the JVM has a lower version than 9, that + * implementation will not be tested. + * + * The hadoop native libraries will only be tested if libhadoop.so is found on + * the "-Djava.library.path". libhadoop.so is not currently bundled with Ozone, + * so it needs to be obtained from a Hadoop build and the test needs to be + * executed on a compatible OS (ie Linux x86): + * + * ozone --jvmargs -Djava.library.path=/home/sodonnell/native genesis -b + * BenchmarkCRC + */ +public class BenchMarkCRCStreaming { + + private static int dataSize = 64 * 1024 * 1024; + + @State(Scope.Thread) + public static class BenchmarkState { + + private final ByteBuffer data = ByteBuffer.allocate(dataSize); + + @Param({"512", "1024", "2048", "4096", "32768", "1048576"}) + private int checksumSize; + + @Param({"pureCRC32", "pureCRC32C", "hadoopCRC32C", "hadoopCRC32", + "zipCRC32", "zipCRC32C", "nativeCRC32", "nativeCRC32C"}) + private String crcImpl; + + private ChecksumByteBuffer checksum; + + public ChecksumByteBuffer checksum() { + return checksum; + } + + public String crcImpl() { + return crcImpl; + } + + public int checksumSize() { + return checksumSize; + } + + @Setup(Level.Trial) + public void setUp() { + switch (crcImpl) { + case "pureCRC32": + checksum = new PureJavaCrc32ByteBuffer(); + break; + case "pureCRC32C": + checksum = new PureJavaCrc32CByteBuffer(); + break; + case "hadoopCRC32": + checksum = new ChecksumByteBufferImpl(new PureJavaCrc32()); + break; + case "hadoopCRC32C": + checksum = new ChecksumByteBufferImpl(new PureJavaCrc32C()); + break; + case "zipCRC32": + checksum = new ChecksumByteBufferImpl(new CRC32()); + break; + case "zipCRC32C": + try { + checksum = new ChecksumByteBufferImpl( + ChecksumByteBufferImpl.Java9Crc32CFactory.createChecksum()); + } catch (Throwable e) { + throw new RuntimeException("zipCRC32C is not available pre Java 9"); + } + break; + case "nativeCRC32": + if (NativeCRC32Wrapper.isAvailable()) { + checksum = new ChecksumByteBufferImpl(new NativeCheckSumCRC32( + NativeCRC32Wrapper.CHECKSUM_CRC32, checksumSize)); + } else { + throw new RuntimeException("Native library is not available"); + } + break; + case "nativeCRC32C": + if (NativeCRC32Wrapper.isAvailable()) { + checksum = new ChecksumByteBufferImpl(new NativeCheckSumCRC32( + NativeCRC32Wrapper.CHECKSUM_CRC32C, checksumSize)); + } else { + throw new RuntimeException("Native library is not available"); + } + break; + default: + } + data.clear(); + data.put(RandomUtils.nextBytes(data.remaining())); + } + } + + @Benchmark + @Threads(1) + @Warmup(iterations = 3, time = 1000, timeUnit = MILLISECONDS) + @Fork(value = 1, warmups = 0) + @Measurement(iterations = 5, time = 2000, timeUnit = MILLISECONDS) + @BenchmarkMode(Mode.Throughput) + public void runCRC(Blackhole blackhole, BenchmarkState state) { + ByteBuffer data = state.data; + data.clear(); + ChecksumByteBuffer csum = state.checksum; + int bytesPerCheckSum = state.checksumSize; + + for (int i=0; i