diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/JavaUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/JavaUtils.java new file mode 100644 index 00000000000..63c29ba7c91 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/JavaUtils.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds; + +/** + * Various reusable utility methods related to Java. + */ +public final class JavaUtils { + // "1.8"->8, "9"->9, "10"->10 + private static final int JAVA_SPEC_VER = Math.max(8, Integer.parseInt( + System.getProperty("java.specification.version").split("\\.")[0])); + + /** + * Query to see if major version of Java specification of the system + * is equal or greater than the parameter. + * + * @param version 8, 9, 10 etc. + * @return comparison with system property, always true for 8 + */ + public static boolean isJavaVersionAtLeast(int version) { + return JAVA_SPEC_VER >= version; + } + + /** + * Private constructor. + */ + private JavaUtils() { + } +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java index d86f7b1c40c..db7a31eea95 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java @@ -75,8 +75,10 @@ private static Function newChecksumByteBufferFunction( /** The algorithms for {@link ChecksumType}. */ enum Algorithm { NONE(() -> data -> ByteString.EMPTY), - CRC32(() -> newChecksumByteBufferFunction(PureJavaCrc32ByteBuffer::new)), - CRC32C(() -> newChecksumByteBufferFunction(PureJavaCrc32CByteBuffer::new)), + CRC32(() -> + newChecksumByteBufferFunction(ChecksumByteBufferFactory::crc32Impl)), + CRC32C(() -> + newChecksumByteBufferFunction(ChecksumByteBufferFactory::crc32CImpl)), SHA256(() -> newMessageDigestFunction("SHA-256")), MD5(() -> newMessageDigestFunction("MD5")); diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferFactory.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferFactory.java new file mode 100644 index 00000000000..a41af84189e --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferFactory.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.common; + +import org.apache.hadoop.hdds.JavaUtils; +import org.apache.hadoop.util.PureJavaCrc32C; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.util.zip.CRC32; +import java.util.zip.Checksum; + +/** + * Class containing factories for creating various checksum impls. + */ +public final class ChecksumByteBufferFactory { + + private static final Logger LOG = + LoggerFactory.getLogger(ChecksumByteBufferImpl.class); + + private static volatile boolean useJava9Crc32C + = JavaUtils.isJavaVersionAtLeast(9); + + public static class Java9Crc32CFactory { + private static final MethodHandle NEW_CRC32C_MH; + + static { + MethodHandle newCRC32C = null; + try { + newCRC32C = MethodHandles.publicLookup() + .findConstructor( + Class.forName("java.util.zip.CRC32C"), + MethodType.methodType(void.class) + ); + } catch (ReflectiveOperationException e) { + // Should not reach here. + throw new RuntimeException(e); + } + NEW_CRC32C_MH = newCRC32C; + } + + public static java.util.zip.Checksum createChecksum() { + try { + // Should throw nothing + return (Checksum) NEW_CRC32C_MH.invoke(); + } catch (Throwable t) { + throw (t instanceof RuntimeException) ? (RuntimeException) t + : new RuntimeException(t); + } + } + }; + + public static ChecksumByteBuffer crc32Impl() { + return new ChecksumByteBufferImpl(new CRC32()); + } + + public static ChecksumByteBuffer crc32CImpl() { + if (useJava9Crc32C) { + try { + return new ChecksumByteBufferImpl(Java9Crc32CFactory.createChecksum()); + } catch (Throwable e) { + // should not happen + LOG.error("CRC32C creation failed, switching to PureJavaCrc32C", e); + useJava9Crc32C = false; + } + } + return new ChecksumByteBufferImpl(new PureJavaCrc32C()); + } + + /** + * Private Constructor. + */ + private ChecksumByteBufferFactory() { + } +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferImpl.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferImpl.java index 18651fe4624..db779b6bc36 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferImpl.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferImpl.java @@ -17,43 +17,11 @@ */ package org.apache.hadoop.ozone.common; -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; import java.nio.ByteBuffer; import java.util.zip.Checksum; public class ChecksumByteBufferImpl implements ChecksumByteBuffer { - public static class Java9Crc32CFactory { - private static final MethodHandle NEW_CRC32C_MH; - - static { - MethodHandle newCRC32C = null; - try { - newCRC32C = MethodHandles.publicLookup() - .findConstructor( - Class.forName("java.util.zip.CRC32C"), - MethodType.methodType(void.class) - ); - } catch (ReflectiveOperationException e) { - // Should not reach here. - throw new RuntimeException(e); - } - NEW_CRC32C_MH = newCRC32C; - } - - public static java.util.zip.Checksum createChecksum() { - try { - // Should throw nothing - return (Checksum) NEW_CRC32C_MH.invoke(); - } catch (Throwable t) { - throw (t instanceof RuntimeException) ? (RuntimeException) t - : new RuntimeException(t); - } - } - }; - private Checksum checksum; public ChecksumByteBufferImpl(Checksum impl) { diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumByteBuffer.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumByteBuffer.java index 2f466377b4b..4aac0b4847e 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumByteBuffer.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumByteBuffer.java @@ -33,14 +33,14 @@ public class TestChecksumByteBuffer { @Test public void testPureJavaCrc32ByteBuffer() { final Checksum expected = new PureJavaCrc32(); - final ChecksumByteBuffer testee = new PureJavaCrc32ByteBuffer(); + final ChecksumByteBuffer testee = ChecksumByteBufferFactory.crc32Impl(); new VerifyChecksumByteBuffer(expected, testee).testCorrectness(); } @Test public void testPureJavaCrc32CByteBuffer() { final Checksum expected = new PureJavaCrc32C(); - final ChecksumByteBuffer testee = new PureJavaCrc32CByteBuffer(); + final ChecksumByteBuffer testee = ChecksumByteBufferFactory.crc32CImpl(); new VerifyChecksumByteBuffer(expected, testee).testCorrectness(); } diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumImplsComputeSameValues.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumImplsComputeSameValues.java index aa4b45462da..a098a26f9d8 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumImplsComputeSameValues.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumImplsComputeSameValues.java @@ -62,7 +62,7 @@ public void testCRC32CImplsMatch() { impls.add(new ChecksumByteBufferImpl(new PureJavaCrc32C())); try { impls.add(new ChecksumByteBufferImpl( - ChecksumByteBufferImpl.Java9Crc32CFactory.createChecksum())); + ChecksumByteBufferFactory.Java9Crc32CFactory.createChecksum())); } catch (Throwable e) { // NOOP } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkCRCStreaming.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkCRCStreaming.java index 0d8ef2fc0b8..5dd5da8390a 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkCRCStreaming.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkCRCStreaming.java @@ -21,6 +21,7 @@ import org.apache.commons.lang3.RandomUtils; import org.apache.hadoop.ozone.common.ChecksumByteBuffer; +import org.apache.hadoop.ozone.common.ChecksumByteBufferFactory; import org.apache.hadoop.ozone.common.ChecksumByteBufferImpl; import org.apache.hadoop.ozone.common.NativeCheckSumCRC32; import org.apache.hadoop.ozone.common.PureJavaCrc32ByteBuffer; @@ -114,7 +115,7 @@ public void setUp() { case "zipCRC32C": try { checksum = new ChecksumByteBufferImpl( - ChecksumByteBufferImpl.Java9Crc32CFactory.createChecksum()); + ChecksumByteBufferFactory.Java9Crc32CFactory.createChecksum()); } catch (Throwable e) { throw new RuntimeException("zipCRC32C is not available pre Java 9"); }