From 55c6baf8856680993b9d0959f2d6c955f10f7039 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Thu, 18 Jul 2019 11:58:58 -0700 Subject: [PATCH 1/2] Cleanup Travis config --- .travis.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 938d5f52..74e36132 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,10 @@ language: java jdk: - - oraclejdk8 - -sudo: false + - openjdk8 + - openjdk9 + - openjdk11 + - openjdk12 script: - travis_wait 20 mvn test - -cache: - directories: - - $HOME/.m2/repository From c9b6be3b4fd56301ed556189d2db51ecffdc906f Mon Sep 17 00:00:00 2001 From: David Phillips Date: Thu, 18 Jul 2019 11:53:25 -0700 Subject: [PATCH 2/2] Remove dependency on Slice --- pom.xml | 6 - .../io/airlift/compress/zstd/XxHash64.java | 139 ++++++++++++++++++ .../compress/zstd/ZstdFrameCompressor.java | 12 +- .../compress/zstd/ZstdFrameDecompressor.java | 11 +- .../airlift/compress/zstd/TestXxHash64.java | 83 +++++++++++ 5 files changed, 224 insertions(+), 27 deletions(-) create mode 100644 src/main/java/io/airlift/compress/zstd/XxHash64.java create mode 100644 src/test/java/io/airlift/compress/zstd/TestXxHash64.java diff --git a/pom.xml b/pom.xml index 6d7080b0..4863b7d8 100644 --- a/pom.xml +++ b/pom.xml @@ -62,12 +62,6 @@ - - io.airlift - slice - 0.36 - - com.google.guava guava diff --git a/src/main/java/io/airlift/compress/zstd/XxHash64.java b/src/main/java/io/airlift/compress/zstd/XxHash64.java new file mode 100644 index 00000000..7d880543 --- /dev/null +++ b/src/main/java/io/airlift/compress/zstd/XxHash64.java @@ -0,0 +1,139 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.airlift.compress.zstd; + +import static io.airlift.compress.zstd.UnsafeUtil.UNSAFE; +import static java.lang.Long.rotateLeft; + +// forked from https://github.com/airlift/slice +final class XxHash64 +{ + private static final long PRIME64_1 = 0x9E3779B185EBCA87L; + private static final long PRIME64_2 = 0xC2B2AE3D27D4EB4FL; + private static final long PRIME64_3 = 0x165667B19E3779F9L; + private static final long PRIME64_4 = 0x85EBCA77C2b2AE63L; + private static final long PRIME64_5 = 0x27D4EB2F165667C5L; + + private XxHash64() {} + + public static long hash(long seed, Object base, long address, int length) + { + long hash; + if (length >= 32) { + hash = updateBody(seed, base, address, length); + } + else { + hash = seed + PRIME64_5; + } + + hash += length; + + // round to the closest 32 byte boundary + // this is the point up to which updateBody() processed + int index = length & 0xFFFFFFE0; + + return updateTail(hash, base, address, index, length); + } + + private static long updateTail(long hash, Object base, long address, int index, int length) + { + while (index <= length - 8) { + hash = updateTail(hash, UNSAFE.getLong(base, address + index)); + index += 8; + } + + if (index <= length - 4) { + hash = updateTail(hash, UNSAFE.getInt(base, address + index)); + index += 4; + } + + while (index < length) { + hash = updateTail(hash, UNSAFE.getByte(base, address + index)); + index++; + } + + hash = finalShuffle(hash); + + return hash; + } + + private static long updateBody(long seed, Object base, long address, int length) + { + long v1 = seed + PRIME64_1 + PRIME64_2; + long v2 = seed + PRIME64_2; + long v3 = seed; + long v4 = seed - PRIME64_1; + + int remaining = length; + while (remaining >= 32) { + v1 = mix(v1, UNSAFE.getLong(base, address)); + v2 = mix(v2, UNSAFE.getLong(base, address + 8)); + v3 = mix(v3, UNSAFE.getLong(base, address + 16)); + v4 = mix(v4, UNSAFE.getLong(base, address + 24)); + + address += 32; + remaining -= 32; + } + + long hash = rotateLeft(v1, 1) + rotateLeft(v2, 7) + rotateLeft(v3, 12) + rotateLeft(v4, 18); + + hash = update(hash, v1); + hash = update(hash, v2); + hash = update(hash, v3); + hash = update(hash, v4); + + return hash; + } + + private static long mix(long current, long value) + { + return rotateLeft(current + value * PRIME64_2, 31) * PRIME64_1; + } + + private static long update(long hash, long value) + { + long temp = hash ^ mix(0, value); + return temp * PRIME64_1 + PRIME64_4; + } + + private static long updateTail(long hash, long value) + { + long temp = hash ^ mix(0, value); + return rotateLeft(temp, 27) * PRIME64_1 + PRIME64_4; + } + + private static long updateTail(long hash, int value) + { + long unsigned = value & 0xFFFF_FFFFL; + long temp = hash ^ (unsigned * PRIME64_1); + return rotateLeft(temp, 23) * PRIME64_2 + PRIME64_3; + } + + private static long updateTail(long hash, byte value) + { + int unsigned = value & 0xFF; + long temp = hash ^ (unsigned * PRIME64_5); + return rotateLeft(temp, 11) * PRIME64_1; + } + + private static long finalShuffle(long hash) + { + hash ^= hash >>> 33; + hash *= PRIME64_2; + hash ^= hash >>> 29; + hash *= PRIME64_3; + hash ^= hash >>> 32; + return hash; + } +} diff --git a/src/main/java/io/airlift/compress/zstd/ZstdFrameCompressor.java b/src/main/java/io/airlift/compress/zstd/ZstdFrameCompressor.java index f231e514..c0debb27 100644 --- a/src/main/java/io/airlift/compress/zstd/ZstdFrameCompressor.java +++ b/src/main/java/io/airlift/compress/zstd/ZstdFrameCompressor.java @@ -13,11 +13,6 @@ */ package io.airlift.compress.zstd; -import io.airlift.slice.Slice; -import io.airlift.slice.Slices; -import io.airlift.slice.UnsafeSliceFactory; -import io.airlift.slice.XxHash64; - import static io.airlift.compress.zstd.Constants.COMPRESSED_BLOCK; import static io.airlift.compress.zstd.Constants.COMPRESSED_LITERALS_BLOCK; import static io.airlift.compress.zstd.Constants.MAGIC_NUMBER; @@ -130,12 +125,7 @@ static int writeChecksum(Object outputBase, long outputAddress, long outputLimit int inputSize = (int) (inputLimit - inputAddress); - Slice slice = Slices.EMPTY_SLICE; - if (inputSize > 0) { - slice = UnsafeSliceFactory.getInstance().newSlice(inputBase, inputAddress, inputSize); - } - - long hash = XxHash64.hash(0, slice); + long hash = XxHash64.hash(0, inputBase, inputAddress, inputSize); UNSAFE.putInt(outputBase, outputAddress, (int) hash); diff --git a/src/main/java/io/airlift/compress/zstd/ZstdFrameDecompressor.java b/src/main/java/io/airlift/compress/zstd/ZstdFrameDecompressor.java index c7b3ed27..1b6c1afb 100644 --- a/src/main/java/io/airlift/compress/zstd/ZstdFrameDecompressor.java +++ b/src/main/java/io/airlift/compress/zstd/ZstdFrameDecompressor.java @@ -14,10 +14,6 @@ package io.airlift.compress.zstd; import io.airlift.compress.MalformedInputException; -import io.airlift.slice.Slice; -import io.airlift.slice.Slices; -import io.airlift.slice.UnsafeSliceFactory; -import io.airlift.slice.XxHash64; import java.util.Arrays; @@ -197,12 +193,7 @@ public int decompress( if (frameHeader.hasChecksum) { int decodedFrameSize = (int) (output - outputStart); - Slice outputSlice = Slices.EMPTY_SLICE; - if (decodedFrameSize > 0) { - outputSlice = UnsafeSliceFactory.getInstance().newSlice(outputBase, outputStart, decodedFrameSize); - } - - long hash = XxHash64.hash(0, outputSlice); + long hash = XxHash64.hash(0, outputBase, outputStart, decodedFrameSize); int checksum = UNSAFE.getInt(inputBase, input); if (checksum != (int) hash) { diff --git a/src/test/java/io/airlift/compress/zstd/TestXxHash64.java b/src/test/java/io/airlift/compress/zstd/TestXxHash64.java new file mode 100644 index 00000000..f48590a5 --- /dev/null +++ b/src/test/java/io/airlift/compress/zstd/TestXxHash64.java @@ -0,0 +1,83 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.airlift.compress.zstd; + +import net.jpountz.xxhash.XXHash64; +import net.jpountz.xxhash.XXHashFactory; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; +import static sun.misc.Unsafe.ARRAY_BYTE_BASE_OFFSET; + +// forked from https://github.com/airlift/slice +public class TestXxHash64 +{ + private static final long PRIME = 2654435761L; + + private final byte[] buffer = new byte[101]; + + public TestXxHash64() + { + long value = PRIME; + for (int i = 0; i < buffer.length; i++) { + buffer[i] = (byte) (value >> 24); + value *= value; + } + } + + @Test + public void testSanity() + { + assertHash(0, buffer, 0, 0xEF46DB3751D8E999L); + + assertHash(0, buffer, 1, 0x4FCE394CC88952D8L); + assertHash(PRIME, buffer, 1, 0x739840CB819FA723L); + + assertHash(0, buffer, 4, 0x9256E58AA397AEF1L); + assertHash(PRIME, buffer, 4, 0x9D5FFDFB928AB4BL); + + assertHash(0, buffer, 8, 0xF74CB1451B32B8CFL); + assertHash(PRIME, buffer, 8, 0x9C44B77FBCC302C5L); + + assertHash(0, buffer, 14, 0xCFFA8DB881BC3A3DL); + assertHash(PRIME, buffer, 14, 0x5B9611585EFCC9CBL); + + assertHash(0, buffer, 32, 0xAF5753D39159EDEEL); + assertHash(PRIME, buffer, 32, 0xDCAB9233B8CA7B0FL); + + assertHash(0, buffer, buffer.length, 0x0EAB543384F878ADL); + assertHash(PRIME, buffer, buffer.length, 0xCAA65939306F1E21L); + } + + @Test + public void testMultipleLengths() + { + XXHash64 jpountz = XXHashFactory.fastestInstance().hash64(); + for (int i = 0; i < 20_000; i++) { + byte[] data = new byte[i]; + long expected = jpountz.hash(data, 0, data.length, 0); + assertHash(0, data, data.length, expected); + } + } + + private static void assertHash(long seed, byte[] data, int length, long expected) + { + assertEquals(hash(seed, data, length), expected); + } + + private static long hash(long seed, byte[] data, int length) + { + return XxHash64.hash(seed, data, ARRAY_BYTE_BASE_OFFSET, length); + } +}