Skip to content

Fix murmur3_32 with UTF-8 encoding for input with non-BMP character #5649

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
findepi wants to merge 4 commits into from
Closed

Fix murmur3_32 with UTF-8 encoding for input with non-BMP character #5649

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fe5eb09
Test murmur3_32 hashBytes for known inputs as well
findepi Jul 16, 2021
1c48d1f
Test murmur3_32 putString and putBytes for known inputs as well
findepi Jul 16, 2021
e85ac63
Test murmur3_32 with UTF_16 as well
findepi Jul 16, 2021
6ac0909
Fix murmur3_32 with UTF-8 encoding for input with non-BMP character
findepi Jul 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 56 additions & 35 deletions guava-tests/test/com/google/common/hash/Murmur3Hash32Test.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,15 @@

import com.google.common.base.Charsets;
import com.google.common.hash.HashTestUtils.HashFn;
import java.nio.charset.Charset;
import java.util.Random;
import junit.framework.TestCase;

/** Tests for {@link Murmur3_32HashFunction}. */
/**
* Tests for {@link Murmur3_32HashFunction}.
*/
public class Murmur3Hash32Test extends TestCase {

public void testKnownIntegerInputs() {
assertHash(593689054, murmur3_32().hashInt(0));
assertHash(-189366624, murmur3_32().hashInt(-42));
Expand All @@ -51,39 +55,49 @@ public void testKnownStringInputs() {
-528633700, murmur3_32().hashUnencodedChars("The quick brown fox jumps over the lazy dog"));
}

public void testKnownUtf8StringInputs() {
assertHash(0, murmur3_32().hashString("", Charsets.UTF_8));
assertHash(0xcfbda5d1, murmur3_32().hashString("k", Charsets.UTF_8));
assertHash(0xa167dbf3, murmur3_32().hashString("hell", Charsets.UTF_8));
assertHash(0x248bfa47, murmur3_32().hashString("hello", Charsets.UTF_8));
assertHash(0x3d41b97c, murmur3_32().hashString("http://www.google.com/", Charsets.UTF_8));
assertHash(
0x2e4ff723,
murmur3_32().hashString("The quick brown fox jumps over the lazy dog", Charsets.UTF_8));
assertHash(0xfc5ba834, murmur3_32().hashString("毎月１日,毎週月曜日", Charsets.UTF_8));
assertHash(
0x3f4aff5c,
murmur3_32().hashString(Character.toString(Character.MAX_VALUE), Charsets.UTF_8));
assertHash(
0x81db5903,
murmur3_32()
.hashString(new String(Character.toChars(Character.MAX_VALUE + 1)), Charsets.UTF_8));
// Note (https://github.com/google/guava/issues/5648) the hash expected here is not correct
assertHash(
0x256068c8,
murmur3_32()
.hashString(new String(Character.toChars(Character.MAX_CODE_POINT)), Charsets.UTF_8));
public void testKnownEncodedStringInputs() {
assertStringHash(0, "", Charsets.UTF_8);
assertStringHash(0xcfbda5d1, "k", Charsets.UTF_8);
assertStringHash(0xa167dbf3, "hell", Charsets.UTF_8);
assertStringHash(0x248bfa47, "hello", Charsets.UTF_8);
assertStringHash(0x3d41b97c, "http://www.google.com/", Charsets.UTF_8);
assertStringHash(0x2e4ff723, "The quick brown fox jumps over the lazy dog", Charsets.UTF_8);
assertStringHash(0xb5a4be05, "ABCDefGHI\u0799", Charsets.UTF_8);
assertStringHash(0xfc5ba834, "毎月１日,毎週月曜日", Charsets.UTF_8);
assertStringHash(0x8a5c3699, "surrogate pair: \uD83D\uDCB0", Charsets.UTF_8);

assertStringHash(0x3f4aff5c, Character.toString(Character.MAX_VALUE), Charsets.UTF_8);
assertStringHash(0x61c178ff, new String(Character.toChars(Character.MAX_VALUE + 1)),
Charsets.UTF_8);
assertStringHash(0x2bc506da, new String(Character.toChars(Character.MAX_CODE_POINT)),
Charsets.UTF_8);

assertStringHash(0, "", Charsets.UTF_16LE);
assertStringHash(0x288418e4, "k", Charsets.UTF_16LE);
assertStringHash(0x5a0cb7c3, "hell", Charsets.UTF_16LE);
assertStringHash(0xd7c31989, "hello", Charsets.UTF_16LE);
assertStringHash(0x73564d8c, "http://www.google.com/", Charsets.UTF_16LE);
assertStringHash(0xe07db09c, "The quick brown fox jumps over the lazy dog", Charsets.UTF_16LE);
assertStringHash(0xfefa3e76, "ABCDefGHI\u0799", Charsets.UTF_16LE);
assertStringHash(0x6a7be132, "毎月１日,毎週月曜日", Charsets.UTF_16LE);
assertStringHash(0x5a2d41c7, "surrogate pair: \uD83D\uDCB0", Charsets.UTF_16LE);

assertStringHash(0x8619621f, Character.toString(Character.MAX_VALUE), Charsets.UTF_16LE);
assertStringHash(0x496a90e1, new String(Character.toChars(Character.MAX_VALUE + 1)),
Charsets.UTF_16LE);
assertStringHash(0x11aff14f, new String(Character.toChars(Character.MAX_CODE_POINT)),
Charsets.UTF_16LE);
}

@SuppressWarnings("deprecation")
public void testSimpleStringUtf8() {
assertEquals(
murmur3_32().hashBytes("ABCDefGHI\u0799".getBytes(Charsets.UTF_8)),
murmur3_32().hashString("ABCDefGHI\u0799", Charsets.UTF_8));
private void assertStringHash(int expected, String string, Charset charset) {
assertHash(expected, murmur3_32().hashString(string, charset));
assertHash(expected, murmur3_32().newHasher().putString(string, charset).hash());
assertHash(expected, murmur3_32().hashBytes(string.getBytes(charset)));
assertHash(expected, murmur3_32().newHasher().putBytes(string.getBytes(charset)).hash());
}

@SuppressWarnings("deprecation")
public void testStringInputsUtf8() {
public void testEncodedStringInputs() {
Random rng = new Random(0);
for (int z = 0; z < 100; z++) {
String str;
Expand All @@ -93,16 +107,23 @@ public void testStringInputsUtf8() {
codePoints[i] = rng.nextInt(0x800);
} while (!Character.isValidCodePoint(codePoints[i])
|| (codePoints[i] >= Character.MIN_SURROGATE
&& codePoints[i] <= Character.MAX_SURROGATE));
&& codePoints[i] <= Character.MAX_SURROGATE));
}
StringBuilder builder = new StringBuilder();
for (int i = 0; i < codePoints.length; i++) {
builder.appendCodePoint(codePoints[i]);
}
str = builder.toString();
assertEquals(
murmur3_32().hashBytes(str.getBytes(Charsets.UTF_8)),
murmur3_32().hashString(str, Charsets.UTF_8));
HashCode hashUtf8 = murmur3_32().hashBytes(str.getBytes(Charsets.UTF_8));
assertEquals(hashUtf8,
murmur3_32().newHasher().putBytes(str.getBytes(Charsets.UTF_8)).hash());
assertEquals(hashUtf8, murmur3_32().hashString(str, Charsets.UTF_8));
assertEquals(hashUtf8, murmur3_32().newHasher().putString(str, Charsets.UTF_8).hash());
HashCode hashUtf16 = murmur3_32().hashBytes(str.getBytes(Charsets.UTF_16LE));
assertEquals(hashUtf16,
murmur3_32().newHasher().putBytes(str.getBytes(Charsets.UTF_16LE)).hash());
assertEquals(hashUtf16, murmur3_32().hashString(str, Charsets.UTF_16LE));
assertEquals(hashUtf16, murmur3_32().newHasher().putString(str, Charsets.UTF_16LE).hash());
}
}

Expand Down Expand Up @@ -146,7 +167,7 @@ public void testInvariants() {
public void testInvalidUnicodeHashString() {
String str =
new String(
new char[] {'a', Character.MIN_HIGH_SURROGATE, Character.MIN_HIGH_SURROGATE, 'z'});
new char[]{'a', Character.MIN_HIGH_SURROGATE, Character.MIN_HIGH_SURROGATE, 'z'});
assertEquals(
murmur3_32().hashBytes(str.getBytes(Charsets.UTF_8)),
murmur3_32().hashString(str, Charsets.UTF_8));
Expand All @@ -155,7 +176,7 @@ public void testInvalidUnicodeHashString() {
public void testInvalidUnicodeHasherPutString() {
String str =
new String(
new char[] {'a', Character.MIN_HIGH_SURROGATE, Character.MIN_HIGH_SURROGATE, 'z'});
new char[]{'a', Character.MIN_HIGH_SURROGATE, Character.MIN_HIGH_SURROGATE, 'z'});
assertEquals(
murmur3_32().hashBytes(str.getBytes(Charsets.UTF_8)),
murmur3_32().newHasher().putString(str, Charsets.UTF_8).hash());
Expand Down
1 change: 1 addition & 0 deletions guava/src/com/google/common/hash/Murmur3_32HashFunction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,7 @@ public HashCode hashString(CharSequence input, Charset charset) {
}
i++;
buffer |= codePointToFourUtf8Bytes(codePoint) << shift;
shift += 32;
len += 4;
}

Expand Down