Adding xxhash Vindex

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
 	github.com/aws/aws-sdk-go v0.0.0-20180223184012-ebef4262e06a
 	github.com/boltdb/bolt v1.3.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1
 	github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292 // indirect
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
 	github.com/coreos/etcd v0.0.0-20170626015032-703663d1f6ed

go.sum

@@ -29,6 +29,9 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
 github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=

go/vt/vtgate/vindexes/xxhash.go

@@ -0,0 +1,88 @@
+/*
+Copyright 2019 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vindexes
+
+import (
+	"bytes"
+	"encoding/binary"
+
+	"github.com/cespare/xxhash/v2"
+
+	"vitess.io/vitess/go/sqltypes"
+	"vitess.io/vitess/go/vt/key"
+)
+
+var (
+	_ Vindex = (*XXHash)(nil)
+)
+
+// XXHash defines vindex that hashes any sql types to a KeyspaceId
+// by using xxhash64. It's Unique and works on any platform giving identical result.
+type XXHash struct {
+	name string
+}
+
+// NewXXHash creates a new XXHash.
+func NewXXHash(name string, m map[string]string) (Vindex, error) {
+	return &XXHash{name: name}, nil
+}
+
+// String returns the name of the vindex.
+func (vind *XXHash) String() string {
+	return vind.name
+}
+
+// Cost returns the cost of this index as 1.
+func (vind *XXHash) Cost() int {
+	return 1
+}
+
+// IsUnique returns true since the Vindex is unique.
+func (vind *XXHash) IsUnique() bool {
+	return true
+}
+
+// Map can map ids to key.Destination objects.
+func (vind *XXHash) Map(cursor VCursor, ids []sqltypes.Value) ([]key.Destination, error) {
+	out := make([]key.Destination, len(ids))
+	for i := range ids {
+		id := ids[i].ToBytes()
+		out[i] = key.DestinationKeyspaceID(vXXHash(id))
+	}
+	return out, nil
+}
+
+// Verify returns true if ids maps to ksids.
+func (vind *XXHash) Verify(_ VCursor, ids []sqltypes.Value, ksids [][]byte) ([]bool, error) {
+	out := make([]bool, len(ids))
+	for i := range ids {
+		id := ids[i].ToBytes()
+		out[i] = bytes.Equal(vXXHash(id), ksids[i])
+	}
+	return out, nil
+}
+
+func init() {
+	Register("xxhash", NewXXHash)
+}
+
+func vXXHash(shardKey []byte) []byte {
+	var hashed [8]byte
+	hashKey := xxhash.Sum64(shardKey)
+	binary.LittleEndian.PutUint64(hashed[:], hashKey)
+	return hashed[:]
+}

go/vt/vtgate/vindexes/xxhash_test.go

@@ -0,0 +1,143 @@
+/*
+Copyright 2019 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vindexes
+
+import (
+	"bytes"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/cespare/xxhash/v2"
+
+	"vitess.io/vitess/go/sqltypes"
+	"vitess.io/vitess/go/vt/key"
+)
+
+var xxHash Vindex
+
+func init() {
+	hv, err := CreateVindex("xxhash", "xxhash_name", map[string]string{"Table": "t", "Column": "c"})
+	if err != nil {
+		panic(err)
+	}
+	xxHash = hv
+}
+
+func TestXXHashCost(t *testing.T) {
+	if xxHash.Cost() != 1 {
+		t.Errorf("Cost(): %d, want 1", xxHash.Cost())
+	}
+}
+
+func TestXXHashString(t *testing.T) {
+	if strings.Compare("xxhash_name", xxHash.String()) != 0 {
+		t.Errorf("String(): %s, want xxhash_name", xxHash.String())
+	}
+}
+
+func TestXXHashMap(t *testing.T) {
+	tcases := []struct {
+		in  sqltypes.Value
+		out []byte
+	}{{
+		in:  sqltypes.NewVarChar("test1"),
+		out: []byte{0xd0, 0x1a, 0xb7, 0xe4, 0xd6, 0x97, 0x8f, 0xb},
+	}, {
+		in:  sqltypes.NewVarChar("test2"),
+		out: []byte{0x87, 0xeb, 0x11, 0x71, 0x4c, 0xa, 0xe, 0x89},
+	}, {
+		in:  sqltypes.NewInt64(1),
+		out: []byte{0xd4, 0x64, 0x5, 0x36, 0x76, 0x12, 0xb4, 0xb7},
+	}, {
+		in:  sqltypes.NULL,
+		out: []byte{0x99, 0xe9, 0xd8, 0x51, 0x37, 0xdb, 0x46, 0xef},
+	}, {
+		in:  sqltypes.NewInt64(-1),
+		out: []byte{0xd8, 0xe2, 0xa6, 0xa7, 0xc8, 0xc7, 0x62, 0x3d},
+	}, {
+		in:  sqltypes.NewUint64(18446744073709551615),
+		out: []byte{0x47, 0x7c, 0xfa, 0x8d, 0x6d, 0x8f, 0x1f, 0x8d},
+	}, {
+		in:  sqltypes.NewInt64(9223372036854775807),
+		out: []byte{0xb3, 0x7e, 0xb0, 0x1f, 0x7b, 0xff, 0xaf, 0xd8},
+	}, {
+		in:  sqltypes.NewUint64(9223372036854775807),
+		out: []byte{0xb3, 0x7e, 0xb0, 0x1f, 0x7b, 0xff, 0xaf, 0xd8},
+	}, {
+		in:  sqltypes.NewInt64(-9223372036854775808),
+		out: []byte{0x10, 0x2c, 0x27, 0xdd, 0xb2, 0x6a, 0x60, 0x9e},
+	}}
+
+	for _, tcase := range tcases {
+		got, err := xxHash.Map(nil, []sqltypes.Value{tcase.in})
+		if err != nil {
+			t.Error(err)
+		}
+		out := []byte(got[0].(key.DestinationKeyspaceID))
+		if !bytes.Equal(tcase.out, out) {
+			t.Errorf("Map(%#v): %#v, want %#v", tcase.in, out, tcase.out)
+		}
+	}
+}
+
+func TestXXHashVerify(t *testing.T) {
+	ids := []sqltypes.Value{sqltypes.NewUint64(1), sqltypes.NewUint64(2)}
+	ksids := [][]byte{{0xd4, 0x64, 0x5, 0x36, 0x76, 0x12, 0xb4, 0xb7}, {0xd4, 0x64, 0x5, 0x36, 0x76, 0x12, 0xb4, 0xb7}}
+	got, err := xxHash.Verify(nil, ids, ksids)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []bool{true, false}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("xxHash.Verify: %v, want %v", got, want)
+	}
+}
+
+func BenchmarkXXHash(b *testing.B) {
+	for _, benchSize := range []struct {
+		name string
+		n    int
+	}{
+		{"8B", 8},
+		{"64B", 64},
+		{"512B", 512},
+		{"1KB", 1e3},
+		{"4KB", 4e3},
+	} {
+		input := make([]byte, benchSize.n)
+		for i := range input {
+			input[i] = byte(i)
+		}
+
+		name := fmt.Sprintf("xxHash,direct,bytes,n=%s", benchSize.name)
+		b.Run(name, func(b *testing.B) {
+			benchmarkHashBytes(b, input)
+		})
+
+	}
+}
+
+var sink uint64
+
+func benchmarkHashBytes(b *testing.B, input []byte) {
+	b.SetBytes(int64(len(input)))
+	for i := 0; i < b.N; i++ {
+		sink = xxhash.Sum64(input)
+	}
+}