diff --git a/go.mod b/go.mod index ae3b4d9aee4..6707d58528a 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect github.com/aws/aws-sdk-go v0.0.0-20180223184012-ebef4262e06a github.com/boltdb/bolt v1.3.1 // indirect + github.com/cespare/xxhash/v2 v2.1.1 github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292 // indirect github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect github.com/coreos/etcd v0.0.0-20170626015032-703663d1f6ed diff --git a/go.sum b/go.sum index ff202019b03..df4e4fb847c 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,9 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= diff --git a/go/vt/vtgate/vindexes/xxhash.go b/go/vt/vtgate/vindexes/xxhash.go new file mode 100644 index 00000000000..48677a736f5 --- /dev/null +++ b/go/vt/vtgate/vindexes/xxhash.go @@ -0,0 +1,88 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vindexes + +import ( + "bytes" + "encoding/binary" + + "github.com/cespare/xxhash/v2" + + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/key" +) + +var ( + _ Vindex = (*XXHash)(nil) +) + +// XXHash defines vindex that hashes any sql types to a KeyspaceId +// by using xxhash64. It's Unique and works on any platform giving identical result. +type XXHash struct { + name string +} + +// NewXXHash creates a new XXHash. +func NewXXHash(name string, m map[string]string) (Vindex, error) { + return &XXHash{name: name}, nil +} + +// String returns the name of the vindex. +func (vind *XXHash) String() string { + return vind.name +} + +// Cost returns the cost of this index as 1. +func (vind *XXHash) Cost() int { + return 1 +} + +// IsUnique returns true since the Vindex is unique. +func (vind *XXHash) IsUnique() bool { + return true +} + +// Map can map ids to key.Destination objects. +func (vind *XXHash) Map(cursor VCursor, ids []sqltypes.Value) ([]key.Destination, error) { + out := make([]key.Destination, len(ids)) + for i := range ids { + id := ids[i].ToBytes() + out[i] = key.DestinationKeyspaceID(vXXHash(id)) + } + return out, nil +} + +// Verify returns true if ids maps to ksids. +func (vind *XXHash) Verify(_ VCursor, ids []sqltypes.Value, ksids [][]byte) ([]bool, error) { + out := make([]bool, len(ids)) + for i := range ids { + id := ids[i].ToBytes() + out[i] = bytes.Equal(vXXHash(id), ksids[i]) + } + return out, nil +} + +func init() { + Register("xxhash", NewXXHash) +} + +func vXXHash(shardKey []byte) []byte { + var hashed [8]byte + hashKey := xxhash.Sum64(shardKey) + binary.LittleEndian.PutUint64(hashed[:], hashKey) + return hashed[:] +} diff --git a/go/vt/vtgate/vindexes/xxhash_test.go b/go/vt/vtgate/vindexes/xxhash_test.go new file mode 100644 index 00000000000..36c1719443c --- /dev/null +++ b/go/vt/vtgate/vindexes/xxhash_test.go @@ -0,0 +1,143 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vindexes + +import ( + "bytes" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/cespare/xxhash/v2" + + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/key" +) + +var xxHash Vindex + +func init() { + hv, err := CreateVindex("xxhash", "xxhash_name", map[string]string{"Table": "t", "Column": "c"}) + if err != nil { + panic(err) + } + xxHash = hv +} + +func TestXXHashCost(t *testing.T) { + if xxHash.Cost() != 1 { + t.Errorf("Cost(): %d, want 1", xxHash.Cost()) + } +} + +func TestXXHashString(t *testing.T) { + if strings.Compare("xxhash_name", xxHash.String()) != 0 { + t.Errorf("String(): %s, want xxhash_name", xxHash.String()) + } +} + +func TestXXHashMap(t *testing.T) { + tcases := []struct { + in sqltypes.Value + out []byte + }{{ + in: sqltypes.NewVarChar("test1"), + out: []byte{0xd0, 0x1a, 0xb7, 0xe4, 0xd6, 0x97, 0x8f, 0xb}, + }, { + in: sqltypes.NewVarChar("test2"), + out: []byte{0x87, 0xeb, 0x11, 0x71, 0x4c, 0xa, 0xe, 0x89}, + }, { + in: sqltypes.NewInt64(1), + out: []byte{0xd4, 0x64, 0x5, 0x36, 0x76, 0x12, 0xb4, 0xb7}, + }, { + in: sqltypes.NULL, + out: []byte{0x99, 0xe9, 0xd8, 0x51, 0x37, 0xdb, 0x46, 0xef}, + }, { + in: sqltypes.NewInt64(-1), + out: []byte{0xd8, 0xe2, 0xa6, 0xa7, 0xc8, 0xc7, 0x62, 0x3d}, + }, { + in: sqltypes.NewUint64(18446744073709551615), + out: []byte{0x47, 0x7c, 0xfa, 0x8d, 0x6d, 0x8f, 0x1f, 0x8d}, + }, { + in: sqltypes.NewInt64(9223372036854775807), + out: []byte{0xb3, 0x7e, 0xb0, 0x1f, 0x7b, 0xff, 0xaf, 0xd8}, + }, { + in: sqltypes.NewUint64(9223372036854775807), + out: []byte{0xb3, 0x7e, 0xb0, 0x1f, 0x7b, 0xff, 0xaf, 0xd8}, + }, { + in: sqltypes.NewInt64(-9223372036854775808), + out: []byte{0x10, 0x2c, 0x27, 0xdd, 0xb2, 0x6a, 0x60, 0x9e}, + }} + + for _, tcase := range tcases { + got, err := xxHash.Map(nil, []sqltypes.Value{tcase.in}) + if err != nil { + t.Error(err) + } + out := []byte(got[0].(key.DestinationKeyspaceID)) + if !bytes.Equal(tcase.out, out) { + t.Errorf("Map(%#v): %#v, want %#v", tcase.in, out, tcase.out) + } + } +} + +func TestXXHashVerify(t *testing.T) { + ids := []sqltypes.Value{sqltypes.NewUint64(1), sqltypes.NewUint64(2)} + ksids := [][]byte{{0xd4, 0x64, 0x5, 0x36, 0x76, 0x12, 0xb4, 0xb7}, {0xd4, 0x64, 0x5, 0x36, 0x76, 0x12, 0xb4, 0xb7}} + got, err := xxHash.Verify(nil, ids, ksids) + if err != nil { + t.Fatal(err) + } + want := []bool{true, false} + if !reflect.DeepEqual(got, want) { + t.Errorf("xxHash.Verify: %v, want %v", got, want) + } +} + +func BenchmarkXXHash(b *testing.B) { + for _, benchSize := range []struct { + name string + n int + }{ + {"8B", 8}, + {"64B", 64}, + {"512B", 512}, + {"1KB", 1e3}, + {"4KB", 4e3}, + } { + input := make([]byte, benchSize.n) + for i := range input { + input[i] = byte(i) + } + + name := fmt.Sprintf("xxHash,direct,bytes,n=%s", benchSize.name) + b.Run(name, func(b *testing.B) { + benchmarkHashBytes(b, input) + }) + + } +} + +var sink uint64 + +func benchmarkHashBytes(b *testing.B, input []byte) { + b.SetBytes(int64(len(input))) + for i := 0; i < b.N; i++ { + sink = xxhash.Sum64(input) + } +}