From c1a3a0d5396d65c974059268db31fd757c93cb56 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Wed, 14 Feb 2018 18:22:04 -0800 Subject: [PATCH 1/3] v3: keyrange support for unique vindexes There are use cases where a unique vindex may not know the target keyspace id. If so, we should allow it to return a keyrange, which will allow the vtgate engine to scatter the query. The unique vindex property is still useful for the optimizer because it can still use the unique property to deduce within-shard joins. This feature may not be 100% relational, but it satisfies certain use cases that would otherwise become very inefficient. --- go/vt/binlog/keyspace_id_resolver.go | 4 +- go/vt/vtgate/engine/route.go | 35 ++++++++++---- go/vt/vtgate/engine/vindex_func.go | 8 +++- go/vt/vtgate/engine/vindex_func_test.go | 47 +++++++++++++++---- go/vt/vtgate/executor.go | 9 +++- go/vt/vtgate/executor_dml_test.go | 33 +++++++++++++ go/vt/vtgate/executor_framework_test.go | 33 +++++++++++-- go/vt/vtgate/executor_select_test.go | 24 +++++++++- go/vt/vtgate/executor_test.go | 3 +- go/vt/vtgate/planbuilder/plan_test.go | 4 +- go/vt/vtgate/vindexes/binary.go | 6 +-- go/vt/vtgate/vindexes/binary_test.go | 2 +- go/vt/vtgate/vindexes/binarymd5.go | 6 +-- go/vt/vtgate/vindexes/binarymd5_test.go | 4 +- go/vt/vtgate/vindexes/hash.go | 8 ++-- go/vt/vtgate/vindexes/hash_test.go | 16 +++---- go/vt/vtgate/vindexes/lookup.go | 8 ++-- go/vt/vtgate/vindexes/lookup_hash.go | 10 ++-- .../vindexes/lookup_hash_unique_test.go | 10 ++-- go/vt/vtgate/vindexes/lookup_unique_test.go | 8 ++-- go/vt/vtgate/vindexes/numeric.go | 8 ++-- go/vt/vtgate/vindexes/numeric_static_map.go | 8 ++-- .../vindexes/numeric_static_map_test.go | 20 ++++---- go/vt/vtgate/vindexes/numeric_test.go | 20 ++++---- go/vt/vtgate/vindexes/unicodeloosemd5.go | 6 +-- go/vt/vtgate/vindexes/unicodeloosemd5_test.go | 2 +- go/vt/vtgate/vindexes/vindex.go | 18 ++++++- go/vt/vtgate/vindexes/vschema_test.go | 4 +- go/vt/worker/key_resolver.go | 4 +- 29 files changed, 261 insertions(+), 107 deletions(-) diff --git a/go/vt/binlog/keyspace_id_resolver.go b/go/vt/binlog/keyspace_id_resolver.go index d128d2554db..b174aa123eb 100644 --- a/go/vt/binlog/keyspace_id_resolver.go +++ b/go/vt/binlog/keyspace_id_resolver.go @@ -178,8 +178,8 @@ func (r *keyspaceIDResolverFactoryV3) keyspaceID(v sqltypes.Value) ([]byte, erro if len(ksids) != 1 { return nil, fmt.Errorf("mapping row to keyspace id returned an invalid array of keyspace ids: %v", ksids) } - if ksids[0] == nil { + if ksids[0].Range != nil || ksids[0].ID == nil { return nil, fmt.Errorf("could not map %v to a keyspace id", v) } - return ksids[0], nil + return ksids[0].ID, nil } diff --git a/go/vt/vtgate/engine/route.go b/go/vt/vtgate/engine/route.go index bbaca89da67..3c73c67c998 100644 --- a/go/vt/vtgate/engine/route.go +++ b/go/vt/vtgate/engine/route.go @@ -612,15 +612,25 @@ func (route *Route) resolveShards(vcursor VCursor, bindVars map[string]*querypb. if err != nil { return "", nil, err } + var shards []string for i, ksid := range ksids { - if ksid == nil { - continue + switch { + case ksid.Range != nil: + // Use the multi-keyspace id API to convert a keyrange to shards. + shards, err = vcursor.GetShardsForKsids(allShards, vindexes.Ksids{Range: ksid.Range}) + if err != nil { + return "", nil, err + } + case ksid.ID != nil: + shard, err := vcursor.GetShardForKeyspaceID(allShards, ksid.ID) + if err != nil { + return "", nil, err + } + shards = []string{shard} } - shard, err := vcursor.GetShardForKeyspaceID(allShards, ksid) - if err != nil { - return "", nil, err + for _, shard := range shards { + routing.Add(shard, sqltypes.ValueToProto(vindexKeys[i])) } - routing.Add(shard, sqltypes.ValueToProto(vindexKeys[i])) } case vindexes.NonUnique: ksidss, err := mapper.Map(vcursor, vindexKeys) @@ -652,7 +662,10 @@ func (route *Route) resolveSingleShard(vcursor VCursor, bindVars map[string]*que if err != nil { return "", "", nil, err } - ksid = ksids[0] + if err := ksids[0].ValidateUnique(); err != nil { + return "", "", nil, err + } + ksid = ksids[0].ID if ksid == nil { return "", "", ksid, nil } @@ -875,10 +888,16 @@ func (route *Route) processPrimary(vcursor VCursor, vindexKeys [][]sqltypes.Valu flattenedVidexKeys = append(flattenedVidexKeys, internalVal) } } - keyspaceIDs, err = mapper.Map(vcursor, flattenedVidexKeys) + ksids, err := mapper.Map(vcursor, flattenedVidexKeys) if err != nil { return nil, err } + for _, ksid := range ksids { + if err := ksid.ValidateUnique(); err != nil { + return nil, err + } + keyspaceIDs = append(keyspaceIDs, ksid.ID) + } for rowNum, vindexKey := range flattenedVidexKeys { if keyspaceIDs[rowNum] == nil { diff --git a/go/vt/vtgate/engine/vindex_func.go b/go/vt/vtgate/engine/vindex_func.go index 9aa8ded99fb..ecb08fe4a60 100644 --- a/go/vt/vtgate/engine/vindex_func.go +++ b/go/vt/vtgate/engine/vindex_func.go @@ -120,9 +120,13 @@ func (vf *VindexFunc) mapVindex(vcursor VCursor, bindVars, joinVars map[string]* if err != nil { return nil, err } - if ksids[0] != nil { + switch { + case ksids[0].Range != nil: + result.Rows = append(result.Rows, vf.buildRow(vkey, nil, ksids[0].Range)) + result.RowsAffected = 1 + case ksids[0].ID != nil: result.Rows = [][]sqltypes.Value{ - vf.buildRow(vkey, ksids[0], nil), + vf.buildRow(vkey, ksids[0].ID, nil), } result.RowsAffected = 1 } diff --git a/go/vt/vtgate/engine/vindex_func_test.go b/go/vt/vtgate/engine/vindex_func_test.go index 0184b801470..c31db34f84e 100644 --- a/go/vt/vtgate/engine/vindex_func_test.go +++ b/go/vt/vtgate/engine/vindex_func_test.go @@ -21,12 +21,13 @@ import ( "testing" "github.com/youtube/vitess/go/sqltypes" - topodatapb "github.com/youtube/vitess/go/vt/proto/topodata" "github.com/youtube/vitess/go/vt/vtgate/vindexes" + + topodatapb "github.com/youtube/vitess/go/vt/proto/topodata" ) // uvindex is Unique. -type uvindex struct{ match bool } +type uvindex struct{ matchid, matchkr bool } func (*uvindex) String() string { return "uvindex" } func (*uvindex) Cost() int { return 1 } @@ -34,13 +35,21 @@ func (*uvindex) Verify(vindexes.VCursor, []sqltypes.Value, [][]byte) ([]bool, er panic("unimplemented") } -func (v *uvindex) Map(vindexes.VCursor, []sqltypes.Value) ([][]byte, error) { - if v.match { - return [][]byte{ - []byte("foo"), +func (v *uvindex) Map(vindexes.VCursor, []sqltypes.Value) ([]vindexes.Ksid, error) { + if v.matchkr { + return []vindexes.Ksid{{ + Range: &topodatapb.KeyRange{ + Start: []byte{0x40}, + End: []byte{0x60}, + }, + }}, nil + } + if v.matchid { + return []vindexes.Ksid{ + {ID: []byte("foo")}, }, nil } - return [][]byte{nil}, nil + return []vindexes.Ksid{{}}, nil } // nvindex is NonUnique. @@ -87,7 +96,7 @@ func TestVindexFuncMap(t *testing.T) { } // Unique Vindex returning 1 row. - vf = testVindexFunc(&uvindex{match: true}) + vf = testVindexFunc(&uvindex{matchid: true}) got, err = vf.Execute(nil, nil, nil, false) if err != nil { t.Fatal(err) @@ -100,6 +109,26 @@ func TestVindexFuncMap(t *testing.T) { t.Errorf("Execute(Map, uvindex(none)):\n%v, want\n%v", got, want) } + // Unique Vindex returning keyrange. + vf = testVindexFunc(&uvindex{matchkr: true}) + got, err = vf.Execute(nil, nil, nil, false) + if err != nil { + t.Fatal(err) + } + want = &sqltypes.Result{ + Fields: sqltypes.MakeTestFields("id|keyspace_id|range_start|range_end", "varbinary|varbinary|varbinary|varbinary"), + Rows: [][]sqltypes.Value{{ + sqltypes.NewVarBinary("1"), + sqltypes.NULL, + sqltypes.MakeTrusted(sqltypes.VarBinary, []byte{0x40}), + sqltypes.MakeTrusted(sqltypes.VarBinary, []byte{0x60}), + }}, + RowsAffected: 1, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("Execute(Map, uvindex(none)):\n%v, want\n%v", got, want) + } + // NonUnique Vindex returning 0 rows. vf = testVindexFunc(&nvindex{}) got, err = vf.Execute(nil, nil, nil, false) @@ -179,7 +208,7 @@ func TestVindexFuncStreamExecute(t *testing.T) { } func TestVindexFuncGetFields(t *testing.T) { - vf := testVindexFunc(&uvindex{match: true}) + vf := testVindexFunc(&uvindex{matchid: true}) got, err := vf.GetFields(nil, nil, nil) if err != nil { t.Fatal(err) diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index 69a29ca8b9b..94d86c1739d 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -902,7 +902,14 @@ func (e *Executor) MessageAck(ctx context.Context, keyspace, name string, ids [] if err != nil { return 0, err } - rss, rssValues, err = e.resolver.resolver.ResolveKeyspaceIdsValues(ctx, table.Keyspace.Name, ids, ksids, topodatapb.TabletType_MASTER) + keyspaceids := make([][]byte, len(ksids)) + for i, ksid := range ksids { + if err := ksid.ValidateUnique(); err != nil { + return 0, err + } + keyspaceids[i] = ksid.ID + } + rss, rssValues, err = e.resolver.resolver.ResolveKeyspaceIdsValues(ctx, table.Keyspace.Name, ids, keyspaceids, topodatapb.TabletType_MASTER) if err != nil { return 0, err } diff --git a/go/vt/vtgate/executor_dml_test.go b/go/vt/vtgate/executor_dml_test.go index 3dff0ab4082..0c7f4e7c5b2 100644 --- a/go/vt/vtgate/executor_dml_test.go +++ b/go/vt/vtgate/executor_dml_test.go @@ -141,6 +141,17 @@ func TestUpdateEqual(t *testing.T) { } } +func TestUpdateEqualKeyrange(t *testing.T) { + executor, _, _, _ := createExecutorEnv() + + // If a unique vindex returns a keyrange, we fail the update + _, err := executorExec(executor, "update keyrange_table set a=2 where krcol_unique = 1", nil) + want := "execUpdateEqual: vindex could not map the value to a unique keyspace id" + if err == nil || err.Error() != want { + t.Errorf("executorExec error: %v, want %s", err, want) + } +} + func TestUpdateMultiOwned(t *testing.T) { vschema := ` { @@ -491,6 +502,17 @@ func TestDeleteEqual(t *testing.T) { } } +func TestDeleteEqualKeyrange(t *testing.T) { + executor, _, _, _ := createExecutorEnv() + + // If a unique vindex returns a keyrange, we fail the delete + _, err := executorExec(executor, "delete from keyrange_table where krcol_unique = 1", nil) + want := "execDeleteEqual: vindex could not map the value to a unique keyspace id" + if err == nil || err.Error() != want { + t.Errorf("executorExec error: %v, want %s", err, want) + } +} + func TestDeleteSharded(t *testing.T) { executor, sbc1, sbc2, _ := createExecutorEnv() _, err := executorExec(executor, "delete from user_extra", nil) @@ -667,6 +689,17 @@ func TestInsertSharded(t *testing.T) { } } +func TestInsertShardedKeyrange(t *testing.T) { + executor, _, _, _ := createExecutorEnv() + + // If a unique vindex returns a keyrange, we fail the insert + _, err := executorExec(executor, "insert into keyrange_table(krcol_unique, krcol) values(1, 1)", nil) + want := "execInsertSharded: getInsertShardedRoute: vindex could not map the value to a unique keyspace id" + if err == nil || err.Error() != want { + t.Errorf("executorExec error: %v, want %s", err, want) + } +} + func TestInsertShardedAutocommitLookup(t *testing.T) { vschema := ` diff --git a/go/vt/vtgate/executor_framework_test.go b/go/vt/vtgate/executor_framework_test.go index b1e103fa629..2655145e0e2 100644 --- a/go/vt/vtgate/executor_framework_test.go +++ b/go/vt/vtgate/executor_framework_test.go @@ -87,6 +87,9 @@ var executorVSchema = ` "keyspace_id": { "type": "numeric" }, + "krcol_unique_vdx": { + "type": "keyrange_lookuper_unique" + }, "krcol_vdx": { "type": "keyrange_lookuper" } @@ -209,8 +212,8 @@ var executorVSchema = ` "keyrange_table": { "column_vindexes": [ { - "column": "id", - "name": "hash_index" + "column": "krcol_unique", + "name": "krcol_unique_vdx" }, { "column": "krcol", @@ -279,12 +282,34 @@ func (*keyRangeLookuper) Map(vindexes.VCursor, []sqltypes.Value) ([]vindexes.Ksi }}, nil } -func newLookupMigrator(name string, params map[string]string) (vindexes.Vindex, error) { +func newKeyRangeLookuper(name string, params map[string]string) (vindexes.Vindex, error) { return &keyRangeLookuper{}, nil } +// keyRangeLookuperUnique is for testing a unique lookup that returns a keyrange. +type keyRangeLookuperUnique struct { +} + +func (v *keyRangeLookuperUnique) String() string { return "keyrange_lookuper" } +func (*keyRangeLookuperUnique) Cost() int { return 0 } +func (*keyRangeLookuperUnique) Verify(vindexes.VCursor, []sqltypes.Value, [][]byte) ([]bool, error) { + return []bool{}, nil +} +func (*keyRangeLookuperUnique) Map(vindexes.VCursor, []sqltypes.Value) ([]vindexes.Ksid, error) { + return []vindexes.Ksid{{ + Range: &topodatapb.KeyRange{ + End: []byte{0x10}, + }, + }}, nil +} + +func newKeyRangeLookuperUnique(name string, params map[string]string) (vindexes.Vindex, error) { + return &keyRangeLookuperUnique{}, nil +} + func init() { - vindexes.Register("keyrange_lookuper", newLookupMigrator) + vindexes.Register("keyrange_lookuper", newKeyRangeLookuper) + vindexes.Register("keyrange_lookuper_unique", newKeyRangeLookuperUnique) } const testBufferSize = 10 diff --git a/go/vt/vtgate/executor_select_test.go b/go/vt/vtgate/executor_select_test.go index e0b2e945e32..04ec99c7297 100644 --- a/go/vt/vtgate/executor_select_test.go +++ b/go/vt/vtgate/executor_select_test.go @@ -586,12 +586,32 @@ func TestStreamSelectEqual(t *testing.T) { func TestSelectKeyRange(t *testing.T) { executor, sbc1, sbc2, _ := createExecutorEnv() - _, err := executorExec(executor, "select id, krcol from keyrange_table where krcol = 1", nil) + _, err := executorExec(executor, "select krcol_unique, krcol from keyrange_table where krcol = 1", nil) if err != nil { t.Error(err) } wantQueries := []*querypb.BoundQuery{{ - Sql: "select id, krcol from keyrange_table where krcol = 1", + Sql: "select krcol_unique, krcol from keyrange_table where krcol = 1", + BindVariables: map[string]*querypb.BindVariable{}, + }} + if !reflect.DeepEqual(sbc1.Queries, wantQueries) { + t.Errorf("sbc1.Queries: %+v, want %+v\n", sbc1.Queries, wantQueries) + } + if sbc2.Queries != nil { + t.Errorf("sbc2.Queries: %+v, want nil\n", sbc2.Queries) + } + sbc1.Queries = nil +} + +func TestSelectKeyRangeUnique(t *testing.T) { + executor, sbc1, sbc2, _ := createExecutorEnv() + + _, err := executorExec(executor, "select krcol_unique, krcol from keyrange_table where krcol_unique = 1", nil) + if err != nil { + t.Error(err) + } + wantQueries := []*querypb.BoundQuery{{ + Sql: "select krcol_unique, krcol from keyrange_table where krcol_unique = 1", BindVariables: map[string]*querypb.BindVariable{}, }} if !reflect.DeepEqual(sbc1.Queries, wantQueries) { diff --git a/go/vt/vtgate/executor_test.go b/go/vt/vtgate/executor_test.go index 7639840fc47..2cdc7ee1a3a 100644 --- a/go/vt/vtgate/executor_test.go +++ b/go/vt/vtgate/executor_test.go @@ -580,12 +580,13 @@ func TestExecutorShow(t *testing.T) { buildVarCharRow("TestExecutor", "idx_noauto", "hash", "", "noauto_table"), buildVarCharRow("TestExecutor", "insert_ignore_idx", "lookup_hash", "from=fromcol; table=ins_lookup; to=tocol", "insert_ignore_test"), buildVarCharRow("TestExecutor", "keyspace_id", "numeric", "", ""), + buildVarCharRow("TestExecutor", "krcol_unique_vdx", "keyrange_lookuper_unique", "", ""), buildVarCharRow("TestExecutor", "krcol_vdx", "keyrange_lookuper", "", ""), buildVarCharRow("TestExecutor", "music_user_map", "lookup_hash_unique", "from=music_id; table=music_user_map; to=user_id", "music"), buildVarCharRow("TestExecutor", "name_lastname_keyspace_id_map", "lookup", "from=name,lastname; table=name_lastname_keyspace_id_map; to=keyspace_id", "user2"), buildVarCharRow("TestExecutor", "name_user_map", "lookup_hash", "from=name; table=name_user_map; to=user_id", "user"), }, - RowsAffected: 9, + RowsAffected: 10, } if !reflect.DeepEqual(qr, wantqr) { t.Errorf("show vindexes:\n%+v, want\n%+v", qr, wantqr) diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index 7893a2582be..557ec936b8b 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -42,7 +42,7 @@ func (*hashIndex) Cost() int { return 1 } func (*hashIndex) Verify(vindexes.VCursor, []sqltypes.Value, [][]byte) ([]bool, error) { return []bool{}, nil } -func (*hashIndex) Map(vindexes.VCursor, []sqltypes.Value) ([][]byte, error) { return nil, nil } +func (*hashIndex) Map(vindexes.VCursor, []sqltypes.Value) ([]vindexes.Ksid, error) { return nil, nil } func newHashIndex(name string, _ map[string]string) (vindexes.Vindex, error) { return &hashIndex{name: name}, nil @@ -58,7 +58,7 @@ func (*lookupIndex) Cost() int { return 2 } func (*lookupIndex) Verify(vindexes.VCursor, []sqltypes.Value, [][]byte) ([]bool, error) { return []bool{}, nil } -func (*lookupIndex) Map(vindexes.VCursor, []sqltypes.Value) ([][]byte, error) { return nil, nil } +func (*lookupIndex) Map(vindexes.VCursor, []sqltypes.Value) ([]vindexes.Ksid, error) { return nil, nil } func (*lookupIndex) Create(vindexes.VCursor, [][]sqltypes.Value, [][]byte, bool) error { return nil } func (*lookupIndex) Delete(vindexes.VCursor, [][]sqltypes.Value, []byte) error { return nil } func (*lookupIndex) Update(vindexes.VCursor, []sqltypes.Value, []byte, []sqltypes.Value) error { diff --git a/go/vt/vtgate/vindexes/binary.go b/go/vt/vtgate/vindexes/binary.go index cf9ceab9f5a..d4692214680 100644 --- a/go/vt/vtgate/vindexes/binary.go +++ b/go/vt/vtgate/vindexes/binary.go @@ -58,10 +58,10 @@ func (vind *Binary) Verify(_ VCursor, ids []sqltypes.Value, ksids [][]byte) ([]b } // Map returns the corresponding keyspace id values for the given ids. -func (vind *Binary) Map(_ VCursor, ids []sqltypes.Value) ([][]byte, error) { - out := make([][]byte, 0, len(ids)) +func (vind *Binary) Map(_ VCursor, ids []sqltypes.Value) ([]Ksid, error) { + out := make([]Ksid, 0, len(ids)) for _, id := range ids { - out = append(out, id.ToBytes()) + out = append(out, Ksid{ID: id.ToBytes()}) } return out, nil } diff --git a/go/vt/vtgate/vindexes/binary_test.go b/go/vt/vtgate/vindexes/binary_test.go index 0c98d0c4b81..85f6e7208cb 100644 --- a/go/vt/vtgate/vindexes/binary_test.go +++ b/go/vt/vtgate/vindexes/binary_test.go @@ -59,7 +59,7 @@ func TestBinaryMap(t *testing.T) { if err != nil { t.Error(err) } - out := []byte(got[0]) + out := []byte(got[0].ID) if bytes.Compare(tcase.out, out) != 0 { t.Errorf("Map(%#v): %#v, want %#v", tcase.in, out, tcase.out) } diff --git a/go/vt/vtgate/vindexes/binarymd5.go b/go/vt/vtgate/vindexes/binarymd5.go index cef2b7b2f92..b8f0b848ea2 100644 --- a/go/vt/vtgate/vindexes/binarymd5.go +++ b/go/vt/vtgate/vindexes/binarymd5.go @@ -57,10 +57,10 @@ func (vind *BinaryMD5) Verify(_ VCursor, ids []sqltypes.Value, ksids [][]byte) ( } // Map returns the corresponding keyspace id values for the given ids. -func (vind *BinaryMD5) Map(_ VCursor, ids []sqltypes.Value) ([][]byte, error) { - out := make([][]byte, 0, len(ids)) +func (vind *BinaryMD5) Map(_ VCursor, ids []sqltypes.Value) ([]Ksid, error) { + out := make([]Ksid, 0, len(ids)) for _, id := range ids { - out = append(out, binHash(id.ToBytes())) + out = append(out, Ksid{ID: binHash(id.ToBytes())}) } return out, nil } diff --git a/go/vt/vtgate/vindexes/binarymd5_test.go b/go/vt/vtgate/vindexes/binarymd5_test.go index db874c1a4c2..dfcf2b54aaa 100644 --- a/go/vt/vtgate/vindexes/binarymd5_test.go +++ b/go/vt/vtgate/vindexes/binarymd5_test.go @@ -61,7 +61,7 @@ func TestBinaryMD5Map(t *testing.T) { if err != nil { t.Error(err) } - out := string(got[0]) + out := string(got[0].ID) if out != tcase.out { t.Errorf("Map(%#v): %#v, want %#v", tcase.in, out, tcase.out) } @@ -87,7 +87,7 @@ func TestSQLValue(t *testing.T) { if err != nil { t.Error(err) } - out := string(got[0]) + out := string(got[0].ID) want := "\f\xbcf\x11\xf5T\vЀ\x9a8\x8d\xc9Za[" if out != want { t.Errorf("Map(%#v): %#v, want %#v", val, out, want) diff --git a/go/vt/vtgate/vindexes/hash.go b/go/vt/vtgate/vindexes/hash.go index c4ff60f4aaa..8ffaed3124d 100644 --- a/go/vt/vtgate/vindexes/hash.go +++ b/go/vt/vtgate/vindexes/hash.go @@ -55,15 +55,15 @@ func (vind *Hash) Cost() int { } // Map returns the corresponding KeyspaceId values for the given ids. -func (vind *Hash) Map(_ VCursor, ids []sqltypes.Value) ([][]byte, error) { - out := make([][]byte, 0, len(ids)) +func (vind *Hash) Map(_ VCursor, ids []sqltypes.Value) ([]Ksid, error) { + out := make([]Ksid, 0, len(ids)) for _, id := range ids { num, err := sqltypes.ToUint64(id) if err != nil { - out = append(out, nil) + out = append(out, Ksid{}) continue } - out = append(out, vhash(num)) + out = append(out, Ksid{ID: vhash(num)}) } return out, nil } diff --git a/go/vt/vtgate/vindexes/hash_test.go b/go/vt/vtgate/vindexes/hash_test.go index 3fede46991a..aa65c809cab 100644 --- a/go/vt/vtgate/vindexes/hash_test.go +++ b/go/vt/vtgate/vindexes/hash_test.go @@ -59,14 +59,14 @@ func TestHashMap(t *testing.T) { if err != nil { t.Error(err) } - want := [][]byte{ - []byte("\x16k@\xb4J\xbaK\xd6"), - []byte("\x06\xe7\xea\"Βp\x8f"), - []byte("N\xb1\x90ɢ\xfa\x16\x9c"), - nil, - []byte("\xd2\xfd\x88g\xd5\r-\xfe"), - []byte("p\xbb\x02<\x81\f\xa8z"), - []byte("\xf0\x98H\n\xc4ľq"), + want := []Ksid{ + {ID: []byte("\x16k@\xb4J\xbaK\xd6")}, + {ID: []byte("\x06\xe7\xea\"Βp\x8f")}, + {ID: []byte("N\xb1\x90ɢ\xfa\x16\x9c")}, + {ID: nil}, + {ID: []byte("\xd2\xfd\x88g\xd5\r-\xfe")}, + {ID: []byte("p\xbb\x02<\x81\f\xa8z")}, + {ID: []byte("\xf0\x98H\n\xc4ľq")}, } if !reflect.DeepEqual(got, want) { t.Errorf("Map(): %#v, want %+v", got, want) diff --git a/go/vt/vtgate/vindexes/lookup.go b/go/vt/vtgate/vindexes/lookup.go index 86f9622720a..eb817ea9f96 100644 --- a/go/vt/vtgate/vindexes/lookup.go +++ b/go/vt/vtgate/vindexes/lookup.go @@ -202,8 +202,8 @@ func (lu *LookupUnique) Cost() int { } // Map returns the corresponding KeyspaceId values for the given ids. -func (lu *LookupUnique) Map(vcursor VCursor, ids []sqltypes.Value) ([][]byte, error) { - out := make([][]byte, 0, len(ids)) +func (lu *LookupUnique) Map(vcursor VCursor, ids []sqltypes.Value) ([]Ksid, error) { + out := make([]Ksid, 0, len(ids)) results, err := lu.lkp.Lookup(vcursor, ids) if err != nil { return nil, err @@ -211,9 +211,9 @@ func (lu *LookupUnique) Map(vcursor VCursor, ids []sqltypes.Value) ([][]byte, er for i, result := range results { switch len(result.Rows) { case 0: - out = append(out, nil) + out = append(out, Ksid{}) case 1: - out = append(out, result.Rows[0][0].ToBytes()) + out = append(out, Ksid{ID: result.Rows[0][0].ToBytes()}) default: return nil, fmt.Errorf("Lookup.Map: unexpected multiple results from vindex %s: %v", lu.lkp.Table, ids[i]) } diff --git a/go/vt/vtgate/vindexes/lookup_hash.go b/go/vt/vtgate/vindexes/lookup_hash.go index 26797c5eea3..c0d74da152a 100644 --- a/go/vt/vtgate/vindexes/lookup_hash.go +++ b/go/vt/vtgate/vindexes/lookup_hash.go @@ -234,8 +234,8 @@ func (lhu *LookupHashUnique) Cost() int { } // Map returns the corresponding KeyspaceId values for the given ids. -func (lhu *LookupHashUnique) Map(vcursor VCursor, ids []sqltypes.Value) ([][]byte, error) { - out := make([][]byte, 0, len(ids)) +func (lhu *LookupHashUnique) Map(vcursor VCursor, ids []sqltypes.Value) ([]Ksid, error) { + out := make([]Ksid, 0, len(ids)) results, err := lhu.lkp.Lookup(vcursor, ids) if err != nil { return nil, err @@ -243,14 +243,14 @@ func (lhu *LookupHashUnique) Map(vcursor VCursor, ids []sqltypes.Value) ([][]byt for i, result := range results { switch len(result.Rows) { case 0: - out = append(out, nil) + out = append(out, Ksid{}) case 1: num, err := sqltypes.ToUint64(result.Rows[0][0]) if err != nil { - out = append(out, nil) + out = append(out, Ksid{}) continue } - out = append(out, vhash(num)) + out = append(out, Ksid{ID: vhash(num)}) default: return nil, fmt.Errorf("LookupHash.Map: unexpected multiple results from vindex %s: %v", lhu.lkp.Table, ids[i]) } diff --git a/go/vt/vtgate/vindexes/lookup_hash_unique_test.go b/go/vt/vtgate/vindexes/lookup_hash_unique_test.go index 80d900f7ee1..788b36a81f3 100644 --- a/go/vt/vtgate/vindexes/lookup_hash_unique_test.go +++ b/go/vt/vtgate/vindexes/lookup_hash_unique_test.go @@ -64,9 +64,9 @@ func TestLookupHashUniqueMap(t *testing.T) { if err != nil { t.Error(err) } - want := [][]byte{ - []byte("\x16k@\xb4J\xbaK\xd6"), - []byte("\x16k@\xb4J\xbaK\xd6"), + want := []Ksid{ + {ID: []byte("\x16k@\xb4J\xbaK\xd6")}, + {ID: []byte("\x16k@\xb4J\xbaK\xd6")}, } if !reflect.DeepEqual(got, want) { t.Errorf("Map(): %#v, want %+v", got, want) @@ -77,7 +77,7 @@ func TestLookupHashUniqueMap(t *testing.T) { if err != nil { t.Error(err) } - want = [][]byte{nil, nil} + want = []Ksid{{}, {}} if !reflect.DeepEqual(got, want) { t.Errorf("Map(): %#v, want %+v", got, want) } @@ -98,7 +98,7 @@ func TestLookupHashUniqueMap(t *testing.T) { if err != nil { t.Error(err) } - want = [][]byte{nil} + want = []Ksid{{}} if !reflect.DeepEqual(got, want) { t.Errorf("Map(): %#v, want %+v", got, want) } diff --git a/go/vt/vtgate/vindexes/lookup_unique_test.go b/go/vt/vtgate/vindexes/lookup_unique_test.go index 6a81cc8a1b0..fe825494d02 100644 --- a/go/vt/vtgate/vindexes/lookup_unique_test.go +++ b/go/vt/vtgate/vindexes/lookup_unique_test.go @@ -73,9 +73,9 @@ func TestLookupUniqueMap(t *testing.T) { if err != nil { t.Error(err) } - want := [][]byte{ - []byte("1"), - []byte("1"), + want := []Ksid{ + {ID: []byte("1")}, + {ID: []byte("1")}, } if !reflect.DeepEqual(got, want) { t.Errorf("Map(): %+v, want %+v", got, want) @@ -86,7 +86,7 @@ func TestLookupUniqueMap(t *testing.T) { if err != nil { t.Error(err) } - want = [][]byte{nil, nil} + want = []Ksid{{}, {}} if !reflect.DeepEqual(got, want) { t.Errorf("Map(): %#v, want %+v", got, want) } diff --git a/go/vt/vtgate/vindexes/numeric.go b/go/vt/vtgate/vindexes/numeric.go index a242be75036..322c651a8e5 100644 --- a/go/vt/vtgate/vindexes/numeric.go +++ b/go/vt/vtgate/vindexes/numeric.go @@ -66,17 +66,17 @@ func (*Numeric) Verify(_ VCursor, ids []sqltypes.Value, ksids [][]byte) ([]bool, } // Map returns the associated keyspace ids for the given ids. -func (*Numeric) Map(_ VCursor, ids []sqltypes.Value) ([][]byte, error) { - out := make([][]byte, 0, len(ids)) +func (*Numeric) Map(_ VCursor, ids []sqltypes.Value) ([]Ksid, error) { + out := make([]Ksid, 0, len(ids)) for _, id := range ids { num, err := sqltypes.ToUint64(id) if err != nil { - out = append(out, nil) + out = append(out, Ksid{}) continue } var keybytes [8]byte binary.BigEndian.PutUint64(keybytes[:], num) - out = append(out, keybytes[:]) + out = append(out, Ksid{ID: keybytes[:]}) } return out, nil } diff --git a/go/vt/vtgate/vindexes/numeric_static_map.go b/go/vt/vtgate/vindexes/numeric_static_map.go index 2290f98383a..47cc1f6074c 100644 --- a/go/vt/vtgate/vindexes/numeric_static_map.go +++ b/go/vt/vtgate/vindexes/numeric_static_map.go @@ -94,12 +94,12 @@ func (vind *NumericStaticMap) Verify(_ VCursor, ids []sqltypes.Value, ksids [][] } // Map returns the associated keyspace ids for the given ids. -func (vind *NumericStaticMap) Map(_ VCursor, ids []sqltypes.Value) ([][]byte, error) { - out := make([][]byte, 0, len(ids)) +func (vind *NumericStaticMap) Map(_ VCursor, ids []sqltypes.Value) ([]Ksid, error) { + out := make([]Ksid, 0, len(ids)) for _, id := range ids { num, err := sqltypes.ToUint64(id) if err != nil { - out = append(out, nil) + out = append(out, Ksid{}) continue } lookupNum, ok := vind.lookup[num] @@ -108,7 +108,7 @@ func (vind *NumericStaticMap) Map(_ VCursor, ids []sqltypes.Value) ([][]byte, er } var keybytes [8]byte binary.BigEndian.PutUint64(keybytes[:], num) - out = append(out, keybytes[:]) + out = append(out, Ksid{ID: keybytes[:]}) } return out, nil } diff --git a/go/vt/vtgate/vindexes/numeric_static_map_test.go b/go/vt/vtgate/vindexes/numeric_static_map_test.go index 2c7661ccb3a..a2acbcdeb5d 100644 --- a/go/vt/vtgate/vindexes/numeric_static_map_test.go +++ b/go/vt/vtgate/vindexes/numeric_static_map_test.go @@ -80,16 +80,16 @@ func TestNumericStaticMapMap(t *testing.T) { // in the third slice, we expect 2 instead of 3 as numeric_static_map_test.json // has 3 mapped to 2 - want := [][]byte{ - []byte("\x00\x00\x00\x00\x00\x00\x00\x01"), - []byte("\x00\x00\x00\x00\x00\x00\x00\x02"), - []byte("\x00\x00\x00\x00\x00\x00\x00\x02"), - nil, - []byte("\x00\x00\x00\x00\x00\x00\x00\x04"), - []byte("\x00\x00\x00\x00\x00\x00\x00\x05"), - []byte("\x00\x00\x00\x00\x00\x00\x00\x06"), - []byte("\x00\x00\x00\x00\x00\x00\x00\x07"), - []byte("\x00\x00\x00\x00\x00\x00\x00\x08"), + want := []Ksid{ + {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x01")}, + {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x02")}, + {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x02")}, + {ID: nil}, + {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x04")}, + {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x05")}, + {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x06")}, + {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x07")}, + {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x08")}, } if !reflect.DeepEqual(got, want) { t.Errorf("Map(): %+v, want %+v", got, want) diff --git a/go/vt/vtgate/vindexes/numeric_test.go b/go/vt/vtgate/vindexes/numeric_test.go index 6691b9aa399..2353cd5d9e1 100644 --- a/go/vt/vtgate/vindexes/numeric_test.go +++ b/go/vt/vtgate/vindexes/numeric_test.go @@ -58,16 +58,16 @@ func TestNumericMap(t *testing.T) { if err != nil { t.Error(err) } - want := [][]byte{ - []byte("\x00\x00\x00\x00\x00\x00\x00\x01"), - []byte("\x00\x00\x00\x00\x00\x00\x00\x02"), - []byte("\x00\x00\x00\x00\x00\x00\x00\x03"), - nil, - []byte("\x00\x00\x00\x00\x00\x00\x00\x04"), - []byte("\x00\x00\x00\x00\x00\x00\x00\x05"), - []byte("\x00\x00\x00\x00\x00\x00\x00\x06"), - []byte("\x00\x00\x00\x00\x00\x00\x00\x07"), - []byte("\x00\x00\x00\x00\x00\x00\x00\x08"), + want := []Ksid{ + {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x01")}, + {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x02")}, + {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x03")}, + {ID: nil}, + {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x04")}, + {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x05")}, + {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x06")}, + {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x07")}, + {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x08")}, } if !reflect.DeepEqual(got, want) { t.Errorf("Map(): %+v, want %+v", got, want) diff --git a/go/vt/vtgate/vindexes/unicodeloosemd5.go b/go/vt/vtgate/vindexes/unicodeloosemd5.go index 5832d836af8..1ea5ffd4ce3 100644 --- a/go/vt/vtgate/vindexes/unicodeloosemd5.go +++ b/go/vt/vtgate/vindexes/unicodeloosemd5.go @@ -70,14 +70,14 @@ func (vind *UnicodeLooseMD5) Verify(_ VCursor, ids []sqltypes.Value, ksids [][]b } // Map returns the corresponding keyspace id values for the given ids. -func (vind *UnicodeLooseMD5) Map(_ VCursor, ids []sqltypes.Value) ([][]byte, error) { - out := make([][]byte, 0, len(ids)) +func (vind *UnicodeLooseMD5) Map(_ VCursor, ids []sqltypes.Value) ([]Ksid, error) { + out := make([]Ksid, 0, len(ids)) for _, id := range ids { data, err := unicodeHash(id) if err != nil { return nil, fmt.Errorf("UnicodeLooseMD5.Map: %v", err) } - out = append(out, data) + out = append(out, Ksid{ID: data}) } return out, nil } diff --git a/go/vt/vtgate/vindexes/unicodeloosemd5_test.go b/go/vt/vtgate/vindexes/unicodeloosemd5_test.go index 7b290f3952c..19780c83e4c 100644 --- a/go/vt/vtgate/vindexes/unicodeloosemd5_test.go +++ b/go/vt/vtgate/vindexes/unicodeloosemd5_test.go @@ -82,7 +82,7 @@ func TestUnicodeLooseMD5Map(t *testing.T) { if err != nil { t.Error(err) } - out := string(got[0]) + out := string(got[0].ID) if out != tcase.out { t.Errorf("Map(%#v): %#v, want %#v", tcase.in, out, tcase.out) } diff --git a/go/vt/vtgate/vindexes/vindex.go b/go/vt/vtgate/vindexes/vindex.go index 39efc85d2b9..9fc57c2ccfd 100644 --- a/go/vt/vtgate/vindexes/vindex.go +++ b/go/vt/vtgate/vindexes/vindex.go @@ -17,6 +17,7 @@ limitations under the License. package vindexes import ( + "errors" "fmt" "github.com/youtube/vitess/go/sqltypes" @@ -57,11 +58,26 @@ type Vindex interface { Verify(cursor VCursor, ids []sqltypes.Value, ksids [][]byte) ([]bool, error) } +// Ksid represents a keyspace id. It's either a single keyspace id +// or a keyrange when the vindex is unable to determine the keyspace id. +type Ksid struct { + Range *topodatapb.KeyRange + ID []byte +} + +// ValidateUnique returns an error if the Ksid represents a KeyRange. +func (k Ksid) ValidateUnique() error { + if k.Range != nil { + return errors.New("vindex could not map the value to a unique keyspace id") + } + return nil +} + // Unique defines the interface for a unique vindex. // For a vindex to be unique, an id has to map to at most // one keyspace id. type Unique interface { - Map(cursor VCursor, ids []sqltypes.Value) ([][]byte, error) + Map(cursor VCursor, ids []sqltypes.Value) ([]Ksid, error) } // Ksids represents keyspace ids. It's either a list of keyspace ids diff --git a/go/vt/vtgate/vindexes/vschema_test.go b/go/vt/vtgate/vindexes/vschema_test.go index d4654f3c566..ac483b6b31c 100644 --- a/go/vt/vtgate/vindexes/vschema_test.go +++ b/go/vt/vtgate/vindexes/vschema_test.go @@ -41,7 +41,7 @@ type stFU struct { func (v *stFU) String() string { return v.name } func (*stFU) Cost() int { return 1 } func (*stFU) Verify(VCursor, []sqltypes.Value, [][]byte) ([]bool, error) { return []bool{}, nil } -func (*stFU) Map(VCursor, []sqltypes.Value) ([][]byte, error) { return nil, nil } +func (*stFU) Map(VCursor, []sqltypes.Value) ([]Ksid, error) { return nil, nil } func NewSTFU(name string, params map[string]string) (Vindex, error) { return &stFU{name: name, Params: params}, nil @@ -93,7 +93,7 @@ type stLU struct { func (v *stLU) String() string { return v.name } func (*stLU) Cost() int { return 2 } func (*stLU) Verify(VCursor, []sqltypes.Value, [][]byte) ([]bool, error) { return []bool{}, nil } -func (*stLU) Map(VCursor, []sqltypes.Value) ([][]byte, error) { return nil, nil } +func (*stLU) Map(VCursor, []sqltypes.Value) ([]Ksid, error) { return nil, nil } func (*stLU) Create(VCursor, [][]sqltypes.Value, [][]byte, bool) error { return nil } func (*stLU) Delete(VCursor, [][]sqltypes.Value, []byte) error { return nil } func (*stLU) Update(VCursor, []sqltypes.Value, []byte, []sqltypes.Value) error { return nil } diff --git a/go/vt/worker/key_resolver.go b/go/vt/worker/key_resolver.go index 6bd42945b83..37fda440521 100644 --- a/go/vt/worker/key_resolver.go +++ b/go/vt/worker/key_resolver.go @@ -180,8 +180,8 @@ func (r *v3Resolver) keyspaceID(row []sqltypes.Value) ([]byte, error) { if len(ksids) != 1 { return nil, fmt.Errorf("mapping row to keyspace id returned an invalid array of keyspace ids: %v", ksids) } - if ksids[0] == nil { + if ksids[0].Range != nil || ksids[0].ID == nil { return nil, fmt.Errorf("could not map %v to a keyspace id", v) } - return ksids[0], nil + return ksids[0].ID, nil } From 251fc829943992b10fd199748b7d18da7aa3082d Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Wed, 14 Feb 2018 18:48:21 -0800 Subject: [PATCH 2/3] v3: write_only support for unique lookup vindexes Now that unique lookup vindexes can return a scatter, we can add write_only support for them. --- go/vt/vtgate/vindexes/lookup.go | 25 +++++++--- go/vt/vtgate/vindexes/lookup_hash.go | 28 ++++++++--- .../vindexes/lookup_hash_unique_test.go | 50 ++++++++++++++++--- go/vt/vtgate/vindexes/lookup_test.go | 25 ++++++---- go/vt/vtgate/vindexes/lookup_unique_test.go | 50 ++++++++++++++++--- 5 files changed, 141 insertions(+), 37 deletions(-) diff --git a/go/vt/vtgate/vindexes/lookup.go b/go/vt/vtgate/vindexes/lookup.go index eb817ea9f96..d68cdf80267 100644 --- a/go/vt/vtgate/vindexes/lookup.go +++ b/go/vt/vtgate/vindexes/lookup.go @@ -18,7 +18,6 @@ package vindexes import ( "encoding/json" - "errors" "fmt" "github.com/youtube/vitess/go/sqltypes" @@ -157,8 +156,9 @@ func ksidsToValues(ksids [][]byte) []sqltypes.Value { // The table is expected to define the id column as unique. It's // Unique and a Lookup. type LookupUnique struct { - name string - lkp lookupInternal + name string + writeOnly bool + lkp lookupInternal } // NewLookupUnique creates a LookupUnique vindex. @@ -169,6 +169,7 @@ type LookupUnique struct { // // The following fields are optional: // autocommit: setting this to "true" will cause deletes to be ignored. +// write_only: in this mode, Map functions return the full keyrange causing a full scatter. func NewLookupUnique(name string, m map[string]string) (Vindex, error) { lu := &LookupUnique{name: name} @@ -176,13 +177,10 @@ func NewLookupUnique(name string, m map[string]string) (Vindex, error) { if err != nil { return nil, err } - scatter, err := boolFromMap(m, "write_only") + lu.writeOnly, err = boolFromMap(m, "write_only") if err != nil { return nil, err } - if scatter { - return nil, errors.New("write_only cannot be true for a unique lookup vindex") - } // Don't allow upserts for unique vindexes. if err := lu.lkp.Init(m, autocommit, false /* upsert */); err != nil { @@ -204,6 +202,12 @@ func (lu *LookupUnique) Cost() int { // Map returns the corresponding KeyspaceId values for the given ids. func (lu *LookupUnique) Map(vcursor VCursor, ids []sqltypes.Value) ([]Ksid, error) { out := make([]Ksid, 0, len(ids)) + if lu.writeOnly { + for range ids { + out = append(out, Ksid{Range: &topodata.KeyRange{}}) + } + return out, nil + } results, err := lu.lkp.Lookup(vcursor, ids) if err != nil { return nil, err @@ -223,6 +227,13 @@ func (lu *LookupUnique) Map(vcursor VCursor, ids []sqltypes.Value) ([]Ksid, erro // Verify returns true if ids maps to ksids. func (lu *LookupUnique) Verify(vcursor VCursor, ids []sqltypes.Value, ksids [][]byte) ([]bool, error) { + if lu.writeOnly { + out := make([]bool, len(ids)) + for i := range ids { + out[i] = true + } + return out, nil + } return lu.lkp.Verify(vcursor, ids, ksidsToValues(ksids)) } diff --git a/go/vt/vtgate/vindexes/lookup_hash.go b/go/vt/vtgate/vindexes/lookup_hash.go index c0d74da152a..03190a6ab78 100644 --- a/go/vt/vtgate/vindexes/lookup_hash.go +++ b/go/vt/vtgate/vindexes/lookup_hash.go @@ -18,7 +18,6 @@ package vindexes import ( "encoding/json" - "errors" "fmt" "github.com/youtube/vitess/go/sqltypes" @@ -130,6 +129,7 @@ func (lh *LookupHash) Verify(vcursor VCursor, ids []sqltypes.Value, ksids [][]by } return out, nil } + values, err := unhashList(ksids) if err != nil { return nil, fmt.Errorf("lookup.Verify.vunhash: %v", err) @@ -189,8 +189,9 @@ func unhashList(ksids [][]byte) ([]sqltypes.Value, error) { // Unique and a Lookup. // Warning: This Vindex is being depcreated in favor of LookupUnique type LookupHashUnique struct { - name string - lkp lookupInternal + name string + writeOnly bool + lkp lookupInternal } // NewLookupHashUnique creates a LookupHashUnique vindex. @@ -201,6 +202,7 @@ type LookupHashUnique struct { // // The following fields are optional: // autocommit: setting this to "true" will cause deletes to be ignored. +// write_only: in this mode, Map functions return the full keyrange causing a full scatter. func NewLookupHashUnique(name string, m map[string]string) (Vindex, error) { lhu := &LookupHashUnique{name: name} @@ -208,13 +210,10 @@ func NewLookupHashUnique(name string, m map[string]string) (Vindex, error) { if err != nil { return nil, err } - scatter, err := boolFromMap(m, "write_only") + lhu.writeOnly, err = boolFromMap(m, "write_only") if err != nil { return nil, err } - if scatter { - return nil, errors.New("write_only cannot be true for a unique lookup vindex") - } // Don't allow upserts for unique vindexes. if err := lhu.lkp.Init(m, autocommit, false /* upsert */); err != nil { @@ -236,6 +235,13 @@ func (lhu *LookupHashUnique) Cost() int { // Map returns the corresponding KeyspaceId values for the given ids. func (lhu *LookupHashUnique) Map(vcursor VCursor, ids []sqltypes.Value) ([]Ksid, error) { out := make([]Ksid, 0, len(ids)) + if lhu.writeOnly { + for range ids { + out = append(out, Ksid{Range: &topodata.KeyRange{}}) + } + return out, nil + } + results, err := lhu.lkp.Lookup(vcursor, ids) if err != nil { return nil, err @@ -260,6 +266,14 @@ func (lhu *LookupHashUnique) Map(vcursor VCursor, ids []sqltypes.Value) ([]Ksid, // Verify returns true if ids maps to ksids. func (lhu *LookupHashUnique) Verify(vcursor VCursor, ids []sqltypes.Value, ksids [][]byte) ([]bool, error) { + if lhu.writeOnly { + out := make([]bool, len(ids)) + for i := range ids { + out[i] = true + } + return out, nil + } + values, err := unhashList(ksids) if err != nil { return nil, fmt.Errorf("lookup.Verify.vunhash: %v", err) diff --git a/go/vt/vtgate/vindexes/lookup_hash_unique_test.go b/go/vt/vtgate/vindexes/lookup_hash_unique_test.go index 788b36a81f3..ee2ce1355ce 100644 --- a/go/vt/vtgate/vindexes/lookup_hash_unique_test.go +++ b/go/vt/vtgate/vindexes/lookup_hash_unique_test.go @@ -21,20 +21,23 @@ import ( "testing" "github.com/youtube/vitess/go/sqltypes" + topodatapb "github.com/youtube/vitess/go/vt/proto/topodata" ) func TestLookupHashUniqueNew(t *testing.T) { - _ = createLookup(t, "lookup_hash_unique", false) + l := createLookup(t, "lookup_hash_unique", false) + if want, got := l.(*LookupHashUnique).writeOnly, false; got != want { + t.Errorf("Create(lookup, false): %v, want %v", got, want) + } - _, err := CreateVindex("lookup_hash_unique", "lookup_hash_unique", map[string]string{ + l, err := CreateVindex("lookup_hash_unique", "lookup_hash_unique", map[string]string{ "table": "t", "from": "fromc", "to": "toc", "write_only": "true", }) - want := "write_only cannot be true for a unique lookup vindex" - if err == nil || err.Error() != want { - t.Errorf("Create(bad_scatter): %v, want %s", err, want) + if want, got := l.(*LookupHashUnique).writeOnly, true; got != want { + t.Errorf("Create(lookup, false): %v, want %v", got, want) } _, err = CreateVindex("lookup_hash_unique", "lookup_hash_unique", map[string]string{ @@ -43,7 +46,7 @@ func TestLookupHashUniqueNew(t *testing.T) { "to": "toc", "write_only": "invalid", }) - want = "write_only value must be 'true' or 'false': 'invalid'" + want := "write_only value must be 'true' or 'false': 'invalid'" if err == nil || err.Error() != want { t.Errorf("Create(bad_scatter): %v, want %s", err, want) } @@ -113,6 +116,24 @@ func TestLookupHashUniqueMap(t *testing.T) { vc.mustFail = false } +func TestLookupHashUniqueMapWriteOnly(t *testing.T) { + lhu := createLookup(t, "lookup_hash_unique", true) + vc := &vcursor{numRows: 0} + + got, err := lhu.(Unique).Map(vc, []sqltypes.Value{sqltypes.NewInt64(1), sqltypes.NewInt64(2)}) + if err != nil { + t.Error(err) + } + want := []Ksid{{ + Range: &topodatapb.KeyRange{}, + }, { + Range: &topodatapb.KeyRange{}, + }} + if !reflect.DeepEqual(got, want) { + t.Errorf("Map(): %#v, want %+v", got, want) + } +} + func TestLookupHashUniqueVerify(t *testing.T) { lhu := createLookup(t, "lookup_hash_unique", false) vc := &vcursor{numRows: 1} @@ -147,6 +168,23 @@ func TestLookupHashUniqueVerify(t *testing.T) { } } +func TestLookupHashUniqueVerifyWriteOnly(t *testing.T) { + lhu := createLookup(t, "lookup_hash_unique", true) + vc := &vcursor{numRows: 0} + + got, err := lhu.Verify(vc, []sqltypes.Value{sqltypes.NewInt64(1)}, [][]byte{[]byte("test")}) + if err != nil { + t.Error(err) + } + want := []bool{true} + if !reflect.DeepEqual(got, want) { + t.Errorf("lhu.Verify: %v, want %v", got, want) + } + if got, want := len(vc.queries), 0; got != want { + t.Errorf("vc.queries length: %v, want %v", got, want) + } +} + func TestLookupHashUniqueCreate(t *testing.T) { lhu := createLookup(t, "lookup_hash_unique", false) vc := &vcursor{} diff --git a/go/vt/vtgate/vindexes/lookup_test.go b/go/vt/vtgate/vindexes/lookup_test.go index 4b72862fbf1..7af3fb82597 100644 --- a/go/vt/vtgate/vindexes/lookup_test.go +++ b/go/vt/vtgate/vindexes/lookup_test.go @@ -216,30 +216,33 @@ func TestLookupNonUniqueMapAutocommit(t *testing.T) { } } -func TestLookupNonUniqueMapAbsent(t *testing.T) { - lookupNonUnique := createLookup(t, "lookup", false) +func TestLookupNonUniqueMapWriteOnly(t *testing.T) { + lookupNonUnique := createLookup(t, "lookup", true) vc := &vcursor{numRows: 0} got, err := lookupNonUnique.(NonUnique).Map(vc, []sqltypes.Value{sqltypes.NewInt64(1), sqltypes.NewInt64(2)}) if err != nil { t.Error(err) } - want := []Ksids{{}, {}} + want := []Ksids{{ + Range: &topodatapb.KeyRange{}, + }, { + Range: &topodatapb.KeyRange{}, + }} if !reflect.DeepEqual(got, want) { t.Errorf("Map(): %#v, want %+v", got, want) } +} - // writeOnly true should return full keyranges. - lookupNonUnique = createLookup(t, "lookup", true) - got, err = lookupNonUnique.(NonUnique).Map(vc, []sqltypes.Value{sqltypes.NewInt64(1), sqltypes.NewInt64(2)}) +func TestLookupNonUniqueMapAbsent(t *testing.T) { + lookupNonUnique := createLookup(t, "lookup", false) + vc := &vcursor{numRows: 0} + + got, err := lookupNonUnique.(NonUnique).Map(vc, []sqltypes.Value{sqltypes.NewInt64(1), sqltypes.NewInt64(2)}) if err != nil { t.Error(err) } - want = []Ksids{{ - Range: &topodatapb.KeyRange{}, - }, { - Range: &topodatapb.KeyRange{}, - }} + want := []Ksids{{}, {}} if !reflect.DeepEqual(got, want) { t.Errorf("Map(): %#v, want %+v", got, want) } diff --git a/go/vt/vtgate/vindexes/lookup_unique_test.go b/go/vt/vtgate/vindexes/lookup_unique_test.go index fe825494d02..e8ceffc206e 100644 --- a/go/vt/vtgate/vindexes/lookup_unique_test.go +++ b/go/vt/vtgate/vindexes/lookup_unique_test.go @@ -23,20 +23,23 @@ import ( "github.com/youtube/vitess/go/sqltypes" querypb "github.com/youtube/vitess/go/vt/proto/query" + topodatapb "github.com/youtube/vitess/go/vt/proto/topodata" ) func TestLookupUniqueNew(t *testing.T) { - _ = createLookup(t, "lookup_unique", false) + l := createLookup(t, "lookup_unique", false) + if want, got := l.(*LookupUnique).writeOnly, false; got != want { + t.Errorf("Create(lookup, false): %v, want %v", got, want) + } - _, err := CreateVindex("lookup_unique", "lookup_unique", map[string]string{ + l, err := CreateVindex("lookup_unique", "lookup_unique", map[string]string{ "table": "t", "from": "fromc", "to": "toc", "write_only": "true", }) - want := "write_only cannot be true for a unique lookup vindex" - if err == nil || err.Error() != want { - t.Errorf("Create(bad_scatter): %v, want %s", err, want) + if want, got := l.(*LookupUnique).writeOnly, true; got != want { + t.Errorf("Create(lookup, false): %v, want %v", got, want) } _, err = CreateVindex("lookup_unique", "lookup_unique", map[string]string{ @@ -45,7 +48,7 @@ func TestLookupUniqueNew(t *testing.T) { "to": "toc", "write_only": "invalid", }) - want = "write_only value must be 'true' or 'false': 'invalid'" + want := "write_only value must be 'true' or 'false': 'invalid'" if err == nil || err.Error() != want { t.Errorf("Create(bad_scatter): %v, want %s", err, want) } @@ -108,6 +111,24 @@ func TestLookupUniqueMap(t *testing.T) { vc.mustFail = false } +func TestLookupUniqueMapWriteOnly(t *testing.T) { + lookupUnique := createLookup(t, "lookup_unique", true) + vc := &vcursor{numRows: 0} + + got, err := lookupUnique.(Unique).Map(vc, []sqltypes.Value{sqltypes.NewInt64(1), sqltypes.NewInt64(2)}) + if err != nil { + t.Error(err) + } + want := []Ksid{{ + Range: &topodatapb.KeyRange{}, + }, { + Range: &topodatapb.KeyRange{}, + }} + if !reflect.DeepEqual(got, want) { + t.Errorf("Map(): %#v, want %+v", got, want) + } +} + func TestLookupUniqueVerify(t *testing.T) { lookupUnique := createLookup(t, "lookup_unique", false) vc := &vcursor{numRows: 1} @@ -121,6 +142,23 @@ func TestLookupUniqueVerify(t *testing.T) { } } +func TestLookupUniqueVerifyWriteOnly(t *testing.T) { + lookupUnique := createLookup(t, "lookup_unique", true) + vc := &vcursor{numRows: 0} + + got, err := lookupUnique.Verify(vc, []sqltypes.Value{sqltypes.NewInt64(1)}, [][]byte{[]byte("test")}) + if err != nil { + t.Error(err) + } + want := []bool{true} + if !reflect.DeepEqual(got, want) { + t.Errorf("lookupUnique.Verify: %v, want %v", got, want) + } + if got, want := len(vc.queries), 0; got != want { + t.Errorf("vc.queries length: %v, want %v", got, want) + } +} + func TestLookupUniqueCreate(t *testing.T) { lookupUnique, err := CreateVindex("lookup_unique", "lookup_unique", map[string]string{ "table": "t", From 47e9a562675b88fa87d7160b8e58f5458a062e3f Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Thu, 22 Feb 2018 11:27:26 -0800 Subject: [PATCH 3/3] v3: vindex: address review comments --- go/vt/vtgate/engine/route.go | 3 +++ go/vt/vtgate/engine/vindex_func_test.go | 8 ++++---- go/vt/vtgate/executor_framework_test.go | 4 ++-- go/vt/vtgate/planbuilder/plan_test.go | 8 ++++++-- go/vt/vtgate/vindexes/binary.go | 6 +++--- go/vt/vtgate/vindexes/binarymd5.go | 6 +++--- go/vt/vtgate/vindexes/hash.go | 8 ++++---- go/vt/vtgate/vindexes/hash_test.go | 2 +- go/vt/vtgate/vindexes/lookup.go | 10 +++++----- go/vt/vtgate/vindexes/lookup_hash.go | 12 ++++++------ go/vt/vtgate/vindexes/lookup_hash_unique_test.go | 8 ++++---- go/vt/vtgate/vindexes/lookup_unique_test.go | 6 +++--- go/vt/vtgate/vindexes/numeric.go | 8 ++++---- go/vt/vtgate/vindexes/numeric_static_map.go | 8 ++++---- go/vt/vtgate/vindexes/numeric_static_map_test.go | 2 +- go/vt/vtgate/vindexes/numeric_test.go | 2 +- go/vt/vtgate/vindexes/unicodeloosemd5.go | 6 +++--- go/vt/vtgate/vindexes/vindex.go | 8 ++++---- go/vt/vtgate/vindexes/vschema_test.go | 4 ++-- 19 files changed, 63 insertions(+), 56 deletions(-) diff --git a/go/vt/vtgate/engine/route.go b/go/vt/vtgate/engine/route.go index 3c73c67c998..044733ec73d 100644 --- a/go/vt/vtgate/engine/route.go +++ b/go/vt/vtgate/engine/route.go @@ -616,6 +616,9 @@ func (route *Route) resolveShards(vcursor VCursor, bindVars map[string]*querypb. for i, ksid := range ksids { switch { case ksid.Range != nil: + // Even for a unique vindex, a KeyRange can be returned if a keypace + // id cannot be identified. For example, this can happen during backfill. + // In such cases, we scatter over the KeyRange. // Use the multi-keyspace id API to convert a keyrange to shards. shards, err = vcursor.GetShardsForKsids(allShards, vindexes.Ksids{Range: ksid.Range}) if err != nil { diff --git a/go/vt/vtgate/engine/vindex_func_test.go b/go/vt/vtgate/engine/vindex_func_test.go index c31db34f84e..3dc42965ab1 100644 --- a/go/vt/vtgate/engine/vindex_func_test.go +++ b/go/vt/vtgate/engine/vindex_func_test.go @@ -35,9 +35,9 @@ func (*uvindex) Verify(vindexes.VCursor, []sqltypes.Value, [][]byte) ([]bool, er panic("unimplemented") } -func (v *uvindex) Map(vindexes.VCursor, []sqltypes.Value) ([]vindexes.Ksid, error) { +func (v *uvindex) Map(vindexes.VCursor, []sqltypes.Value) ([]vindexes.KsidOrRange, error) { if v.matchkr { - return []vindexes.Ksid{{ + return []vindexes.KsidOrRange{{ Range: &topodatapb.KeyRange{ Start: []byte{0x40}, End: []byte{0x60}, @@ -45,11 +45,11 @@ func (v *uvindex) Map(vindexes.VCursor, []sqltypes.Value) ([]vindexes.Ksid, erro }}, nil } if v.matchid { - return []vindexes.Ksid{ + return []vindexes.KsidOrRange{ {ID: []byte("foo")}, }, nil } - return []vindexes.Ksid{{}}, nil + return []vindexes.KsidOrRange{{}}, nil } // nvindex is NonUnique. diff --git a/go/vt/vtgate/executor_framework_test.go b/go/vt/vtgate/executor_framework_test.go index 2655145e0e2..5214f401cc0 100644 --- a/go/vt/vtgate/executor_framework_test.go +++ b/go/vt/vtgate/executor_framework_test.go @@ -295,8 +295,8 @@ func (*keyRangeLookuperUnique) Cost() int { return 0 } func (*keyRangeLookuperUnique) Verify(vindexes.VCursor, []sqltypes.Value, [][]byte) ([]bool, error) { return []bool{}, nil } -func (*keyRangeLookuperUnique) Map(vindexes.VCursor, []sqltypes.Value) ([]vindexes.Ksid, error) { - return []vindexes.Ksid{{ +func (*keyRangeLookuperUnique) Map(vindexes.VCursor, []sqltypes.Value) ([]vindexes.KsidOrRange, error) { + return []vindexes.KsidOrRange{{ Range: &topodatapb.KeyRange{ End: []byte{0x10}, }, diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index 557ec936b8b..48484691b53 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -42,7 +42,9 @@ func (*hashIndex) Cost() int { return 1 } func (*hashIndex) Verify(vindexes.VCursor, []sqltypes.Value, [][]byte) ([]bool, error) { return []bool{}, nil } -func (*hashIndex) Map(vindexes.VCursor, []sqltypes.Value) ([]vindexes.Ksid, error) { return nil, nil } +func (*hashIndex) Map(vindexes.VCursor, []sqltypes.Value) ([]vindexes.KsidOrRange, error) { + return nil, nil +} func newHashIndex(name string, _ map[string]string) (vindexes.Vindex, error) { return &hashIndex{name: name}, nil @@ -58,7 +60,9 @@ func (*lookupIndex) Cost() int { return 2 } func (*lookupIndex) Verify(vindexes.VCursor, []sqltypes.Value, [][]byte) ([]bool, error) { return []bool{}, nil } -func (*lookupIndex) Map(vindexes.VCursor, []sqltypes.Value) ([]vindexes.Ksid, error) { return nil, nil } +func (*lookupIndex) Map(vindexes.VCursor, []sqltypes.Value) ([]vindexes.KsidOrRange, error) { + return nil, nil +} func (*lookupIndex) Create(vindexes.VCursor, [][]sqltypes.Value, [][]byte, bool) error { return nil } func (*lookupIndex) Delete(vindexes.VCursor, [][]sqltypes.Value, []byte) error { return nil } func (*lookupIndex) Update(vindexes.VCursor, []sqltypes.Value, []byte, []sqltypes.Value) error { diff --git a/go/vt/vtgate/vindexes/binary.go b/go/vt/vtgate/vindexes/binary.go index d4692214680..361bd638eec 100644 --- a/go/vt/vtgate/vindexes/binary.go +++ b/go/vt/vtgate/vindexes/binary.go @@ -58,10 +58,10 @@ func (vind *Binary) Verify(_ VCursor, ids []sqltypes.Value, ksids [][]byte) ([]b } // Map returns the corresponding keyspace id values for the given ids. -func (vind *Binary) Map(_ VCursor, ids []sqltypes.Value) ([]Ksid, error) { - out := make([]Ksid, 0, len(ids)) +func (vind *Binary) Map(_ VCursor, ids []sqltypes.Value) ([]KsidOrRange, error) { + out := make([]KsidOrRange, 0, len(ids)) for _, id := range ids { - out = append(out, Ksid{ID: id.ToBytes()}) + out = append(out, KsidOrRange{ID: id.ToBytes()}) } return out, nil } diff --git a/go/vt/vtgate/vindexes/binarymd5.go b/go/vt/vtgate/vindexes/binarymd5.go index b8f0b848ea2..2534cbfc72c 100644 --- a/go/vt/vtgate/vindexes/binarymd5.go +++ b/go/vt/vtgate/vindexes/binarymd5.go @@ -57,10 +57,10 @@ func (vind *BinaryMD5) Verify(_ VCursor, ids []sqltypes.Value, ksids [][]byte) ( } // Map returns the corresponding keyspace id values for the given ids. -func (vind *BinaryMD5) Map(_ VCursor, ids []sqltypes.Value) ([]Ksid, error) { - out := make([]Ksid, 0, len(ids)) +func (vind *BinaryMD5) Map(_ VCursor, ids []sqltypes.Value) ([]KsidOrRange, error) { + out := make([]KsidOrRange, 0, len(ids)) for _, id := range ids { - out = append(out, Ksid{ID: binHash(id.ToBytes())}) + out = append(out, KsidOrRange{ID: binHash(id.ToBytes())}) } return out, nil } diff --git a/go/vt/vtgate/vindexes/hash.go b/go/vt/vtgate/vindexes/hash.go index 8ffaed3124d..cdef3a60a83 100644 --- a/go/vt/vtgate/vindexes/hash.go +++ b/go/vt/vtgate/vindexes/hash.go @@ -55,15 +55,15 @@ func (vind *Hash) Cost() int { } // Map returns the corresponding KeyspaceId values for the given ids. -func (vind *Hash) Map(_ VCursor, ids []sqltypes.Value) ([]Ksid, error) { - out := make([]Ksid, 0, len(ids)) +func (vind *Hash) Map(_ VCursor, ids []sqltypes.Value) ([]KsidOrRange, error) { + out := make([]KsidOrRange, 0, len(ids)) for _, id := range ids { num, err := sqltypes.ToUint64(id) if err != nil { - out = append(out, Ksid{}) + out = append(out, KsidOrRange{}) continue } - out = append(out, Ksid{ID: vhash(num)}) + out = append(out, KsidOrRange{ID: vhash(num)}) } return out, nil } diff --git a/go/vt/vtgate/vindexes/hash_test.go b/go/vt/vtgate/vindexes/hash_test.go index aa65c809cab..44c7fd76f3b 100644 --- a/go/vt/vtgate/vindexes/hash_test.go +++ b/go/vt/vtgate/vindexes/hash_test.go @@ -59,7 +59,7 @@ func TestHashMap(t *testing.T) { if err != nil { t.Error(err) } - want := []Ksid{ + want := []KsidOrRange{ {ID: []byte("\x16k@\xb4J\xbaK\xd6")}, {ID: []byte("\x06\xe7\xea\"Βp\x8f")}, {ID: []byte("N\xb1\x90ɢ\xfa\x16\x9c")}, diff --git a/go/vt/vtgate/vindexes/lookup.go b/go/vt/vtgate/vindexes/lookup.go index d68cdf80267..a351d4ae715 100644 --- a/go/vt/vtgate/vindexes/lookup.go +++ b/go/vt/vtgate/vindexes/lookup.go @@ -200,11 +200,11 @@ func (lu *LookupUnique) Cost() int { } // Map returns the corresponding KeyspaceId values for the given ids. -func (lu *LookupUnique) Map(vcursor VCursor, ids []sqltypes.Value) ([]Ksid, error) { - out := make([]Ksid, 0, len(ids)) +func (lu *LookupUnique) Map(vcursor VCursor, ids []sqltypes.Value) ([]KsidOrRange, error) { + out := make([]KsidOrRange, 0, len(ids)) if lu.writeOnly { for range ids { - out = append(out, Ksid{Range: &topodata.KeyRange{}}) + out = append(out, KsidOrRange{Range: &topodata.KeyRange{}}) } return out, nil } @@ -215,9 +215,9 @@ func (lu *LookupUnique) Map(vcursor VCursor, ids []sqltypes.Value) ([]Ksid, erro for i, result := range results { switch len(result.Rows) { case 0: - out = append(out, Ksid{}) + out = append(out, KsidOrRange{}) case 1: - out = append(out, Ksid{ID: result.Rows[0][0].ToBytes()}) + out = append(out, KsidOrRange{ID: result.Rows[0][0].ToBytes()}) default: return nil, fmt.Errorf("Lookup.Map: unexpected multiple results from vindex %s: %v", lu.lkp.Table, ids[i]) } diff --git a/go/vt/vtgate/vindexes/lookup_hash.go b/go/vt/vtgate/vindexes/lookup_hash.go index 03190a6ab78..9eeef26ae06 100644 --- a/go/vt/vtgate/vindexes/lookup_hash.go +++ b/go/vt/vtgate/vindexes/lookup_hash.go @@ -233,11 +233,11 @@ func (lhu *LookupHashUnique) Cost() int { } // Map returns the corresponding KeyspaceId values for the given ids. -func (lhu *LookupHashUnique) Map(vcursor VCursor, ids []sqltypes.Value) ([]Ksid, error) { - out := make([]Ksid, 0, len(ids)) +func (lhu *LookupHashUnique) Map(vcursor VCursor, ids []sqltypes.Value) ([]KsidOrRange, error) { + out := make([]KsidOrRange, 0, len(ids)) if lhu.writeOnly { for range ids { - out = append(out, Ksid{Range: &topodata.KeyRange{}}) + out = append(out, KsidOrRange{Range: &topodata.KeyRange{}}) } return out, nil } @@ -249,14 +249,14 @@ func (lhu *LookupHashUnique) Map(vcursor VCursor, ids []sqltypes.Value) ([]Ksid, for i, result := range results { switch len(result.Rows) { case 0: - out = append(out, Ksid{}) + out = append(out, KsidOrRange{}) case 1: num, err := sqltypes.ToUint64(result.Rows[0][0]) if err != nil { - out = append(out, Ksid{}) + out = append(out, KsidOrRange{}) continue } - out = append(out, Ksid{ID: vhash(num)}) + out = append(out, KsidOrRange{ID: vhash(num)}) default: return nil, fmt.Errorf("LookupHash.Map: unexpected multiple results from vindex %s: %v", lhu.lkp.Table, ids[i]) } diff --git a/go/vt/vtgate/vindexes/lookup_hash_unique_test.go b/go/vt/vtgate/vindexes/lookup_hash_unique_test.go index ee2ce1355ce..41cbb5ee12b 100644 --- a/go/vt/vtgate/vindexes/lookup_hash_unique_test.go +++ b/go/vt/vtgate/vindexes/lookup_hash_unique_test.go @@ -67,7 +67,7 @@ func TestLookupHashUniqueMap(t *testing.T) { if err != nil { t.Error(err) } - want := []Ksid{ + want := []KsidOrRange{ {ID: []byte("\x16k@\xb4J\xbaK\xd6")}, {ID: []byte("\x16k@\xb4J\xbaK\xd6")}, } @@ -80,7 +80,7 @@ func TestLookupHashUniqueMap(t *testing.T) { if err != nil { t.Error(err) } - want = []Ksid{{}, {}} + want = []KsidOrRange{{}, {}} if !reflect.DeepEqual(got, want) { t.Errorf("Map(): %#v, want %+v", got, want) } @@ -101,7 +101,7 @@ func TestLookupHashUniqueMap(t *testing.T) { if err != nil { t.Error(err) } - want = []Ksid{{}} + want = []KsidOrRange{{}} if !reflect.DeepEqual(got, want) { t.Errorf("Map(): %#v, want %+v", got, want) } @@ -124,7 +124,7 @@ func TestLookupHashUniqueMapWriteOnly(t *testing.T) { if err != nil { t.Error(err) } - want := []Ksid{{ + want := []KsidOrRange{{ Range: &topodatapb.KeyRange{}, }, { Range: &topodatapb.KeyRange{}, diff --git a/go/vt/vtgate/vindexes/lookup_unique_test.go b/go/vt/vtgate/vindexes/lookup_unique_test.go index e8ceffc206e..804e90d16c2 100644 --- a/go/vt/vtgate/vindexes/lookup_unique_test.go +++ b/go/vt/vtgate/vindexes/lookup_unique_test.go @@ -76,7 +76,7 @@ func TestLookupUniqueMap(t *testing.T) { if err != nil { t.Error(err) } - want := []Ksid{ + want := []KsidOrRange{ {ID: []byte("1")}, {ID: []byte("1")}, } @@ -89,7 +89,7 @@ func TestLookupUniqueMap(t *testing.T) { if err != nil { t.Error(err) } - want = []Ksid{{}, {}} + want = []KsidOrRange{{}, {}} if !reflect.DeepEqual(got, want) { t.Errorf("Map(): %#v, want %+v", got, want) } @@ -119,7 +119,7 @@ func TestLookupUniqueMapWriteOnly(t *testing.T) { if err != nil { t.Error(err) } - want := []Ksid{{ + want := []KsidOrRange{{ Range: &topodatapb.KeyRange{}, }, { Range: &topodatapb.KeyRange{}, diff --git a/go/vt/vtgate/vindexes/numeric.go b/go/vt/vtgate/vindexes/numeric.go index 322c651a8e5..cb4df93a40f 100644 --- a/go/vt/vtgate/vindexes/numeric.go +++ b/go/vt/vtgate/vindexes/numeric.go @@ -66,17 +66,17 @@ func (*Numeric) Verify(_ VCursor, ids []sqltypes.Value, ksids [][]byte) ([]bool, } // Map returns the associated keyspace ids for the given ids. -func (*Numeric) Map(_ VCursor, ids []sqltypes.Value) ([]Ksid, error) { - out := make([]Ksid, 0, len(ids)) +func (*Numeric) Map(_ VCursor, ids []sqltypes.Value) ([]KsidOrRange, error) { + out := make([]KsidOrRange, 0, len(ids)) for _, id := range ids { num, err := sqltypes.ToUint64(id) if err != nil { - out = append(out, Ksid{}) + out = append(out, KsidOrRange{}) continue } var keybytes [8]byte binary.BigEndian.PutUint64(keybytes[:], num) - out = append(out, Ksid{ID: keybytes[:]}) + out = append(out, KsidOrRange{ID: keybytes[:]}) } return out, nil } diff --git a/go/vt/vtgate/vindexes/numeric_static_map.go b/go/vt/vtgate/vindexes/numeric_static_map.go index 47cc1f6074c..148d339f738 100644 --- a/go/vt/vtgate/vindexes/numeric_static_map.go +++ b/go/vt/vtgate/vindexes/numeric_static_map.go @@ -94,12 +94,12 @@ func (vind *NumericStaticMap) Verify(_ VCursor, ids []sqltypes.Value, ksids [][] } // Map returns the associated keyspace ids for the given ids. -func (vind *NumericStaticMap) Map(_ VCursor, ids []sqltypes.Value) ([]Ksid, error) { - out := make([]Ksid, 0, len(ids)) +func (vind *NumericStaticMap) Map(_ VCursor, ids []sqltypes.Value) ([]KsidOrRange, error) { + out := make([]KsidOrRange, 0, len(ids)) for _, id := range ids { num, err := sqltypes.ToUint64(id) if err != nil { - out = append(out, Ksid{}) + out = append(out, KsidOrRange{}) continue } lookupNum, ok := vind.lookup[num] @@ -108,7 +108,7 @@ func (vind *NumericStaticMap) Map(_ VCursor, ids []sqltypes.Value) ([]Ksid, erro } var keybytes [8]byte binary.BigEndian.PutUint64(keybytes[:], num) - out = append(out, Ksid{ID: keybytes[:]}) + out = append(out, KsidOrRange{ID: keybytes[:]}) } return out, nil } diff --git a/go/vt/vtgate/vindexes/numeric_static_map_test.go b/go/vt/vtgate/vindexes/numeric_static_map_test.go index a2acbcdeb5d..7bf5b0b4403 100644 --- a/go/vt/vtgate/vindexes/numeric_static_map_test.go +++ b/go/vt/vtgate/vindexes/numeric_static_map_test.go @@ -80,7 +80,7 @@ func TestNumericStaticMapMap(t *testing.T) { // in the third slice, we expect 2 instead of 3 as numeric_static_map_test.json // has 3 mapped to 2 - want := []Ksid{ + want := []KsidOrRange{ {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x01")}, {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x02")}, {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x02")}, diff --git a/go/vt/vtgate/vindexes/numeric_test.go b/go/vt/vtgate/vindexes/numeric_test.go index 2353cd5d9e1..66aabd54690 100644 --- a/go/vt/vtgate/vindexes/numeric_test.go +++ b/go/vt/vtgate/vindexes/numeric_test.go @@ -58,7 +58,7 @@ func TestNumericMap(t *testing.T) { if err != nil { t.Error(err) } - want := []Ksid{ + want := []KsidOrRange{ {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x01")}, {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x02")}, {ID: []byte("\x00\x00\x00\x00\x00\x00\x00\x03")}, diff --git a/go/vt/vtgate/vindexes/unicodeloosemd5.go b/go/vt/vtgate/vindexes/unicodeloosemd5.go index 1ea5ffd4ce3..c242f4cecc3 100644 --- a/go/vt/vtgate/vindexes/unicodeloosemd5.go +++ b/go/vt/vtgate/vindexes/unicodeloosemd5.go @@ -70,14 +70,14 @@ func (vind *UnicodeLooseMD5) Verify(_ VCursor, ids []sqltypes.Value, ksids [][]b } // Map returns the corresponding keyspace id values for the given ids. -func (vind *UnicodeLooseMD5) Map(_ VCursor, ids []sqltypes.Value) ([]Ksid, error) { - out := make([]Ksid, 0, len(ids)) +func (vind *UnicodeLooseMD5) Map(_ VCursor, ids []sqltypes.Value) ([]KsidOrRange, error) { + out := make([]KsidOrRange, 0, len(ids)) for _, id := range ids { data, err := unicodeHash(id) if err != nil { return nil, fmt.Errorf("UnicodeLooseMD5.Map: %v", err) } - out = append(out, Ksid{ID: data}) + out = append(out, KsidOrRange{ID: data}) } return out, nil } diff --git a/go/vt/vtgate/vindexes/vindex.go b/go/vt/vtgate/vindexes/vindex.go index 9fc57c2ccfd..89e59088c88 100644 --- a/go/vt/vtgate/vindexes/vindex.go +++ b/go/vt/vtgate/vindexes/vindex.go @@ -58,15 +58,15 @@ type Vindex interface { Verify(cursor VCursor, ids []sqltypes.Value, ksids [][]byte) ([]bool, error) } -// Ksid represents a keyspace id. It's either a single keyspace id +// KsidOrRange represents a keyspace id. It's either a single keyspace id // or a keyrange when the vindex is unable to determine the keyspace id. -type Ksid struct { +type KsidOrRange struct { Range *topodatapb.KeyRange ID []byte } // ValidateUnique returns an error if the Ksid represents a KeyRange. -func (k Ksid) ValidateUnique() error { +func (k KsidOrRange) ValidateUnique() error { if k.Range != nil { return errors.New("vindex could not map the value to a unique keyspace id") } @@ -77,7 +77,7 @@ func (k Ksid) ValidateUnique() error { // For a vindex to be unique, an id has to map to at most // one keyspace id. type Unique interface { - Map(cursor VCursor, ids []sqltypes.Value) ([]Ksid, error) + Map(cursor VCursor, ids []sqltypes.Value) ([]KsidOrRange, error) } // Ksids represents keyspace ids. It's either a list of keyspace ids diff --git a/go/vt/vtgate/vindexes/vschema_test.go b/go/vt/vtgate/vindexes/vschema_test.go index ac483b6b31c..261c7cca7ee 100644 --- a/go/vt/vtgate/vindexes/vschema_test.go +++ b/go/vt/vtgate/vindexes/vschema_test.go @@ -41,7 +41,7 @@ type stFU struct { func (v *stFU) String() string { return v.name } func (*stFU) Cost() int { return 1 } func (*stFU) Verify(VCursor, []sqltypes.Value, [][]byte) ([]bool, error) { return []bool{}, nil } -func (*stFU) Map(VCursor, []sqltypes.Value) ([]Ksid, error) { return nil, nil } +func (*stFU) Map(VCursor, []sqltypes.Value) ([]KsidOrRange, error) { return nil, nil } func NewSTFU(name string, params map[string]string) (Vindex, error) { return &stFU{name: name, Params: params}, nil @@ -93,7 +93,7 @@ type stLU struct { func (v *stLU) String() string { return v.name } func (*stLU) Cost() int { return 2 } func (*stLU) Verify(VCursor, []sqltypes.Value, [][]byte) ([]bool, error) { return []bool{}, nil } -func (*stLU) Map(VCursor, []sqltypes.Value) ([]Ksid, error) { return nil, nil } +func (*stLU) Map(VCursor, []sqltypes.Value) ([]KsidOrRange, error) { return nil, nil } func (*stLU) Create(VCursor, [][]sqltypes.Value, [][]byte, bool) error { return nil } func (*stLU) Delete(VCursor, [][]sqltypes.Value, []byte) error { return nil } func (*stLU) Update(VCursor, []sqltypes.Value, []byte, []sqltypes.Value) error { return nil }