vrepl: CreateLookupVindex command with backfill

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/evanphx/json-patch v4.5.0+incompatible
 	github.com/go-critic/go-critic v0.4.0 // indirect
 	github.com/go-ini/ini v1.12.0 // indirect
-	github.com/gogo/protobuf v1.3.1 // indirect
+	github.com/gogo/protobuf v1.3.1
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
 	github.com/golang/mock v1.3.1

go/vt/vtctl/vtctl.go

@@ -312,8 +312,11 @@ var commands = []commandGroup{
 				"[-skip_schema_copy] <keyspace.workflow> <source_shards> <target_shards>",
 				"Start a Resharding process. Example: Reshard ks.workflow001 '0' '-80,80-'"},
 			{"Migrate", commandMigrate,
-				"[-cell=<cell>] [-tablet_types=<tablet_types>] -workflow=<workflow> <source_keyspace> <target_keyspace> <table_specs>",
+				"[-cell=<cell>] [-tablet_types=<source_tablet_types>] -workflow=<workflow> <source_keyspace> <target_keyspace> <table_specs>",
 				`Start a table(s) migration, table_specs is a list of tables or the tables section of the vschema for the target keyspace. Example: '{"t1":{"column_vindexes": [{""column": "id1", "name": "hash"}]}, "t2":{"column_vindexes": [{""column": "id2", "name": "hash"}]}}`},
+			{"CreateLookupVindex", commandCreateLookupVindex,
+				"[-cell=<cell>] [-tablet_types=<source_tablet_types>] <keyspace> <json_spec>",
+				`Create and backfill a lookup vindex. the json_spec must contain the vindex and colvindex specs for the new lookup.`},
 			{"Materialize", commandMaterialize,
 				`<json_spec>, example : '{"workflow": "aaa", "source_keyspace": "source", "target_keyspace": "target", "table_settings": [{"target_table": "customer", "source_expression": "select * from customer", "create_ddl": "copy"}]}'`,
 				"Performs materialization based on the json spec."},
@@ -1830,6 +1833,23 @@ func commandMigrate(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.F
 	return wr.Migrate(ctx, *workflow, source, target, tableSpecs, *cell, *tabletTypes)
 }
 
+func commandCreateLookupVindex(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
+	cell := subFlags.String("cell", "", "Cell to replicate from.")
+	tabletTypes := subFlags.String("tablet_types", "", "Source tablet types to replicate from.")
+	if err := subFlags.Parse(args); err != nil {
+		return err
+	}
+	if subFlags.NArg() != 2 {
+		return fmt.Errorf("two arguments are required: keyspace and json_spec")
+	}
+	keyspace := subFlags.Arg(0)
+	specs := &vschemapb.Keyspace{}
+	if err := json2.Unmarshal([]byte(subFlags.Arg(1)), specs); err != nil {
+		return err
+	}
+	return wr.CreateLookupVindex(ctx, keyspace, specs, *cell, *tabletTypes)
+}
+
 func commandMaterialize(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
 	if err := subFlags.Parse(args); err != nil {
 		return err

go/vt/vtgate/vindexes/vschema.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 
 	"vitess.io/vitess/go/json2"
+	"vitess.io/vitess/go/sqltypes"
 	"vitess.io/vitess/go/vt/sqlparser"
 
 	querypb "vitess.io/vitess/go/vt/proto/query"
@@ -614,6 +615,19 @@ func LoadFormalKeyspace(filename string) (*vschemapb.Keyspace, error) {
 	return formal, nil
 }
 
+// ChooseVindexForType chooses the most appropriate vindex for the give type.
+func ChooseVindexForType(typ querypb.Type) (string, error) {
+	switch {
+	case sqltypes.IsIntegral(typ):
+		return "hash", nil
+	case sqltypes.IsText(typ):
+		return "unicode_loose_md5", nil
+	case sqltypes.IsBinary(typ):
+		return "binary_md5", nil
+	}
+	return "", fmt.Errorf("type %v is not recommended for a vindex", typ)
+}
+
 // FindBestColVindex finds the best ColumnVindex for VReplication.
 func FindBestColVindex(table *Table) (*ColumnVindex, error) {
 	if len(table.ColumnVindexes) == 0 {

go/vt/vtgate/vindexes/vschema_test.go

@@ -820,6 +820,115 @@ func TestVSchemaRoutingRules(t *testing.T) {
 	}
 }
 
+func TestChooseVindexForType(t *testing.T) {
+	testcases := []struct {
+		in  querypb.Type
+		out string
+	}{{
+		in:  sqltypes.Null,
+		out: "",
+	}, {
+		in:  sqltypes.Int8,
+		out: "hash",
+	}, {
+		in:  sqltypes.Uint8,
+		out: "hash",
+	}, {
+		in:  sqltypes.Int16,
+		out: "hash",
+	}, {
+		in:  sqltypes.Uint16,
+		out: "hash",
+	}, {
+		in:  sqltypes.Int24,
+		out: "hash",
+	}, {
+		in:  sqltypes.Uint24,
+		out: "hash",
+	}, {
+		in:  sqltypes.Int32,
+		out: "hash",
+	}, {
+		in:  sqltypes.Uint32,
+		out: "hash",
+	}, {
+		in:  sqltypes.Int64,
+		out: "hash",
+	}, {
+		in:  sqltypes.Uint64,
+		out: "hash",
+	}, {
+		in:  sqltypes.Float32,
+		out: "hash",
+	}, {
+		in:  sqltypes.Float64,
+		out: "",
+	}, {
+		in:  sqltypes.Timestamp,
+		out: "",
+	}, {
+		in:  sqltypes.Date,
+		out: "",
+	}, {
+		in:  sqltypes.Time,
+		out: "",
+	}, {
+		in:  sqltypes.Datetime,
+		out: "",
+	}, {
+		in:  sqltypes.Year,
+		out: "hash",
+	}, {
+		in:  sqltypes.Decimal,
+		out: "",
+	}, {
+		in:  sqltypes.Text,
+		out: "unicode_loose_md5",
+	}, {
+		in:  sqltypes.Blob,
+		out: "binary_md5",
+	}, {
+		in:  sqltypes.VarChar,
+		out: "unicode_loose_md5",
+	}, {
+		in:  sqltypes.VarBinary,
+		out: "binary_md5",
+	}, {
+		in:  sqltypes.Char,
+		out: "unicode_loose_md5",
+	}, {
+		in:  sqltypes.Binary,
+		out: "binary_md5",
+	}, {
+		in:  sqltypes.Bit,
+		out: "",
+	}, {
+		in:  sqltypes.Enum,
+		out: "",
+	}, {
+		in:  sqltypes.Set,
+		out: "",
+	}, {
+		in:  sqltypes.Geometry,
+		out: "",
+	}, {
+		in:  sqltypes.TypeJSON,
+		out: "",
+	}, {
+		in:  sqltypes.Expression,
+		out: "",
+	}}
+
+	for _, tcase := range testcases {
+		out, err := ChooseVindexForType(tcase.in)
+		if out == "" {
+			assert.Error(t, err, tcase.in)
+			continue
+		}
+		assert.Equal(t, out, tcase.out, tcase.in)
+	}
+}
+
 func TestFindBestColVindex(t *testing.T) {
 	testSrvVSchema := &vschemapb.SrvVSchema{
 		Keyspaces: map[string]*vschemapb.Keyspace{