diff --git a/go/test/dbg/dbg.go b/go/test/dbg/dbg.go index 94ce7a385c2..b54449d1b39 100644 --- a/go/test/dbg/dbg.go +++ b/go/test/dbg/dbg.go @@ -23,6 +23,7 @@ import ( "go/format" "go/parser" "go/token" + "io" "os" "path" "runtime" @@ -163,6 +164,17 @@ func V[Val any](v Val) Val { // P prints all the arguments passed to the function in verbose debug form func P(vals ...any) { + dump(vals, os.Stdout) +} + +// S returns a string with all the arguments passed to the function in verbose debug form +func S(vals ...any) string { + buf := &strings.Builder{} + dump(vals, buf) + return buf.String() +} + +func dump(vals []any, writer io.Writer) { var p *params if _, f, lineno, ok := runtime.Caller(1); ok { p = defaultCache.resolve(f, lineno) @@ -176,5 +188,7 @@ func P(vals ...any) { w := text.NewIndentWriter(&buf, nil, bytes.Repeat([]byte{' '}, indent)) fmt.Fprintf(w, "%# v\n", pretty.Formatter(v)) } - _, _ = buf.WriteTo(os.Stdout) + + _, _ = buf.WriteTo(writer) + } diff --git a/go/vt/sqlparser/ast.go b/go/vt/sqlparser/ast.go index acc34faabb3..4a9a5a6b17a 100644 --- a/go/vt/sqlparser/ast.go +++ b/go/vt/sqlparser/ast.go @@ -2470,11 +2470,11 @@ type ( // This is a struct that the parser will never produce - it's written and read by the gen4 planner // CAUTION: you should only change argName and hasValuesArg through the setter methods ExtractedSubquery struct { - Original Expr // original expression that was replaced by this ExtractedSubquery - OpCode int // this should really be engine.PulloutOpCode, but we cannot depend on engine :( - Subquery *Subquery - OtherSide Expr // represents the side of the comparison, this field will be nil if Original is not a comparison - NeedsRewrite bool // tells whether we need to rewrite this subquery to Original or not + Original Expr // original expression that was replaced by this ExtractedSubquery + OpCode int // this should really be engine.PulloutOpCode, but we cannot depend on engine :( + Subquery *Subquery + OtherSide Expr // represents the side of the comparison, this field will be nil if Original is not a comparison + Merged bool // tells whether we need to rewrite this subquery to Original or not hasValuesArg string argName string diff --git a/go/vt/sqlparser/ast_equals.go b/go/vt/sqlparser/ast_equals.go index d66f6192f5c..eec67c2029b 100644 --- a/go/vt/sqlparser/ast_equals.go +++ b/go/vt/sqlparser/ast_equals.go @@ -2367,7 +2367,7 @@ func (cmp *Comparator) RefOfExtractedSubquery(a, b *ExtractedSubquery) bool { return false } return a.OpCode == b.OpCode && - a.NeedsRewrite == b.NeedsRewrite && + a.Merged == b.Merged && a.hasValuesArg == b.hasValuesArg && a.argName == b.argName && cmp.Expr(a.Original, b.Original) && diff --git a/go/vt/vtgate/executor_framework_test.go b/go/vt/vtgate/executor_framework_test.go index 6c5acbc72cf..fe26e20a4c4 100644 --- a/go/vt/vtgate/executor_framework_test.go +++ b/go/vt/vtgate/executor_framework_test.go @@ -19,6 +19,7 @@ package vtgate import ( "bytes" "context" + _ "embed" "fmt" "strconv" "strings" @@ -45,359 +46,11 @@ import ( topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) -var executorVSchema = ` -{ - "sharded": true, - "vindexes": { - "hash_index": { - "type": "hash" - }, - "music_user_map": { - "type": "lookup_hash_unique", - "owner": "music", - "params": { - "table": "music_user_map", - "from": "music_id", - "to": "user_id" - } - }, - "name_user_map": { - "type": "lookup_hash", - "owner": "user", - "params": { - "table": "name_user_map", - "from": "name", - "to": "user_id" - } - }, - "name_lastname_keyspace_id_map": { - "type": "lookup", - "owner": "user2", - "params": { - "table": "name_lastname_keyspace_id_map", - "from": "name,lastname", - "to": "keyspace_id" - } - }, - "insert_ignore_idx": { - "type": "lookup_hash", - "owner": "insert_ignore_test", - "params": { - "table": "ins_lookup", - "from": "fromcol", - "to": "tocol" - } - }, - "idx1": { - "type": "hash" - }, - "idx_noauto": { - "type": "hash", - "owner": "noauto_table" - }, - "keyspace_id": { - "type": "numeric" - }, - "krcol_unique_vdx": { - "type": "keyrange_lookuper_unique" - }, - "krcol_vdx": { - "type": "keyrange_lookuper" - }, - "t1_lkp_vdx": { - "type": "consistent_lookup_unique", - "params": { - "table": "t1_lkp_idx", - "from": "unq_col", - "to": "keyspace_id" - }, - "owner": "t1" - }, - "t2_wo_lu_vdx": { - "type": "lookup_unique", - "params": { - "table": "TestUnsharded.wo_lu_idx", - "from": "wo_lu_col", - "to": "keyspace_id", - "write_only": "true" - }, - "owner": "t2_lookup" - }, - "t2_erl_lu_vdx": { - "type": "lookup_unique", - "params": { - "table": "TestUnsharded.erl_lu_idx", - "from": "erl_lu_col", - "to": "keyspace_id", - "read_lock": "exclusive" - }, - "owner": "t2_lookup" - }, - "t2_srl_lu_vdx": { - "type": "lookup_unique", - "params": { - "table": "TestUnsharded.srl_lu_idx", - "from": "srl_lu_col", - "to": "keyspace_id", - "read_lock": "shared" - }, - "owner": "t2_lookup" - }, - "t2_nrl_lu_vdx": { - "type": "lookup_unique", - "params": { - "table": "TestUnsharded.nrl_lu_idx", - "from": "nrl_lu_col", - "to": "keyspace_id", - "read_lock": "none" - }, - "owner": "t2_lookup" - }, - "t2_nv_lu_vdx": { - "type": "lookup_unique", - "params": { - "table": "TestUnsharded.nv_lu_idx", - "from": "nv_lu_col", - "to": "keyspace_id", - "no_verify": "true" - }, - "owner": "t2_lookup" - }, - "t2_lu_vdx": { - "type": "lookup_hash_unique", - "params": { - "table": "TestUnsharded.lu_idx", - "from": "lu_col", - "to": "keyspace_id" - }, - "owner": "t2_lookup" - }, - "regional_vdx": { - "type": "region_experimental", - "params": { - "region_bytes": "1" - } - }, - "cfc": { - "type": "cfc" - } - }, - "tables": { - "user": { - "column_vindexes": [ - { - "column": "Id", - "name": "hash_index" - }, - { - "column": "name", - "name": "name_user_map" - } - ], - "auto_increment": { - "column": "id", - "sequence": "user_seq" - }, - "columns": [ - { - "name": "textcol", - "type": "VARCHAR" - } - ] - }, - "user2": { - "column_vindexes": [ - { - "column": "id", - "name": "hash_index" - }, - { - "columns": ["name", "lastname"], - "name": "name_lastname_keyspace_id_map" - } - ] - }, - "user_extra": { - "column_vindexes": [ - { - "column": "user_id", - "name": "hash_index" - } - ] - }, - "sharded_user_msgs": { - "column_vindexes": [ - { - "column": "user_id", - "name": "hash_index" - } - ] - }, - "music": { - "column_vindexes": [ - { - "column": "user_id", - "name": "hash_index" - }, - { - "column": "id", - "name": "music_user_map" - } - ], - "auto_increment": { - "column": "id", - "sequence": "user_seq" - } - }, - "music_extra": { - "column_vindexes": [ - { - "column": "user_id", - "name": "hash_index" - }, - { - "column": "music_id", - "name": "music_user_map" - } - ] - }, - "music_extra_reversed": { - "column_vindexes": [ - { - "column": "music_id", - "name": "music_user_map" - }, - { - "column": "user_id", - "name": "hash_index" - } - ] - }, - "insert_ignore_test": { - "column_vindexes": [ - { - "column": "pv", - "name": "music_user_map" - }, - { - "column": "owned", - "name": "insert_ignore_idx" - }, - { - "column": "verify", - "name": "hash_index" - } - ] - }, - "noauto_table": { - "column_vindexes": [ - { - "column": "id", - "name": "idx_noauto" - } - ] - }, - "keyrange_table": { - "column_vindexes": [ - { - "column": "krcol_unique", - "name": "krcol_unique_vdx" - }, - { - "column": "krcol", - "name": "krcol_vdx" - } - ] - }, - "ksid_table": { - "column_vindexes": [ - { - "column": "keyspace_id", - "name": "keyspace_id" - } - ] - }, - "t1": { - "column_vindexes": [ - { - "column": "id", - "name": "hash_index" - }, - { - "column": "unq_col", - "name": "t1_lkp_vdx" - } - ] - }, - "t1_lkp_idx": { - "column_vindexes": [ - { - "column": "unq_col", - "name": "hash_index" - } - ] - }, - "t2_lookup": { - "column_vindexes": [ - { - "column": "id", - "name": "hash_index" - }, - { - "column": "wo_lu_col", - "name": "t2_wo_lu_vdx" - }, - { - "column": "erl_lu_col", - "name": "t2_erl_lu_vdx" - }, - { - "column": "srl_lu_col", - "name": "t2_srl_lu_vdx" - }, - { - "column": "nrl_lu_col", - "name": "t2_nrl_lu_vdx" - }, - { - "column": "nv_lu_col", - "name": "t2_nv_lu_vdx" - }, - { - "column": "lu_col", - "name": "t2_lu_vdx" - } - ] - }, - "user_region": { - "column_vindexes": [ - { - "columns": ["cola","colb"], - "name": "regional_vdx" - } - ] - }, - "tbl_cfc": { - "column_vindexes": [ - { - "column": "c1", - "name": "cfc" - } - ], - "columns": [ - { - "name": "c2", - "type": "VARCHAR" - } - ] - }, - "zip_detail": { - "type": "reference", - "source": "TestUnsharded.zip_detail" - } - } -} -` +//go:embed testdata/executorVSchema.json +var executorVSchema string + +//go:embed testdata/unshardedVschema.json +var unshardedVSchema string var badVSchema = ` { @@ -408,36 +61,6 @@ var badVSchema = ` } ` -var unshardedVSchema = ` -{ - "sharded": false, - "tables": { - "user_seq": { - "type": "sequence" - }, - "music_user_map": {}, - "name_user_map": {}, - "name_lastname_keyspace_id_map": {}, - "user_msgs": {}, - "ins_lookup": {}, - "main1": { - "auto_increment": { - "column": "id", - "sequence": "user_seq" - } - }, - "wo_lu_idx": {}, - "erl_lu_idx": {}, - "srl_lu_idx": {}, - "nrl_lu_idx": {}, - "nv_lu_idx": {}, - "lu_idx": {}, - "simple": {}, - "zip_detail": {} - } -} -` - const ( testBufferSize = 10 ) diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index 9dc2dcd6e0f..2ec3efbac46 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -124,9 +124,8 @@ func transformHorizon(ctx *plancontext.PlanningContext, op *operators.Horizon, i } return planLimit(node.Limit, plan) - default: - panic("only SELECT and UNION implement the SelectStatement interface") } + return nil, vterrors.VT13001("only SELECT and UNION implement the SelectStatement interface") } func transformApplyJoinPlan(ctx *plancontext.PlanningContext, n *operators.ApplyJoin) (logicalPlan, error) { @@ -158,25 +157,32 @@ func routeToEngineRoute(ctx *plancontext.PlanningContext, op *operators.Route) ( if err != nil { return nil, err } - var vindex vindexes.Vindex - var values []evalengine.Expr - if op.SelectedVindex() != nil { - vindex = op.Selected.FoundVindex - values = op.Selected.Values + + rp := newRoutingParams(ctx, op.Routing.OpCode()) + err = op.Routing.UpdateRoutingParams(ctx, rp) + if err != nil { + return nil, err } + return &engine.Route{ - TableName: strings.Join(tableNames, ", "), - RoutingParameters: &engine.RoutingParameters{ - Opcode: op.RouteOpCode, - Keyspace: op.Keyspace, - Vindex: vindex, - Values: values, - SysTableTableName: op.SysTableTableName, - SysTableTableSchema: op.SysTableTableSchema, - }, + TableName: strings.Join(tableNames, ", "), + RoutingParameters: rp, }, nil } +func newRoutingParams(ctx *plancontext.PlanningContext, opCode engine.Opcode) *engine.RoutingParameters { + ks, _ := ctx.VSchema.DefaultKeyspace() + if ks == nil { + // if we don't have a selected keyspace, any keyspace will do + // this is used by operators that do not set the keyspace + ks, _ = ctx.VSchema.AnyKeyspace() + } + return &engine.RoutingParameters{ + Opcode: opCode, + Keyspace: ks, + } +} + func transformRoutePlan(ctx *plancontext.PlanningContext, op *operators.Route) (logicalPlan, error) { switch src := op.Source.(type) { case *operators.Update: @@ -204,89 +210,70 @@ func transformRoutePlan(ctx *plancontext.PlanningContext, op *operators.Route) ( } func transformUpdatePlan(ctx *plancontext.PlanningContext, op *operators.Route, upd *operators.Update) (logicalPlan, error) { - var vindex vindexes.Vindex - var values []evalengine.Expr - if op.Selected != nil { - vindex = op.Selected.FoundVindex - values = op.Selected.Values - } ast := upd.AST replaceSubQuery(ctx, ast) + rp := newRoutingParams(ctx, op.Routing.OpCode()) + err := op.Routing.UpdateRoutingParams(ctx, rp) + if err != nil { + return nil, err + } edml := &engine.DML{ Query: generateQuery(ast), Table: []*vindexes.Table{ upd.VTable, }, - OwnedVindexQuery: upd.OwnedVindexQuery, - RoutingParameters: &engine.RoutingParameters{ - Opcode: op.RouteOpCode, - Keyspace: op.Keyspace, - Vindex: vindex, - Values: values, - TargetDestination: op.TargetDestination, - }, + OwnedVindexQuery: upd.OwnedVindexQuery, + RoutingParameters: rp, } - directives := upd.AST.GetParsedComments().Directives() - if directives.IsSet(sqlparser.DirectiveMultiShardAutocommit) { - edml.MultiShardAutocommit = true - } - edml.QueryTimeout = queryTimeout(directives) + transformDMLPlan(upd.AST, upd.VTable, edml, op.Routing, len(upd.ChangedVindexValues) > 0) e := &engine.Update{ ChangedVindexValues: upd.ChangedVindexValues, - } - e.DML = edml - - if op.RouteOpCode != engine.Unsharded && len(upd.ChangedVindexValues) > 0 { - primary := upd.VTable.ColumnVindexes[0] - e.DML.KsidVindex = primary.Vindex - e.DML.KsidLength = len(primary.Columns) + DML: edml, } return &primitiveWrapper{prim: e}, nil } func transformDeletePlan(ctx *plancontext.PlanningContext, op *operators.Route, del *operators.Delete) (logicalPlan, error) { - var vindex vindexes.Vindex - var values []evalengine.Expr - if op.Selected != nil { - vindex = op.Selected.FoundVindex - values = op.Selected.Values - } ast := del.AST replaceSubQuery(ctx, ast) + rp := newRoutingParams(ctx, op.Routing.OpCode()) + err := op.Routing.UpdateRoutingParams(ctx, rp) + if err != nil { + return nil, err + } edml := &engine.DML{ Query: generateQuery(ast), Table: []*vindexes.Table{ del.VTable, }, - OwnedVindexQuery: del.OwnedVindexQuery, - RoutingParameters: &engine.RoutingParameters{ - Opcode: op.RouteOpCode, - Keyspace: op.Keyspace, - Vindex: vindex, - Values: values, - TargetDestination: op.TargetDestination, - }, + OwnedVindexQuery: del.OwnedVindexQuery, + RoutingParameters: rp, + } + + transformDMLPlan(del.AST, del.VTable, edml, op.Routing, del.OwnedVindexQuery != "") + + e := &engine.Delete{ + DML: edml, } - directives := del.AST.GetParsedComments().Directives() + return &primitiveWrapper{prim: e}, nil +} + +func transformDMLPlan(stmt sqlparser.Commented, vtable *vindexes.Table, edml *engine.DML, routing operators.Routing, setVindex bool) { + directives := stmt.GetParsedComments().Directives() if directives.IsSet(sqlparser.DirectiveMultiShardAutocommit) { edml.MultiShardAutocommit = true } edml.QueryTimeout = queryTimeout(directives) - e := &engine.Delete{} - e.DML = edml - - if op.RouteOpCode != engine.Unsharded && del.OwnedVindexQuery != "" { - primary := del.VTable.ColumnVindexes[0] - e.DML.KsidVindex = primary.Vindex - e.DML.KsidLength = len(primary.Columns) + if routing.OpCode() != engine.Unsharded && setVindex { + primary := vtable.ColumnVindexes[0] + edml.KsidVindex = primary.Vindex + edml.KsidLength = len(primary.Columns) } - - return &primitiveWrapper{prim: e}, nil } func replaceSubQuery(ctx *plancontext.PlanningContext, sel sqlparser.Statement) { @@ -304,34 +291,39 @@ func replaceSubQuery(ctx *plancontext.PlanningContext, sel sqlparser.Statement) } func getVindexPredicate(ctx *plancontext.PlanningContext, op *operators.Route) sqlparser.Expr { + tr, ok := op.Routing.(*operators.ShardedRouting) + if !ok || tr.Selected == nil { + return nil + } var condition sqlparser.Expr - if op.Selected != nil { - if len(op.Selected.ValueExprs) > 0 { - condition = op.Selected.ValueExprs[0] - } - _, isMultiColumn := op.Selected.FoundVindex.(vindexes.MultiColumn) - for idx, predicate := range op.Selected.Predicates { - switch predicate := predicate.(type) { - case *sqlparser.ComparisonExpr: - if predicate.Operator == sqlparser.InOp { - switch predicate.Left.(type) { - case *sqlparser.ColName: - if subq, isSubq := predicate.Right.(*sqlparser.Subquery); isSubq { - extractedSubquery := ctx.SemTable.FindSubqueryReference(subq) - if extractedSubquery != nil { - extractedSubquery.SetArgName(engine.ListVarName) - } - } - if isMultiColumn { - predicate.Right = sqlparser.ListArg(engine.ListVarName + strconv.Itoa(idx)) - continue - } - predicate.Right = sqlparser.ListArg(engine.ListVarName) - } - } - } + if len(tr.Selected.ValueExprs) > 0 { + condition = tr.Selected.ValueExprs[0] + } + _, isMultiColumn := tr.Selected.FoundVindex.(vindexes.MultiColumn) + for idx, expr := range tr.Selected.Predicates { + cmp, ok := expr.(*sqlparser.ComparisonExpr) + if !ok || cmp.Operator != sqlparser.InOp { + continue + } + _, ok = cmp.Left.(*sqlparser.ColName) + if !ok { + continue + } + + var argName string + if isMultiColumn { + argName = engine.ListVarName + strconv.Itoa(idx) + } else { + argName = engine.ListVarName } + if subq, isSubq := cmp.Right.(*sqlparser.Subquery); isSubq { + extractedSubquery := ctx.SemTable.FindSubqueryReference(subq) + if extractedSubquery != nil { + extractedSubquery.SetArgName(argName) + } + } + cmp.Right = sqlparser.ListArg(argName) } return condition } diff --git a/go/vt/vtgate/planbuilder/operators/dml_planning.go b/go/vt/vtgate/planbuilder/operators/dml_planning.go index 4fc3965e112..d1d331abe4b 100644 --- a/go/vt/vtgate/planbuilder/operators/dml_planning.go +++ b/go/vt/vtgate/planbuilder/operators/dml_planning.go @@ -59,9 +59,6 @@ func getVindexInformation( return primaryVindex, vindexesAndPredicates, nil } -// buildChangedVindexesValues adds to the plan all the lookup vindexes that are changing. -// Updates can only be performed to secondary lookup vindexes with no complex expressions -// in the set clause. func buildChangedVindexesValues(update *sqlparser.Update, table *vindexes.Table, ksidCols []sqlparser.IdentifierCI) (map[string]*engine.VindexValues, string, error) { changedVindexes := make(map[string]*engine.VindexValues) buf, offset := initialQuery(ksidCols, table) @@ -151,9 +148,9 @@ func extractValueFromUpdate(upd *sqlparser.UpdateExpr) (evalengine.Expr, error) if sq, ok := expr.(*sqlparser.ExtractedSubquery); ok { // if we are planning an update that needs one or more values from the outside, we can trust that they have // been correctly extracted from this query before we reach this far - // if NeedsRewrite is true, it means that this subquery was happily merged with the outer. + // if Merged is true, it means that this subquery was happily merged with the outer. // But in that case we should not be here, so we fail - if sq.NeedsRewrite { + if sq.Merged { return nil, invalidUpdateExpr(upd, expr) } expr = sqlparser.NewArgument(sq.GetArgName()) diff --git a/go/vt/vtgate/planbuilder/operators/info_schema_planning.go b/go/vt/vtgate/planbuilder/operators/info_schema_planning.go new file mode 100644 index 00000000000..2b665fea91d --- /dev/null +++ b/go/vt/vtgate/planbuilder/operators/info_schema_planning.go @@ -0,0 +1,321 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operators + +import ( + "strings" + + "vitess.io/vitess/go/vt/vtgate/planbuilder/operators/ops" + "vitess.io/vitess/go/vt/vtgate/vindexes" + + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" + + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/servenv" + "vitess.io/vitess/go/vt/sqlparser" + "vitess.io/vitess/go/vt/vterrors" + "vitess.io/vitess/go/vt/vtgate/engine" + "vitess.io/vitess/go/vt/vtgate/evalengine" + "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" +) + +// InfoSchemaRouting used for information_schema queries. +// They are special because we usually don't know at plan-time +// what keyspace the query go to, because we don't see normalized literal values +type InfoSchemaRouting struct { + SysTableTableSchema []sqlparser.Expr + SysTableTableName map[string]sqlparser.Expr + Table *QueryTable +} + +func (isr *InfoSchemaRouting) UpdateRoutingParams(_ *plancontext.PlanningContext, rp *engine.RoutingParameters) error { + rp.SysTableTableSchema = nil + for _, expr := range isr.SysTableTableSchema { + eexpr, err := evalengine.Translate(expr, ¬ImplementedSchemaInfoConverter{}) + if err != nil { + return err + } + rp.SysTableTableSchema = append(rp.SysTableTableSchema, eexpr) + } + + rp.SysTableTableName = make(map[string]evalengine.Expr, len(isr.SysTableTableName)) + for k, expr := range isr.SysTableTableName { + eexpr, err := evalengine.Translate(expr, ¬ImplementedSchemaInfoConverter{}) + if err != nil { + return err + } + + rp.SysTableTableName[k] = eexpr + } + return nil +} + +func (isr *InfoSchemaRouting) Clone() Routing { + return &InfoSchemaRouting{ + SysTableTableSchema: slices.Clone(isr.SysTableTableSchema), + SysTableTableName: maps.Clone(isr.SysTableTableName), + Table: isr.Table, + } +} + +func (isr *InfoSchemaRouting) updateRoutingLogic(ctx *plancontext.PlanningContext, expr sqlparser.Expr) (Routing, error) { + isTableSchema, bvName, out := extractInfoSchemaRoutingPredicate(expr, ctx.ReservedVars) + if out == nil { + return isr, nil + } + + if isr.SysTableTableName == nil { + isr.SysTableTableName = map[string]sqlparser.Expr{} + } + + if isTableSchema { + for _, s := range isr.SysTableTableSchema { + if sqlparser.Equals.Expr(out, s) { + // we already have this expression in the list + // stating it again does not add value + return isr, nil + } + } + isr.SysTableTableSchema = append(isr.SysTableTableSchema, out) + } else { + isr.SysTableTableName[bvName] = out + } + return isr, nil +} + +func (isr *InfoSchemaRouting) Cost() int { + return 0 +} + +func (isr *InfoSchemaRouting) OpCode() engine.Opcode { + return engine.DBA +} + +func (isr *InfoSchemaRouting) Keyspace() *vindexes.Keyspace { + // TODO: for some info schema queries, we do know which keyspace it will go to + // if we had this information, more routes could be merged. + return nil +} + +func extractInfoSchemaRoutingPredicate(in sqlparser.Expr, reservedVars *sqlparser.ReservedVars) (bool, string, sqlparser.Expr) { + cmp, ok := in.(*sqlparser.ComparisonExpr) + if !ok || cmp.Operator != sqlparser.EqualOp { + return false, "", nil + } + + isSchemaName, col := isTableOrSchemaRoutable(cmp) + rhs := cmp.Right + if col == nil || !shouldRewrite(rhs) { + return false, "", nil + } + + // here we are just checking if this query can be translated to an evalengine expression + // we'll need to do this translation again later when building the engine.Route + _, err := evalengine.Translate(rhs, ¬ImplementedSchemaInfoConverter{}) + if err != nil { + // if we can't translate this to an evalengine expression, + // we are not going to be able to route based on this expression, + // and might as well move on + return false, "", nil + } + var name string + if isSchemaName { + name = sqltypes.BvSchemaName + } else { + name = reservedVars.ReserveColName(col) + } + cmp.Right = sqlparser.NewArgument(name) + return isSchemaName, name, rhs +} + +// isTableOrSchemaRoutable searches for a comparison where one side is a table or schema name column. +// if it finds the correct column name being used, +// it also makes sure that the LHS of the comparison contains the column, and the RHS the value sought after +func isTableOrSchemaRoutable(cmp *sqlparser.ComparisonExpr) ( + isSchema bool, // tells if we are dealing with a table or a schema name comparator + col *sqlparser.ColName, // which is the colName we are comparing against +) { + if col, schema, table := IsTableSchemaOrName(cmp.Left); schema || table { + return schema, col + } + if col, schema, table := IsTableSchemaOrName(cmp.Right); schema || table { + // to make the rest of the code easier, we shuffle these around so the ColName is always on the LHS + cmp.Right, cmp.Left = cmp.Left, cmp.Right + return schema, col + } + + return false, nil +} + +func tryMergeInfoSchemaRoutings(routingA, routingB Routing, m merger, lhsRoute, rhsRoute *Route) (ops.Operator, error) { + // we have already checked type earlier, so this should always be safe + isrA := routingA.(*InfoSchemaRouting) + isrB := routingB.(*InfoSchemaRouting) + emptyA := len(isrA.SysTableTableName) == 0 && len(isrA.SysTableTableSchema) == 0 + emptyB := len(isrB.SysTableTableName) == 0 && len(isrB.SysTableTableSchema) == 0 + + switch { + // if either side has no predicates to help us route, we can merge them + case emptyA: + return m.merge(lhsRoute, rhsRoute, isrB) + case emptyB: + return m.merge(lhsRoute, rhsRoute, isrA) + + // if we have no schema predicates on either side, we can merge if the table info is the same + case len(isrA.SysTableTableSchema) == 0 && len(isrB.SysTableTableSchema) == 0: + for k, expr := range isrB.SysTableTableName { + if e, found := isrA.SysTableTableName[k]; found && !sqlparser.Equals.Expr(expr, e) { + // schema names are the same, but we have contradicting table names, so we give up + return nil, nil + } + isrA.SysTableTableName[k] = expr + } + return m.merge(lhsRoute, rhsRoute, isrA) + + // if both sides have the same schema predicate, we can safely merge them + case sqlparser.Equals.Exprs(isrA.SysTableTableSchema, isrB.SysTableTableSchema): + for k, expr := range isrB.SysTableTableName { + isrA.SysTableTableName[k] = expr + } + return m.merge(lhsRoute, rhsRoute, isrA) + + // give up + default: + return nil, nil + } + +} + +var ( + // these are filled in by the init() function below + schemaColumns57 = map[string]any{} + schemaColumns80 = map[string]any{} + + schemaColName57 = map[string][]string{ + "COLUMN_PRIVILEGES": {"TABLE_SCHEMA"}, + "COLUMNS": {"TABLE_SCHEMA"}, + "EVENTS": {"EVENT_SCHEMA"}, + "FILES": {"TABLE_SCHEMA"}, + "KEY_COLUMN_USAGE": {"CONSTRAINT_SCHEMA", "TABLE_SCHEMA", "REFERENCED_TABLE_SCHEMA"}, + "PARAMETERS": {"SPECIFIC_SCHEMA"}, + "PARTITIONS": {"TABLE_SCHEMA"}, + "REFERENTIAL_CONSTRAINTS": {"CONSTRAINT_SCHEMA", "UNIQUE_CONSTRAINT_SCHEMA"}, + "ROUTINES": {"ROUTINE_SCHEMA"}, + "SCHEMA_PRIVILEGES": {"TABLE_SCHEMA"}, + "STATISTICS": {"TABLE_SCHEMA"}, + "SCHEMATA": {"SCHEMA_NAME"}, + "TABLE_CONSTRAINTS": {"TABLE_SCHEMA", "CONSTRAINT_SCHEMA"}, + "TABLE_PRIVILEGES": {"TABLE_SCHEMA"}, + "TABLES": {"TABLE_SCHEMA"}, + "TRIGGERS": {"TRIGGER_SCHEMA", "EVENT_OBJECT_SCHEMA"}, + "VIEW": {"TRIGGER_SCHEMA"}, + } + schemaColName80 = map[string][]string{ + "CHECK_CONSTRAINTS": {"CONSTRAINT_SCHEMA"}, + "COLUMN_PRIVILEGES": {"TABLE_SCHEMA"}, + "COLUMN_STATISTICS": {"SCHEMA_NAME"}, + "COLUMNS": {"TABLE_SCHEMA"}, + "COLUMNS_EXTENSIONS": {"TABLE_SCHEMA"}, + "EVENTS": {"EVENT_SCHEMA"}, + "FILES": {"TABLE_SCHEMA"}, + "KEY_COLUMN_USAGE": {"CONSTRAINT_SCHEMA", "TABLE_SCHEMA", "REFERENCED_TABLE_SCHEMA"}, + "PARAMETERS": {"SPECIFIC_SCHEMA"}, + "PARTITIONS": {"TABLE_SCHEMA"}, + "REFERENTIAL_CONSTRAINTS": {"CONSTRAINT_SCHEMA", "UNIQUE_CONSTRAINT_SCHEMA"}, + "ROLE_COLUMN_GRANTS": {"TABLE_SCHEMA"}, + "ROLE_ROUTINE_GRANTS": {"SPECIFIC_SCHEMA", "ROUTINE_SCHEMA"}, + "ROLE_TABLE_GRANTS": {"TABLE_SCHEMA"}, + "ROUTINES": {"ROUTINE_SCHEMA"}, + "SCHEMA_PRIVILEGES": {"TABLE_SCHEMA"}, + "SCHEMATA": {"SCHEMA_NAME"}, + "SCHEMATA_EXTENSIONS": {"SCHEMA_NAME"}, + "ST_GEOMETRY_COLUMNS": {"TABLE_SCHEMA"}, + "STATISTICS": {"TABLE_SCHEMA"}, + "TABLE_CONSTRAINTS": {"TABLE_SCHEMA", "CONSTRAINT_SCHEMA"}, + "TABLE_CONSTRAINTS_EXTENSIONS": {"CONSTRAINT_SCHEMA"}, + "TABLE_PRIVILEGES": {"TABLE_SCHEMA"}, + "TABLES": {"TABLE_SCHEMA"}, + "TABLES_EXTENSIONS": {"TABLE_SCHEMA"}, + "TRIGGERS": {"TRIGGER_SCHEMA", "EVENT_OBJECT_SCHEMA"}, + "VIEW_ROUTINE_USAGE": {"TABLE_SCHEMA", "SPECIFIC_SCHEMA"}, + "VIEW_TABLE_USAGE": {"TABLE_SCHEMA", "VIEW_SCHEMA"}, + "VIEWS": {"TABLE_SCHEMA"}, + } +) + +func init() { + for _, cols := range schemaColName57 { + for _, col := range cols { + schemaColumns57[strings.ToLower(col)] = nil + } + } + for _, cols := range schemaColName80 { + for _, col := range cols { + schemaColumns80[strings.ToLower(col)] = nil + } + } +} + +func shouldRewrite(e sqlparser.Expr) bool { + switch node := e.(type) { + case *sqlparser.FuncExpr: + // we should not rewrite database() calls against information_schema + return !(node.Name.EqualString("database") || node.Name.EqualString("schema")) + } + return true +} + +func IsTableSchemaOrName(e sqlparser.Expr) (col *sqlparser.ColName, isTableSchema bool, isTableName bool) { + col, ok := e.(*sqlparser.ColName) + if !ok { + return nil, false, false + } + return col, isDbNameCol(col), isTableNameCol(col) +} + +func isDbNameCol(col *sqlparser.ColName) bool { + version := servenv.MySQLServerVersion() + var schemaColumns map[string]any + if strings.HasPrefix(version, "5.7") { + schemaColumns = schemaColumns57 + } else { + schemaColumns = schemaColumns80 + } + + _, found := schemaColumns[col.Name.Lowered()] + return found +} + +func isTableNameCol(col *sqlparser.ColName) bool { + return col.Name.EqualString("table_name") || col.Name.EqualString("referenced_table_name") +} + +type notImplementedSchemaInfoConverter struct{} + +func (f *notImplementedSchemaInfoConverter) ColumnLookup(*sqlparser.ColName) (int, error) { + return 0, vterrors.VT12001("comparing table schema name with a column name") +} + +func (f *notImplementedSchemaInfoConverter) CollationForExpr(sqlparser.Expr) collations.ID { + return collations.Unknown +} + +func (f *notImplementedSchemaInfoConverter) DefaultCollation() collations.ID { + return collations.Default() +} diff --git a/go/vt/vtgate/planbuilder/operators/logical.go b/go/vt/vtgate/planbuilder/operators/logical.go index e843f115f57..87bcd42709e 100644 --- a/go/vt/vtgate/planbuilder/operators/logical.go +++ b/go/vt/vtgate/planbuilder/operators/logical.go @@ -116,7 +116,7 @@ func createOperatorFromUpdate(ctx *plancontext.PlanningContext, updStmt *sqlpars assignments[set.Name.Name.String()] = set.Expr } - vindexTable, opCode, dest, err := buildVindexTableForDML(ctx, tableInfo, qt, "update") + vindexTable, routing, err := buildVindexTableForDML(ctx, tableInfo, qt, "update") if err != nil { return nil, err } @@ -126,33 +126,36 @@ func createOperatorFromUpdate(ctx *plancontext.PlanningContext, updStmt *sqlpars return nil, err } - r := &Route{ - Source: &Update{ - QTable: qt, - VTable: vindexTable, - Assignments: assignments, - ChangedVindexValues: cvv, - OwnedVindexQuery: ovq, - AST: updStmt, - }, - RouteOpCode: opCode, - Keyspace: vindexTable.Keyspace, - VindexPreds: vp, - TargetDestination: dest, + tr, ok := routing.(*ShardedRouting) + if ok { + tr.VindexPreds = vp } for _, predicate := range qt.Predicates { - err := r.UpdateRoutingLogic(ctx, predicate) + var err error + routing, err = UpdateRoutingLogic(ctx, predicate, routing) if err != nil { return nil, err } } - if r.RouteOpCode == engine.Scatter && updStmt.Limit != nil { + if routing.OpCode() == engine.Scatter && updStmt.Limit != nil { // TODO systay: we should probably check for other op code types - IN could also hit multiple shards (2022-04-07) return nil, vterrors.VT12001("multi shard UPDATE with LIMIT") } + r := &Route{ + Source: &Update{ + QTable: qt, + VTable: vindexTable, + Assignments: assignments, + ChangedVindexValues: cvv, + OwnedVindexQuery: ovq, + AST: updStmt, + }, + Routing: routing, + } + subq, err := createSubqueryFromStatement(ctx, updStmt) if err != nil { return nil, err @@ -170,7 +173,7 @@ func createOperatorFromDelete(ctx *plancontext.PlanningContext, deleteStmt *sqlp return nil, err } - vindexTable, opCode, dest, err := buildVindexTableForDML(ctx, tableInfo, qt, "delete") + vindexTable, routing, err := buildVindexTableForDML(ctx, tableInfo, qt, "delete") if err != nil { return nil, err } @@ -181,10 +184,8 @@ func createOperatorFromDelete(ctx *plancontext.PlanningContext, deleteStmt *sqlp AST: deleteStmt, } route := &Route{ - Source: del, - RouteOpCode: opCode, - Keyspace: vindexTable.Keyspace, - TargetDestination: dest, + Source: del, + Routing: routing, } if !vindexTable.Keyspace.Sharded { @@ -196,7 +197,10 @@ func createOperatorFromDelete(ctx *plancontext.PlanningContext, deleteStmt *sqlp return nil, err } - route.VindexPreds = vindexAndPredicates + tr, ok := routing.(*ShardedRouting) + if ok { + tr.VindexPreds = vindexAndPredicates + } var ovq string if len(vindexTable.Owned) > 0 { @@ -207,13 +211,14 @@ func createOperatorFromDelete(ctx *plancontext.PlanningContext, deleteStmt *sqlp del.OwnedVindexQuery = ovq for _, predicate := range qt.Predicates { - err := route.UpdateRoutingLogic(ctx, predicate) + var err error + route.Routing, err = UpdateRoutingLogic(ctx, predicate, route.Routing) if err != nil { return nil, err } } - if route.RouteOpCode == engine.Scatter && deleteStmt.Limit != nil { + if routing.OpCode() == engine.Scatter && deleteStmt.Limit != nil { // TODO systay: we should probably check for other op code types - IN could also hit multiple shards (2022-04-07) return nil, vterrors.VT12001("multi shard DELETE with LIMIT") } diff --git a/go/vt/vtgate/planbuilder/operators/merging.go b/go/vt/vtgate/planbuilder/operators/merging.go new file mode 100644 index 00000000000..369016e28b9 --- /dev/null +++ b/go/vt/vtgate/planbuilder/operators/merging.go @@ -0,0 +1,324 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operators + +import ( + "fmt" + "reflect" + + "vitess.io/vitess/go/vt/sqlparser" + "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" + + "vitess.io/vitess/go/vt/vtgate/planbuilder/operators/ops" +) + +// Merge checks whether two operators can be merged into a single one. +// If they can be merged, a new operator with the merged routing is returned +// If they cannot be merged, nil is returned. +func Merge(ctx *plancontext.PlanningContext, lhs, rhs ops.Operator, joinPredicates []sqlparser.Expr, m merger) (ops.Operator, error) { + lhsRoute, rhsRoute := operatorsToRoutes(lhs, rhs) + if lhsRoute == nil || rhsRoute == nil { + return nil, nil + } + + lhsRoute, rhsRoute, routingA, routingB, sameKeyspace := getRoutesOrAlternates(lhsRoute, rhsRoute) + + a, b := getRoutingType(routingA), getRoutingType(routingB) + if getTypeName(routingA) < getTypeName(routingB) { + // while deciding if two routes can be merged, the LHS/RHS order of the routes is not important. + // for the actual merging, we still need to remember which side was inner and which was outer for subqueries + a, b = b, a + routingA, routingB = routingB, routingA + } + + switch { + // if either side is a dual query, we can always merge them together + case a == dual: + return m.merge(lhsRoute, rhsRoute, routingB) + case b == dual: + return m.merge(lhsRoute, rhsRoute, routingA) + + // an unsharded/reference route can be merged with anything going to that keyspace + case a == anyShard && sameKeyspace: + return m.merge(lhsRoute, rhsRoute, routingB) + case b == anyShard && sameKeyspace: + return m.merge(lhsRoute, rhsRoute, routingA) + + // None routing can always be merged, as long as we are aiming for the same keyspace + case a == none && sameKeyspace: + return m.merge(lhsRoute, rhsRoute, routingA) + case b == none && sameKeyspace: + return m.merge(lhsRoute, rhsRoute, routingB) + + // infoSchema routing is complex, so we handle it in a separate method + case a == infoSchema && b == infoSchema: + return tryMergeInfoSchemaRoutings(routingA, routingB, m, lhsRoute, rhsRoute) + + // sharded routing is complex, so we handle it in a separate method + case a == sharded && b == sharded: + return tryMergeShardedRouting(ctx, lhsRoute, rhsRoute, m, joinPredicates) + + default: + return nil, nil + } +} + +type ( + merger interface { + mergeTables(r1, r2 *ShardedRouting, op1, op2 *Route) (*Route, error) + merge(op1, op2 *Route, r Routing) (*Route, error) + } + + joinMerger struct { + ctx *plancontext.PlanningContext + predicates []sqlparser.Expr + innerJoin bool + } + + subQueryMerger struct { + ctx *plancontext.PlanningContext + subq *SubQueryInner + } + + // mergeDecorator runs the inner merge and also runs the additional function f. + mergeDecorator struct { + inner merger + f func() error + } + + routingType int +) + +const ( + sharded routingType = iota + infoSchema + anyShard + none + dual + targeted +) + +func (rt routingType) String() string { + switch rt { + case sharded: + return "sharded" + case infoSchema: + return "infoSchema" + case anyShard: + return "anyShard" + case none: + return "none" + case dual: + return "dual" + case targeted: + return "targeted" + } + panic("switch should be exhaustive") +} + +// getRoutesOrAlternates gets the Routings from each Route. If they are from different keyspaces, +// we check if this is a table with alternates in other keyspaces that we can use +func getRoutesOrAlternates(lhsRoute, rhsRoute *Route) (*Route, *Route, Routing, Routing, bool) { + routingA := lhsRoute.Routing + routingB := rhsRoute.Routing + sameKeyspace := routingA.Keyspace() == routingB.Keyspace() + + if sameKeyspace || + // if either of these is missing a keyspace, we are not going to be able to find an alternative + routingA.Keyspace() == nil || + routingB.Keyspace() == nil { + return lhsRoute, rhsRoute, routingA, routingB, sameKeyspace + } + + if refA, ok := routingA.(*AnyShardRouting); ok { + if altARoute := refA.AlternateInKeyspace(routingB.Keyspace()); altARoute != nil { + return altARoute, rhsRoute, altARoute.Routing, routingB, true + } + } + + if refB, ok := routingB.(*AnyShardRouting); ok { + if altBRoute := refB.AlternateInKeyspace(routingA.Keyspace()); altBRoute != nil { + return lhsRoute, altBRoute, routingA, altBRoute.Routing, true + } + } + + return lhsRoute, rhsRoute, routingA, routingB, sameKeyspace +} + +func getTypeName(myvar interface{}) string { + return reflect.TypeOf(myvar).String() +} + +func getRoutingType(r Routing) routingType { + switch r.(type) { + case *InfoSchemaRouting: + return infoSchema + case *AnyShardRouting: + return anyShard + case *DualRouting: + return dual + case *ShardedRouting: + return sharded + case *NoneRouting: + return none + case *TargetedRouting: + return targeted + } + panic(fmt.Sprintf("switch should be exhaustive, got %T", r)) +} + +func newJoinMerge(ctx *plancontext.PlanningContext, predicates []sqlparser.Expr, innerJoin bool) merger { + return &joinMerger{ + ctx: ctx, + predicates: predicates, + innerJoin: innerJoin, + } +} + +func (jm *joinMerger) mergeTables(r1, r2 *ShardedRouting, op1, op2 *Route) (*Route, error) { + tr := &ShardedRouting{ + VindexPreds: append(r1.VindexPreds, r2.VindexPreds...), + keyspace: r1.keyspace, + RouteOpCode: r1.RouteOpCode, + SeenPredicates: r1.SeenPredicates, + } + if r1.SelectedVindex() == r2.SelectedVindex() { + tr.Selected = r1.Selected + } else { + tr.PickBestAvailableVindex() + } + + return &Route{ + Source: jm.getApplyJoin(op1, op2), + MergedWith: []*Route{op2}, + Routing: tr, + }, nil +} + +func (jm *joinMerger) getApplyJoin(op1, op2 *Route) *ApplyJoin { + return NewApplyJoin(op1.Source, op2.Source, jm.ctx.SemTable.AndExpressions(jm.predicates...), !jm.innerJoin) +} + +func (jm *joinMerger) merge(op1, op2 *Route, r Routing) (*Route, error) { + return &Route{ + Source: jm.getApplyJoin(op1, op2), + MergedWith: []*Route{op2}, + Routing: r, + }, nil +} + +func newSubQueryMerge(ctx *plancontext.PlanningContext, subq *SubQueryInner) merger { + return &subQueryMerger{ctx: ctx, subq: subq} +} + +// markPredicateInOuterRouting merges a subquery with the outer routing. +// If the subquery was a predicate on the outer side, we see if we can use +// predicates from the subquery to help with routing +func (s *subQueryMerger) markPredicateInOuterRouting(outer *ShardedRouting, inner Routing) (Routing, error) { + // When merging an inner query with its outer query, we can remove the + // inner query from the list of predicates that can influence routing of + // the outer query. + // + // Note that not all inner queries necessarily are part of the routing + // predicates list, so this might be a no-op. + subQueryWasPredicate := false + for i, predicate := range outer.SeenPredicates { + if s.ctx.SemTable.EqualsExpr(predicate, s.subq.ExtractedSubquery) { + outer.SeenPredicates = append(outer.SeenPredicates[:i], outer.SeenPredicates[i+1:]...) + + subQueryWasPredicate = true + + // The `ExtractedSubquery` of an inner query is unique (due to the uniqueness of bind variable names) + // so we can stop after the first match. + break + } + } + + if !subQueryWasPredicate { + // if the subquery was not a predicate, we are done here + return outer, nil + } + + switch inner := inner.(type) { + case *ShardedRouting: + // Copy Vindex predicates from the inner route to the upper route. + // If we can route based on some of these predicates, the routing can improve + outer.VindexPreds = append(outer.VindexPreds, inner.VindexPreds...) + outer.SeenPredicates = append(outer.SeenPredicates, inner.SeenPredicates...) + routing, err := outer.ResetRoutingLogic(s.ctx) + if err != nil { + return nil, err + } + return routing, nil + case *NoneRouting: + // if we have an ANDed subquery, and we know that it will not find anything, + // we can safely assume that the outer query will also not return anything + return &NoneRouting{keyspace: outer.keyspace}, nil + default: + return outer, nil + } +} + +func (s *subQueryMerger) mergeTables(outer, inner *ShardedRouting, op1, op2 *Route) (*Route, error) { + s.subq.ExtractedSubquery.Merged = true + + routing, err := s.markPredicateInOuterRouting(outer, inner) + if err != nil { + return nil, err + } + op1.Routing = routing + op1.MergedWith = append(op1.MergedWith, op2) + return op1, nil +} + +func (s *subQueryMerger) merge(outer, inner *Route, routing Routing) (*Route, error) { + s.subq.ExtractedSubquery.Merged = true + + if outerSR, ok := outer.Routing.(*ShardedRouting); ok { + var err error + routing, err = s.markPredicateInOuterRouting(outerSR, inner.Routing) + if err != nil { + return nil, err + } + } + + outer.Routing = routing + outer.MergedWith = append(outer.MergedWith, inner) + return outer, nil +} + +func (d *mergeDecorator) mergeTables(outer, inner *ShardedRouting, op1, op2 *Route) (*Route, error) { + merged, err := d.inner.mergeTables(outer, inner, op1, op2) + if err != nil { + return nil, err + } + if err := d.f(); err != nil { + return nil, err + } + return merged, nil +} + +func (d *mergeDecorator) merge(outer, inner *Route, r Routing) (*Route, error) { + merged, err := d.inner.merge(outer, inner, r) + if err != nil { + return nil, err + } + if err := d.f(); err != nil { + return nil, err + } + return merged, nil +} diff --git a/go/vt/vtgate/planbuilder/operators/misc_routing.go b/go/vt/vtgate/planbuilder/operators/misc_routing.go new file mode 100644 index 00000000000..81301f975b4 --- /dev/null +++ b/go/vt/vtgate/planbuilder/operators/misc_routing.go @@ -0,0 +1,210 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operators + +import ( + "vitess.io/vitess/go/vt/key" + "vitess.io/vitess/go/vt/sqlparser" + "vitess.io/vitess/go/vt/vtgate/engine" + "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" + "vitess.io/vitess/go/vt/vtgate/vindexes" +) + +type ( + // NoneRouting is used when we know that this Route will return no results. + // Can be merged with any other route going to the same keyspace + NoneRouting struct { + keyspace *vindexes.Keyspace + } + + // TargetedRouting is used when the user has used syntax to target the + // Route against a specific set of shards and/or tablet type. Can't be merged with anything else. + TargetedRouting struct { + keyspace *vindexes.Keyspace + + // targetDestination specifies an explicit target destination tablet type + TargetDestination key.Destination + } + + // AnyShardRouting is used for routing logic where any shard in the keyspace can be used. + // Shared by unsharded and reference routing + AnyShardRouting struct { + keyspace *vindexes.Keyspace + Alternates map[*vindexes.Keyspace]*Route + } + + // DualRouting represents the dual-table. + // It is special compared to all other tables because it can be merged with tables in any keyspace + DualRouting struct{} + + SequenceRouting struct { + keyspace *vindexes.Keyspace + } +) + +var ( + _ Routing = (*NoneRouting)(nil) + _ Routing = (*TargetedRouting)(nil) + _ Routing = (*AnyShardRouting)(nil) + _ Routing = (*DualRouting)(nil) + _ Routing = (*SequenceRouting)(nil) +) + +func (tr *TargetedRouting) UpdateRoutingParams(_ *plancontext.PlanningContext, rp *engine.RoutingParameters) error { + rp.Keyspace = tr.keyspace + rp.TargetDestination = tr.TargetDestination + return nil +} + +func (tr *TargetedRouting) Clone() Routing { + newTr := *tr + return &newTr +} + +func (tr *TargetedRouting) updateRoutingLogic(_ *plancontext.PlanningContext, _ sqlparser.Expr) (Routing, error) { + return tr, nil +} + +func (tr *TargetedRouting) Cost() int { + return 1 +} + +func (tr *TargetedRouting) OpCode() engine.Opcode { + return engine.ByDestination +} + +func (tr *TargetedRouting) Keyspace() *vindexes.Keyspace { + return tr.keyspace +} + +func (n *NoneRouting) UpdateRoutingParams(_ *plancontext.PlanningContext, rp *engine.RoutingParameters) error { + rp.Keyspace = n.keyspace + return nil +} + +func (n *NoneRouting) Clone() Routing { + return n +} + +func (n *NoneRouting) updateRoutingLogic(*plancontext.PlanningContext, sqlparser.Expr) (Routing, error) { + return n, nil +} + +func (n *NoneRouting) Cost() int { + return 0 +} + +func (n *NoneRouting) OpCode() engine.Opcode { + return engine.None +} + +func (n *NoneRouting) Keyspace() *vindexes.Keyspace { + return n.keyspace +} + +func (rr *AnyShardRouting) UpdateRoutingParams(_ *plancontext.PlanningContext, rp *engine.RoutingParameters) error { + rp.Keyspace = rr.keyspace + return nil +} + +func (rr *AnyShardRouting) Clone() Routing { + return &AnyShardRouting{ + keyspace: rr.keyspace, + Alternates: rr.Alternates, + } +} + +func (rr *AnyShardRouting) updateRoutingLogic(*plancontext.PlanningContext, sqlparser.Expr) (Routing, error) { + return rr, nil +} + +func (rr *AnyShardRouting) Cost() int { + return 0 +} + +func (rr *AnyShardRouting) OpCode() engine.Opcode { + if rr.keyspace.Sharded { + return engine.Reference + } + return engine.Unsharded +} + +func (rr *AnyShardRouting) Keyspace() *vindexes.Keyspace { + return rr.keyspace +} + +func (rr *AnyShardRouting) AlternateInKeyspace(keyspace *vindexes.Keyspace) *Route { + if keyspace.Name == rr.keyspace.Name { + return nil + } + + if route, ok := rr.Alternates[keyspace]; ok { + return route + } + + return nil +} + +func (dr *DualRouting) UpdateRoutingParams(*plancontext.PlanningContext, *engine.RoutingParameters) error { + return nil +} + +func (dr *DualRouting) Clone() Routing { + return &DualRouting{} +} + +func (dr *DualRouting) updateRoutingLogic(*plancontext.PlanningContext, sqlparser.Expr) (Routing, error) { + return dr, nil +} + +func (dr *DualRouting) Cost() int { + return 0 +} + +func (dr *DualRouting) OpCode() engine.Opcode { + return engine.Reference +} + +func (dr *DualRouting) Keyspace() *vindexes.Keyspace { + return nil +} + +func (sr *SequenceRouting) UpdateRoutingParams(_ *plancontext.PlanningContext, rp *engine.RoutingParameters) error { + rp.Opcode = engine.Next + rp.Keyspace = sr.keyspace + return nil +} + +func (sr *SequenceRouting) Clone() Routing { + return &SequenceRouting{keyspace: sr.keyspace} +} + +func (sr *SequenceRouting) updateRoutingLogic(*plancontext.PlanningContext, sqlparser.Expr) (Routing, error) { + return sr, nil +} + +func (sr *SequenceRouting) Cost() int { + return 0 +} + +func (sr *SequenceRouting) OpCode() engine.Opcode { + return engine.Next +} + +func (sr *SequenceRouting) Keyspace() *vindexes.Keyspace { + return nil +} diff --git a/go/vt/vtgate/planbuilder/operators/route.go b/go/vt/vtgate/planbuilder/operators/route.go index 46d9515b707..276de2a23c1 100644 --- a/go/vt/vtgate/planbuilder/operators/route.go +++ b/go/vt/vtgate/planbuilder/operators/route.go @@ -17,8 +17,6 @@ limitations under the License. package operators import ( - "vitess.io/vitess/go/mysql/collations" - "vitess.io/vitess/go/vt/key" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/engine" @@ -34,33 +32,10 @@ type ( Route struct { Source ops.Operator - RouteOpCode engine.Opcode - Keyspace *vindexes.Keyspace - - // here we store the possible vindexes we can use so that when we add predicates to the plan, - // we can quickly check if the new predicates enables any new vindex Options - VindexPreds []*VindexPlusPredicates - - // the best option available is stored here - Selected *VindexOption - - // The following two fields are used when routing information_schema queries - SysTableTableSchema []evalengine.Expr - SysTableTableName map[string]evalengine.Expr - - // SeenPredicates contains all the predicates that have had a chance to influence routing. - // If we need to replan routing, we'll use this list - SeenPredicates []sqlparser.Expr - - // TargetDestination specifies an explicit target destination tablet type - TargetDestination key.Destination - - // Alternates contains alternate routes to equivalent sources in - // other keyspaces. - Alternates map[*vindexes.Keyspace]*Route - // Routes that have been merged into this one. MergedWith []*Route + + Routing Routing } // VindexPlusPredicates is a struct used to store all the predicates that the vindex can be used to query @@ -91,289 +66,127 @@ type ( IsUnique bool OpCode engine.Opcode } -) -var _ ops.PhysicalOperator = (*Route)(nil) + // Routing is used for the routing and merging logic of `Route`s. Every Route has a Routing object, and + // this object is updated when predicates are found, and when merging `Route`s together + Routing interface { + // UpdateRoutingParams allows a Routing to control the routing params that will be used by the engine Route + // OpCode is already set, and the default keyspace is set for read queries + UpdateRoutingParams(ctx *plancontext.PlanningContext, rp *engine.RoutingParameters) error -// IPhysical implements the PhysicalOperator interface -func (*Route) IPhysical() {} + // Clone returns a copy of the routing. Since we are trying different variation of merging, + // one Routing can be used in different constellations. + // We don't want these different alternatives to influence each other, and cloning allows this + Clone() Routing -// Cost implements the Operator interface -func (r *Route) Cost() int { - switch r.RouteOpCode { - case // these op codes will never be compared with each other - they are assigned by a rule and not a comparison - engine.DBA, - engine.Next, - engine.None, - engine.Reference, - engine.Unsharded: - return 0 - // TODO revisit these costs when more of the gen4 planner is done - case engine.EqualUnique: - return 1 - case engine.Equal: - return 5 - case engine.IN: - return 10 - case engine.MultiEqual: - return 10 - case engine.Scatter: - return 20 - } - return 1 -} + // Cost returns the cost of this Route. + Cost() int + OpCode() engine.Opcode + Keyspace() *vindexes.Keyspace // note that all routings do not have a keyspace, so this method can return nil -// Clone implements the Operator interface -func (r *Route) Clone(inputs []ops.Operator) ops.Operator { - cloneRoute := *r - cloneRoute.Source = inputs[0] - cloneRoute.VindexPreds = make([]*VindexPlusPredicates, len(r.VindexPreds)) - for i, pred := range r.VindexPreds { - // we do this to create a copy of the struct - p := *pred - cloneRoute.VindexPreds[i] = &p + // updateRoutingLogic updates the routing to take predicates into account. This can be used for routing + // using vindexes or for figuring out which keyspace an information_schema query should be sent to. + updateRoutingLogic(ctx *plancontext.PlanningContext, expr sqlparser.Expr) (Routing, error) } - return &cloneRoute -} - -// Inputs implements the Operator interface -func (r *Route) Inputs() []ops.Operator { - return []ops.Operator{r.Source} -} +) -func (r *Route) UpdateRoutingLogic(ctx *plancontext.PlanningContext, expr sqlparser.Expr) error { - r.SeenPredicates = append(r.SeenPredicates, expr) - return r.tryImprovingVindex(ctx, expr) -} +var _ ops.PhysicalOperator = (*Route)(nil) -func (r *Route) tryImprovingVindex(ctx *plancontext.PlanningContext, expr sqlparser.Expr) error { - if r.canImprove() { - newVindexFound, err := r.searchForNewVindexes(ctx, expr) +// UpdateRoutingLogic first checks if we are dealing with a predicate that +func UpdateRoutingLogic(ctx *plancontext.PlanningContext, expr sqlparser.Expr, r Routing) (Routing, error) { + ks := r.Keyspace() + if ks == nil { + var err error + ks, err = ctx.VSchema.AnyKeyspace() if err != nil { - return err - } - - // if we didn't open up any new vindex Options, no need to enter here - if newVindexFound { - r.PickBestAvailableVindex() + return nil, err } } - return nil -} + nr := &NoneRouting{keyspace: ks} -func (r *Route) searchForNewVindexes(ctx *plancontext.PlanningContext, predicate sqlparser.Expr) (bool, error) { - newVindexFound := false - switch node := predicate.(type) { - case *sqlparser.ExtractedSubquery: - originalCmp, ok := node.Original.(*sqlparser.ComparisonExpr) - if !ok { - break - } + if isConstantFalse(expr) { + return nr, nil + } - // using the node.subquery which is the rewritten version of our subquery - cmp := &sqlparser.ComparisonExpr{ - Left: node.OtherSide, - Right: &sqlparser.Subquery{Select: node.Subquery.Select}, - Operator: originalCmp.Operator, - } - found, exitEarly, err := r.planComparison(ctx, cmp) - if err != nil || exitEarly { - return false, err - } - newVindexFound = newVindexFound || found + exit := func() (Routing, error) { + return r.updateRoutingLogic(ctx, expr) + } - case *sqlparser.ComparisonExpr: - found, exitEarly, err := r.planComparison(ctx, node) - if err != nil || exitEarly { - return false, err - } - newVindexFound = newVindexFound || found - case *sqlparser.IsExpr: - found := r.planIsExpr(ctx, node) - newVindexFound = newVindexFound || found + // For some expressions, even if we can't evaluate them, we know that they will always return false or null + cmp, ok := expr.(*sqlparser.ComparisonExpr) + if !ok { + return exit() } - return newVindexFound, nil -} -func (r *Route) planComparison(ctx *plancontext.PlanningContext, cmp *sqlparser.ComparisonExpr) (found bool, exitEarly bool, err error) { if cmp.Operator != sqlparser.NullSafeEqualOp && (sqlparser.IsNull(cmp.Left) || sqlparser.IsNull(cmp.Right)) { - // we are looking at ANDed predicates in the WHERE clause. - // since we know that nothing returns true when compared to NULL, - // so we can safely bail out here - r.setSelectNoneOpcode() - return false, true, nil + // any comparison against a literal null, except a null safe equality (<=>), will return null + return nr, nil + } + + tuples, ok := cmp.Right.(sqlparser.ValTuple) + if !ok { + return exit() } switch cmp.Operator { - case sqlparser.EqualOp: - found := r.planEqualOp(ctx, cmp) - return found, false, nil - case sqlparser.InOp: - if r.isImpossibleIN(cmp) { - return false, true, nil - } - found := r.planInOp(ctx, cmp) - return found, false, nil case sqlparser.NotInOp: - // NOT IN is always a scatter, except when we can be sure it would return nothing - if r.isImpossibleNotIN(cmp) { - return false, true, nil + for _, n := range tuples { + // If any of the values in the tuple is a literal null, we know that this comparison will always return NULL + if sqlparser.IsNull(n) { + return nr, nil + } + } + case sqlparser.InOp: + // WHERE col IN (null) + if len(tuples) == 1 && sqlparser.IsNull(tuples[0]) { + return nr, nil } - case sqlparser.LikeOp: - found := r.planLikeOp(ctx, cmp) - return found, false, nil - } - return false, false, nil -} -func (r *Route) setSelectNoneOpcode() { - r.RouteOpCode = engine.None - // clear any chosen vindex as this query does not need to be sent down. - r.Selected = nil + return exit() } -func (r *Route) planEqualOp(ctx *plancontext.PlanningContext, node *sqlparser.ComparisonExpr) bool { - column, ok := node.Left.(*sqlparser.ColName) - other := node.Right - vdValue := other - if !ok { - column, ok = node.Right.(*sqlparser.ColName) - if !ok { - // either the LHS or RHS have to be a column to be useful for the vindex - return false - } - vdValue = node.Left +// isConstantFalse checks whether this predicate can be evaluated at plan-time. If it returns `false` or `null`, +// we know that the query will not return anything, and this can be used to produce better plans +func isConstantFalse(expr sqlparser.Expr) bool { + eenv := evalengine.EmptyExpressionEnv() + eexpr, err := evalengine.Translate(expr, nil) + if err != nil { + return false } - val := r.makeEvalEngineExpr(ctx, vdValue) - if val == nil { + eres, err := eenv.Evaluate(eexpr) + if err != nil { return false } - - return r.haveMatchingVindex(ctx, node, vdValue, column, val, equalOrEqualUnique, justTheVindex) + if eres.Value().IsNull() { + return false + } + b, err := eres.ToBooleanStrict() + if err != nil { + return false + } + return !b } -// makePlanValue transforms the given sqlparser.Expr into a sqltypes.PlanValue. -// If the given sqlparser.Expr is an argument and can be found in the r.argToReplaceBySelect then the -// method will stops and return nil values. -// Otherwise, the method will try to apply makePlanValue for any equality the sqlparser.Expr n has. -// The first PlanValue that is successfully produced will be returned. -func (r *Route) makeEvalEngineExpr(ctx *plancontext.PlanningContext, n sqlparser.Expr) evalengine.Expr { - if ctx.IsSubQueryToReplace(n) { - return nil - } - - for _, expr := range ctx.SemTable.GetExprAndEqualities(n) { - if subq, isSubq := expr.(*sqlparser.Subquery); isSubq { - extractedSubquery := ctx.SemTable.FindSubqueryReference(subq) - if extractedSubquery == nil { - continue - } - switch engine.PulloutOpcode(extractedSubquery.OpCode) { - case engine.PulloutIn, engine.PulloutNotIn: - expr = sqlparser.NewListArg(extractedSubquery.GetArgName()) - case engine.PulloutValue, engine.PulloutExists: - expr = sqlparser.NewArgument(extractedSubquery.GetArgName()) - } - } - pv, _ := evalengine.Translate(expr, ctx.SemTable) - if pv != nil { - return pv - } - } +// IPhysical implements the PhysicalOperator interface +func (*Route) IPhysical() {} - return nil +// Cost implements the Operator interface +func (r *Route) Cost() int { + return r.Routing.Cost() } -func (r *Route) hasVindex(column *sqlparser.ColName) bool { - for _, v := range r.VindexPreds { - for _, col := range v.ColVindex.Columns { - if column.Name.Equal(col) { - return true - } - } - } - return false +// Clone implements the Operator interface +func (r *Route) Clone(inputs []ops.Operator) ops.Operator { + cloneRoute := *r + cloneRoute.Source = inputs[0] + cloneRoute.Routing = r.Routing.Clone() + return &cloneRoute } -func (r *Route) haveMatchingVindex( - ctx *plancontext.PlanningContext, - node sqlparser.Expr, - valueExpr sqlparser.Expr, - column *sqlparser.ColName, - value evalengine.Expr, - opcode func(*vindexes.ColumnVindex) engine.Opcode, - vfunc func(*vindexes.ColumnVindex) vindexes.Vindex, -) bool { - newVindexFound := false - for _, v := range r.VindexPreds { - // check that the - if !ctx.SemTable.DirectDeps(column).IsSolvedBy(v.TableID) { - continue - } - switch v.ColVindex.Vindex.(type) { - case vindexes.SingleColumn: - col := v.ColVindex.Columns[0] - if column.Name.Equal(col) { - // single column vindex - just add the option - routeOpcode := opcode(v.ColVindex) - vindex := vfunc(v.ColVindex) - if vindex == nil || routeOpcode == engine.Scatter { - continue - } - v.Options = append(v.Options, &VindexOption{ - Values: []evalengine.Expr{value}, - ValueExprs: []sqlparser.Expr{valueExpr}, - Predicates: []sqlparser.Expr{node}, - OpCode: routeOpcode, - FoundVindex: vindex, - Cost: costFor(v.ColVindex, routeOpcode), - Ready: true, - }) - newVindexFound = true - } - case vindexes.MultiColumn: - colLoweredName := "" - indexOfCol := -1 - for idx, col := range v.ColVindex.Columns { - if column.Name.Equal(col) { - colLoweredName = column.Name.Lowered() - indexOfCol = idx - break - } - } - if colLoweredName == "" { - break - } - - var newOption []*VindexOption - for _, op := range v.Options { - if op.Ready { - continue - } - _, isPresent := op.ColsSeen[colLoweredName] - if isPresent { - continue - } - option := copyOption(op) - optionReady := option.updateWithNewColumn(colLoweredName, valueExpr, indexOfCol, value, node, v.ColVindex, opcode) - if optionReady { - newVindexFound = true - } - newOption = append(newOption, option) - } - v.Options = append(v.Options, newOption...) - - // multi column vindex - just always add as new option - option := createOption(v.ColVindex, vfunc) - optionReady := option.updateWithNewColumn(colLoweredName, valueExpr, indexOfCol, value, node, v.ColVindex, opcode) - if optionReady { - newVindexFound = true - } - v.Options = append(v.Options, option) - } - } - return newVindexFound +// Inputs implements the Operator interface +func (r *Route) Inputs() []ops.Operator { + return []ops.Operator{r.Source} } func createOption( @@ -438,192 +251,14 @@ func (option *VindexOption) updateWithNewColumn( return option.Ready } -// PickBestAvailableVindex goes over the available vindexes for this route and picks the best one available. -func (r *Route) PickBestAvailableVindex() { - for _, v := range r.VindexPreds { - option := v.bestOption() - if option != nil && (r.Selected == nil || less(option.Cost, r.Selected.Cost)) { - r.Selected = option - r.RouteOpCode = option.OpCode - } - } -} - -// canImprove returns true if additional predicates could help improving this plan -func (r *Route) canImprove() bool { - return r.RouteOpCode != engine.None -} - func (r *Route) IsSingleShard() bool { - switch r.RouteOpCode { + switch r.Routing.OpCode() { case engine.Unsharded, engine.DBA, engine.Next, engine.EqualUnique, engine.Reference: return true } return false } -func (r *Route) SelectedVindex() vindexes.Vindex { - if r.Selected == nil { - return nil - } - return r.Selected.FoundVindex -} - -func (r *Route) VindexExpressions() []sqlparser.Expr { - if r.Selected == nil { - return nil - } - return r.Selected.ValueExprs -} - -func (r *Route) isImpossibleIN(node *sqlparser.ComparisonExpr) bool { - switch nodeR := node.Right.(type) { - case sqlparser.ValTuple: - // WHERE col IN (null) - if len(nodeR) == 1 && sqlparser.IsNull(nodeR[0]) { - r.setSelectNoneOpcode() - return true - } - } - return false -} - -func (r *Route) planInOp(ctx *plancontext.PlanningContext, cmp *sqlparser.ComparisonExpr) bool { - switch left := cmp.Left.(type) { - case *sqlparser.ColName: - vdValue := cmp.Right - - valTuple, isTuple := vdValue.(sqlparser.ValTuple) - if isTuple && len(valTuple) == 1 { - return r.planEqualOp(ctx, &sqlparser.ComparisonExpr{Left: left, Right: valTuple[0], Operator: sqlparser.EqualOp}) - } - - value := r.makeEvalEngineExpr(ctx, vdValue) - if value == nil { - return false - } - opcode := func(*vindexes.ColumnVindex) engine.Opcode { return engine.IN } - return r.haveMatchingVindex(ctx, cmp, vdValue, left, value, opcode, justTheVindex) - case sqlparser.ValTuple: - right, rightIsValTuple := cmp.Right.(sqlparser.ValTuple) - if !rightIsValTuple { - return false - } - return r.planCompositeInOpRecursive(ctx, cmp, left, right, nil) - } - - return false -} - -func (r *Route) isImpossibleNotIN(node *sqlparser.ComparisonExpr) bool { - switch node := node.Right.(type) { - case sqlparser.ValTuple: - for _, n := range node { - if sqlparser.IsNull(n) { - r.setSelectNoneOpcode() - return true - } - } - } - - return false -} - -func (r *Route) planLikeOp(ctx *plancontext.PlanningContext, node *sqlparser.ComparisonExpr) bool { - column, ok := node.Left.(*sqlparser.ColName) - if !ok { - return false - } - - vdValue := node.Right - val := r.makeEvalEngineExpr(ctx, vdValue) - if val == nil { - return false - } - selectEqual := func(*vindexes.ColumnVindex) engine.Opcode { return engine.Equal } - vdx := func(vindex *vindexes.ColumnVindex) vindexes.Vindex { - if prefixable, ok := vindex.Vindex.(vindexes.Prefixable); ok { - return prefixable.PrefixVindex() - } - - // if we can't use the vindex as a prefix-vindex, we can't use this vindex at all - return nil - } - return r.haveMatchingVindex(ctx, node, vdValue, column, val, selectEqual, vdx) - -} - -func (r *Route) planCompositeInOpRecursive( - ctx *plancontext.PlanningContext, - cmp *sqlparser.ComparisonExpr, - left, right sqlparser.ValTuple, - coordinates []int, -) bool { - foundVindex := false - cindex := len(coordinates) - coordinates = append(coordinates, 0) - for i, expr := range left { - coordinates[cindex] = i - switch expr := expr.(type) { - case sqlparser.ValTuple: - ok := r.planCompositeInOpRecursive(ctx, cmp, expr, right, coordinates) - return ok || foundVindex - case *sqlparser.ColName: - // check if left col is a vindex - if !r.hasVindex(expr) { - continue - } - - rightVals := make(sqlparser.ValTuple, len(right)) - for j, currRight := range right { - switch currRight := currRight.(type) { - case sqlparser.ValTuple: - val := tupleAccess(currRight, coordinates) - if val == nil { - return false - } - rightVals[j] = val - default: - return false - } - } - newPlanValues := r.makeEvalEngineExpr(ctx, rightVals) - if newPlanValues == nil { - return false - } - - opcode := func(*vindexes.ColumnVindex) engine.Opcode { return engine.MultiEqual } - newVindex := r.haveMatchingVindex(ctx, cmp, rightVals, expr, newPlanValues, opcode, justTheVindex) - foundVindex = newVindex || foundVindex - } - } - return foundVindex -} - -// Reset all vindex predicates on this route and re-build their options from -// the list of seen routing predicates. -func (r *Route) resetRoutingSelections(ctx *plancontext.PlanningContext) error { - switch r.RouteOpCode { - case engine.DBA, engine.Next, engine.Reference, engine.Unsharded: - // these we keep as is - default: - r.RouteOpCode = engine.Scatter - } - - r.Selected = nil - for i, vp := range r.VindexPreds { - r.VindexPreds[i] = &VindexPlusPredicates{ColVindex: vp.ColVindex, TableID: vp.TableID} - } - - for _, predicate := range r.SeenPredicates { - err := r.tryImprovingVindex(ctx, predicate) - if err != nil { - return err - } - } - return nil -} - func tupleAccess(expr sqlparser.Expr, coordinates []int) sqlparser.Expr { tuple, _ := expr.(sqlparser.ValTuple) for _, idx := range coordinates { @@ -699,30 +334,6 @@ func (vpp *VindexPlusPredicates) bestOption() *VindexOption { return best } -func (r *Route) planIsExpr(ctx *plancontext.PlanningContext, node *sqlparser.IsExpr) bool { - // we only handle IS NULL correct. IsExpr can contain other expressions as well - if node.Right != sqlparser.IsNullOp { - return false - } - column, ok := node.Left.(*sqlparser.ColName) - if !ok { - return false - } - vdValue := &sqlparser.NullVal{} - val := r.makeEvalEngineExpr(ctx, vdValue) - if val == nil { - return false - } - opcodeF := func(vindex *vindexes.ColumnVindex) engine.Opcode { - if _, ok := vindex.Vindex.(vindexes.Lookup); ok { - return engine.Scatter - } - return equalOrEqualUnique(vindex) - } - - return r.haveMatchingVindex(ctx, node, vdValue, column, val, opcodeF, justTheVindex) -} - // createRoute returns either an information_schema route, or else consults the // VSchema to find a suitable table, and then creates a route from that. func createRoute( @@ -791,97 +402,56 @@ func createRouteFromVSchemaTable( QTable: queryTable, VTable: vschemaTable, }, - Keyspace: vschemaTable.Keyspace, - } - - for _, columnVindex := range vschemaTable.ColumnVindexes { - plan.VindexPreds = append(plan.VindexPreds, &VindexPlusPredicates{ColVindex: columnVindex, TableID: solves}) } - switch { - case vschemaTable.Type == vindexes.TypeSequence: - plan.RouteOpCode = engine.Next - case vschemaTable.Type == vindexes.TypeReference: - plan.RouteOpCode = engine.Reference - case !vschemaTable.Keyspace.Sharded: - plan.RouteOpCode = engine.Unsharded - case vschemaTable.Pinned != nil: - // Pinned tables have their keyspace ids already assigned. - // Use the Binary vindex, which is the identity function - // for keyspace id. - plan.RouteOpCode = engine.EqualUnique - vindex, _ := vindexes.NewBinary("binary", nil) - plan.Selected = &VindexOption{ - Ready: true, - Values: []evalengine.Expr{evalengine.NewLiteralString(vschemaTable.Pinned, collations.TypedCollation{})}, - ValueExprs: nil, - Predicates: nil, - OpCode: engine.EqualUnique, - FoundVindex: vindex, - Cost: Cost{ - OpCode: engine.EqualUnique, - }, - } - default: - plan.RouteOpCode = engine.Scatter - } + // We create the appropiate Routing struct here, depending on the type of table we are dealing with. + routing := createRoutingForVTable(vschemaTable, solves) for _, predicate := range queryTable.Predicates { - err := plan.UpdateRoutingLogic(ctx, predicate) + var err error + routing, err = UpdateRoutingLogic(ctx, predicate, routing) if err != nil { return nil, err } } - if plan.RouteOpCode == engine.Scatter && len(queryTable.Predicates) > 0 { - // If we have a scatter query, it's worth spending a little extra time seeing if we can't improve it - oldPredicates := queryTable.Predicates - queryTable.Predicates = nil - plan.SeenPredicates = nil - for _, pred := range oldPredicates { - rewritten := sqlparser.RewritePredicate(pred) - predicates := sqlparser.SplitAndExpression(nil, rewritten.(sqlparser.Expr)) - for _, predicate := range predicates { - queryTable.Predicates = append(queryTable.Predicates, predicate) - err := plan.UpdateRoutingLogic(ctx, predicate) - if err != nil { - return nil, err - } - } - } + plan.Routing = routing - if plan.RouteOpCode == engine.Scatter { - // if we _still_ haven't found a better route, we can run this additional rewrite on any ORs we have - for _, expr := range queryTable.Predicates { - or, ok := expr.(*sqlparser.OrExpr) - if !ok { - continue - } - for _, predicate := range sqlparser.ExtractINFromOR(or) { - err := plan.UpdateRoutingLogic(ctx, predicate) - if err != nil { - return nil, err - } - } + switch routing := routing.(type) { + case *ShardedRouting: + if routing.isScatter() && len(queryTable.Predicates) > 0 { + var err error + // If we have a scatter query, it's worth spending a little extra time seeing if we can't improve it + plan.Routing, err = routing.tryImprove(ctx, queryTable) + if err != nil { + return nil, err } } - } - - if planAlternates { - alternates, err := createAlternateRoutesFromVSchemaTable( - ctx, - queryTable, - vschemaTable, - solves, - ) - if err != nil { - return nil, err + case *AnyShardRouting: + if planAlternates { + alternates, err := createAlternateRoutesFromVSchemaTable(ctx, queryTable, vschemaTable, solves) + if err != nil { + return nil, err + } + routing.Alternates = alternates } - plan.Alternates = alternates } return plan, nil } +func createRoutingForVTable(vschemaTable *vindexes.Table, id semantics.TableSet) Routing { + switch { + case vschemaTable.Type == vindexes.TypeSequence: + return &SequenceRouting{keyspace: vschemaTable.Keyspace} + case vschemaTable.Type == vindexes.TypeReference && vschemaTable.Name.String() == "dual": + return &DualRouting{} + case vschemaTable.Type == vindexes.TypeReference || !vschemaTable.Keyspace.Sharded: + return &AnyShardRouting{keyspace: vschemaTable.Keyspace} + default: + return newShardedRouting(vschemaTable, id) + } +} + func createAlternateRoutesFromVSchemaTable( ctx *plancontext.PlanningContext, queryTable *QueryTable, @@ -906,7 +476,7 @@ func createAlternateRoutesFromVSchemaTable( if err != nil { return nil, err } - routes[route.Keyspace] = route + routes[referenceTable.Keyspace] = route } if vschemaTable.Source != nil { @@ -920,7 +490,10 @@ func createAlternateRoutesFromVSchemaTable( if err != nil { return nil, err } - routes[route.Keyspace] = route + keyspace := route.Routing.Keyspace() + if keyspace != nil { + routes[keyspace] = route + } } } @@ -928,10 +501,14 @@ func createAlternateRoutesFromVSchemaTable( } func (r *Route) AddPredicate(ctx *plancontext.PlanningContext, expr sqlparser.Expr) (ops.Operator, error) { - err := r.UpdateRoutingLogic(ctx, expr) + // first we see if the predicate changes how we route + newRouting, err := UpdateRoutingLogic(ctx, expr, r.Routing) if err != nil { return nil, err } + r.Routing = newRouting + + // we also need to push the predicate down into the query newSrc, err := r.Source.AddPredicate(ctx, expr) if err != nil { return nil, err @@ -944,18 +521,6 @@ func (r *Route) AddColumn(ctx *plancontext.PlanningContext, e sqlparser.Expr) (i return r.Source.AddColumn(ctx, e) } -func (r *Route) AlternateInKeyspace(keyspace *vindexes.Keyspace) *Route { - if keyspace.Name == r.Keyspace.Name { - return nil - } - - if route, ok := r.Alternates[keyspace]; ok { - return route - } - - return nil -} - // TablesUsed returns tables used by MergedWith routes, which are not included // in Inputs() and thus not a part of the operator tree func (r *Route) TablesUsed() []string { diff --git a/go/vt/vtgate/planbuilder/operators/route_planning.go b/go/vt/vtgate/planbuilder/operators/route_planning.go index 770066dd552..7ba3c402f8b 100644 --- a/go/vt/vtgate/planbuilder/operators/route_planning.go +++ b/go/vt/vtgate/planbuilder/operators/route_planning.go @@ -29,7 +29,6 @@ import ( "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/engine" - "vitess.io/vitess/go/vt/vtgate/evalengine" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" "vitess.io/vitess/go/vt/vtgate/semantics" "vitess.io/vitess/go/vt/vtgate/vindexes" @@ -102,7 +101,7 @@ func optimizeDerived(ctx *plancontext.PlanningContext, op *Derived) (ops.Operato return op, rewrite.SameTree, nil } - if !(innerRoute.RouteOpCode == engine.EqualUnique) && !op.IsMergeable(ctx) { + if !(innerRoute.Routing.OpCode() == engine.EqualUnique) && !op.IsMergeable(ctx) { // no need to check anything if we are sure that we will only hit a single shard return op, rewrite.SameTree, nil } @@ -140,39 +139,55 @@ func optimizeQueryGraph(ctx *plancontext.PlanningContext, op *QueryGraph) (resul return } -func buildVindexTableForDML(ctx *plancontext.PlanningContext, tableInfo semantics.TableInfo, table *QueryTable, dmlType string) (*vindexes.Table, engine.Opcode, key.Destination, error) { +func buildVindexTableForDML( + ctx *plancontext.PlanningContext, + tableInfo semantics.TableInfo, + table *QueryTable, + dmlType string, +) (*vindexes.Table, Routing, error) { vindexTable := tableInfo.GetVindexTable() - opCode := engine.Unsharded - if vindexTable.Keyspace.Sharded { - opCode = engine.Scatter - } - if vindexTable.Source != nil { sourceTable, _, _, _, _, err := ctx.VSchema.FindTableOrVindex(vindexTable.Source.TableName) if err != nil { - return nil, 0, nil, err + return nil, nil, err } vindexTable = sourceTable } + if !vindexTable.Keyspace.Sharded { + return vindexTable, &AnyShardRouting{keyspace: vindexTable.Keyspace}, nil + } + var dest key.Destination var typ topodatapb.TabletType var err error tblName, ok := table.Alias.Expr.(sqlparser.TableName) - if ok { - _, _, _, typ, dest, err = ctx.VSchema.FindTableOrVindex(tblName) - if err != nil { - return nil, 0, nil, err - } - if dest != nil { - if typ != topodatapb.TabletType_PRIMARY { - return nil, 0, nil, vterrors.VT09002(dmlType) - } - // we are dealing with an explicitly targeted UPDATE - opCode = engine.ByDestination + if !ok { + return nil, nil, vterrors.VT12001("multi shard UPDATE with LIMIT") + } + + _, _, _, typ, dest, err = ctx.VSchema.FindTableOrVindex(tblName) + if err != nil { + return nil, nil, err + } + if dest == nil { + routing := &ShardedRouting{ + keyspace: vindexTable.Keyspace, + RouteOpCode: engine.Scatter, } + return vindexTable, routing, nil } - return vindexTable, opCode, dest, nil + + if typ != topodatapb.TabletType_PRIMARY { + return nil, nil, vterrors.VT09002(dmlType) + } + + // we are dealing with an explicitly targeted UPDATE + routing := &TargetedRouting{ + keyspace: vindexTable.Keyspace, + TargetDestination: dest, + } + return vindexTable, routing, nil } func generateOwnedVindexQuery(tblExpr sqlparser.TableExpr, del *sqlparser.Delete, table *vindexes.Table, ksidCols []sqlparser.IdentifierCI) string { @@ -284,37 +299,24 @@ func createInfSchemaRoute(ctx *plancontext.PlanningContext, table *QueryTable) ( if err != nil { return nil, err } - r := &Route{ - RouteOpCode: engine.DBA, - Source: &Table{ - QTable: table, - VTable: &vindexes.Table{ - Name: table.Table.Name, - Keyspace: ks, - }, + var src ops.Operator = &Table{ + QTable: table, + VTable: &vindexes.Table{ + Name: table.Table.Name, + Keyspace: ks, }, - Keyspace: ks, } + var routing Routing = &InfoSchemaRouting{} for _, pred := range table.Predicates { - isTableSchema, bvName, out, err := extractInfoSchemaRoutingPredicate(pred, ctx.ReservedVars) + routing, err = UpdateRoutingLogic(ctx, pred, routing) if err != nil { return nil, err } - if out == nil { - // we didn't find a predicate to use for routing, continue to look for next predicate - continue - } - - if isTableSchema { - r.SysTableTableSchema = append(r.SysTableTableSchema, out) - } else { - if r.SysTableTableName == nil { - r.SysTableTableName = map[string]evalengine.Expr{} - } - r.SysTableTableName[bvName] = out - } } - return r, nil + return &Route{ + Source: src, + Routing: routing, + }, nil } func mergeRoutes(ctx *plancontext.PlanningContext, qg *QueryGraph, physicalOps []ops.Operator, planCache opCacheMap, crossJoinsOK bool) (ops.Operator, error) { @@ -423,11 +425,10 @@ func requiresSwitchingSides(ctx *plancontext.PlanningContext, op ops.Operator) b } func mergeOrJoin(ctx *plancontext.PlanningContext, lhs, rhs ops.Operator, joinPredicates []sqlparser.Expr, inner bool) (ops.Operator, error) { - merger := func(a, b *Route) (*Route, error) { - return createRouteOperatorForJoin(ctx, a, b, joinPredicates, inner) + newPlan, err := Merge(ctx, lhs, rhs, joinPredicates, newJoinMerge(ctx, joinPredicates, inner)) + if err != nil { + return nil, err } - - newPlan, _ := tryMerge(ctx, lhs, rhs, joinPredicates, merger) if newPlan != nil { return newPlan, nil } @@ -449,38 +450,6 @@ func mergeOrJoin(ctx *plancontext.PlanningContext, lhs, rhs ops.Operator, joinPr return pushJoinPredicates(ctx, joinPredicates, join) } -func createRouteOperatorForJoin(ctx *plancontext.PlanningContext, aRoute, bRoute *Route, joinPredicates []sqlparser.Expr, inner bool) (*Route, error) { - // append system table names from both the routes. - sysTableName := aRoute.SysTableTableName - if sysTableName == nil { - sysTableName = bRoute.SysTableTableName - } else { - for k, v := range bRoute.SysTableTableName { - sysTableName[k] = v - } - } - - join := NewApplyJoin(aRoute.Source, bRoute.Source, ctx.SemTable.AndExpressions(joinPredicates...), !inner) - r := &Route{ - RouteOpCode: aRoute.RouteOpCode, - Keyspace: aRoute.Keyspace, - VindexPreds: append(aRoute.VindexPreds, bRoute.VindexPreds...), - SysTableTableSchema: append(aRoute.SysTableTableSchema, bRoute.SysTableTableSchema...), - SeenPredicates: append(aRoute.SeenPredicates, bRoute.SeenPredicates...), - SysTableTableName: sysTableName, - Source: join, - MergedWith: []*Route{bRoute}, - } - - if aRoute.SelectedVindex() == bRoute.SelectedVindex() { - r.Selected = aRoute.Selected - } - - return r, nil -} - -type mergeFunc func(a, b *Route) (*Route, error) - func operatorsToRoutes(a, b ops.Operator) (*Route, *Route) { aRoute, ok := a.(*Route) if !ok { @@ -493,149 +462,6 @@ func operatorsToRoutes(a, b ops.Operator) (*Route, *Route) { return aRoute, bRoute } -func tryMerge( - ctx *plancontext.PlanningContext, - a, b ops.Operator, - joinPredicates []sqlparser.Expr, - merger mergeFunc, -) (ops.Operator, error) { - aRoute, bRoute := operatorsToRoutes(Clone(a), Clone(b)) - if aRoute == nil || bRoute == nil { - return nil, nil - } - - sameKeyspace := aRoute.Keyspace == bRoute.Keyspace - - if !sameKeyspace { - if altARoute := aRoute.AlternateInKeyspace(bRoute.Keyspace); altARoute != nil { - aRoute = altARoute - sameKeyspace = true - } else if altBRoute := bRoute.AlternateInKeyspace(aRoute.Keyspace); altBRoute != nil { - bRoute = altBRoute - sameKeyspace = true - } - } - - if sameKeyspace || (isDualTable(aRoute) || isDualTable(bRoute)) { - tree, err := tryMergeReferenceTable(aRoute, bRoute, merger) - if tree != nil || err != nil { - return tree, err - } - } - - switch aRoute.RouteOpCode { - case engine.Unsharded, engine.DBA: - if aRoute.RouteOpCode == bRoute.RouteOpCode && sameKeyspace { - return merger(aRoute, bRoute) - } - case engine.EqualUnique: - // If the two routes fully match, they can be merged together. - if bRoute.RouteOpCode == engine.EqualUnique { - aVdx := aRoute.SelectedVindex() - bVdx := bRoute.SelectedVindex() - aExpr := aRoute.VindexExpressions() - bExpr := bRoute.VindexExpressions() - if aVdx == bVdx && gen4ValuesEqual(ctx, aExpr, bExpr) { - return merger(aRoute, bRoute) - } - } - - // If the two routes don't match, fall through to the next case and see if we - // can merge via join predicates instead. - fallthrough - - case engine.Scatter, engine.IN, engine.None: - if len(joinPredicates) == 0 { - // If we are doing two Scatters, we have to make sure that the - // joins are on the correct vindex to allow them to be merged - // no join predicates - no vindex - return nil, nil - } - - if !sameKeyspace { - return nil, nil - } - - canMerge := canMergeOnFilters(ctx, aRoute, bRoute, joinPredicates) - if !canMerge { - return nil, nil - } - r, err := merger(aRoute, bRoute) - if err != nil { - return nil, err - } - - // If we have a `None` route opcode, we want to keep it - - // we only try to find a better Vindex for other route opcodes - if aRoute.RouteOpCode != engine.None { - r.PickBestAvailableVindex() - } - - return r, nil - } - return nil, nil -} - -func isDualTable(route *Route) bool { - sources := leaves(route) - if len(sources) > 1 { - return false - } - src, ok := sources[0].(*Table) - if !ok { - return false - } - return src.VTable.Name.String() == "dual" && src.QTable.Table.Qualifier.IsEmpty() -} - -func leaves(op ops.Operator) (sources []ops.Operator) { - switch op := op.(type) { - // these are the leaves - case *Table: - return []ops.Operator{op} - // physical - case *ApplyJoin: - return []ops.Operator{op.LHS, op.RHS} - case *Filter: - return []ops.Operator{op.Source} - case *Route: - return []ops.Operator{op.Source} - } - - panic(fmt.Sprintf("leaves unknown type: %T", op)) -} - -func tryMergeReferenceTable(aRoute, bRoute *Route, merger mergeFunc) (*Route, error) { - var ( - // if either side is a reference table, we can just merge it and use the opcode of the other side - opCode engine.Opcode - vindex *VindexOption - ks *vindexes.Keyspace - ) - - switch { - case aRoute.RouteOpCode == engine.Reference: - vindex = bRoute.Selected - opCode = bRoute.RouteOpCode - ks = bRoute.Keyspace - case bRoute.RouteOpCode == engine.Reference: - vindex = aRoute.Selected - opCode = aRoute.RouteOpCode - ks = aRoute.Keyspace - default: - return nil, nil - } - - r, err := merger(aRoute, bRoute) - if err != nil { - return nil, err - } - r.RouteOpCode = opCode - r.Selected = vindex - r.Keyspace = ks - return r, nil -} - func canMergeOnFilter(ctx *plancontext.PlanningContext, a, b *Route, predicate sqlparser.Expr) bool { comparison, ok := predicate.(*sqlparser.ComparisonExpr) if !ok { diff --git a/go/vt/vtgate/planbuilder/operators/sharded_routing.go b/go/vt/vtgate/planbuilder/operators/sharded_routing.go new file mode 100644 index 00000000000..be23698975e --- /dev/null +++ b/go/vt/vtgate/planbuilder/operators/sharded_routing.go @@ -0,0 +1,617 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operators + +import ( + "golang.org/x/exp/slices" + + "vitess.io/vitess/go/vt/vtgate/planbuilder/operators/ops" + + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/vt/sqlparser" + "vitess.io/vitess/go/vt/vterrors" + "vitess.io/vitess/go/vt/vtgate/engine" + "vitess.io/vitess/go/vt/vtgate/evalengine" + "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" + "vitess.io/vitess/go/vt/vtgate/semantics" + "vitess.io/vitess/go/vt/vtgate/vindexes" +) + +// ShardedRouting is what we use for all tables that exist in a sharded keyspace +// It knows about available vindexes and can use them for routing when applicable +type ShardedRouting struct { + // here we store the possible vindexes we can use so that when we add predicates to the plan, + // we can quickly check if the new predicates enables any new vindex Options + VindexPreds []*VindexPlusPredicates + + // the best option available is stored here + Selected *VindexOption + + keyspace *vindexes.Keyspace + + RouteOpCode engine.Opcode + + // SeenPredicates contains all the predicates that have had a chance to influence routing. + // If we need to replan routing, we'll use this list + SeenPredicates []sqlparser.Expr +} + +var _ Routing = (*ShardedRouting)(nil) + +func newShardedRouting(vtable *vindexes.Table, id semantics.TableSet) Routing { + routing := &ShardedRouting{ + RouteOpCode: engine.Scatter, + keyspace: vtable.Keyspace, + } + + if vtable.Pinned != nil { + // Pinned tables have their keyspace ids already assigned. + // Use the Binary vindex, which is the identity function + // for keyspace id. + routing.RouteOpCode = engine.EqualUnique + vindex, _ := vindexes.NewBinary("binary", nil) + routing.Selected = &VindexOption{ + Ready: true, + Values: []evalengine.Expr{evalengine.NewLiteralString(vtable.Pinned, collations.TypedCollation{})}, + ValueExprs: nil, + Predicates: nil, + OpCode: engine.EqualUnique, + FoundVindex: vindex, + Cost: Cost{ + OpCode: engine.EqualUnique, + }, + } + + } + for _, columnVindex := range vtable.ColumnVindexes { + routing.VindexPreds = append(routing.VindexPreds, &VindexPlusPredicates{ColVindex: columnVindex, TableID: id}) + } + return routing +} + +func (tr *ShardedRouting) isScatter() bool { + return tr.RouteOpCode == engine.Scatter +} + +// tryImprove rewrites the predicates for this query to see if we can produce a better plan. +// The rewrites are two: +// 1. first we turn the predicate a conjunctive normal form - an AND of ORs. +// This can sometimes push a predicate to the top so it's not hiding inside of an OR +// 2. If that is not enough, an additional rewrite pass is performed where we try to +// turn ORs into IN, which is easier for the planner to plan +func (tr *ShardedRouting) tryImprove(ctx *plancontext.PlanningContext, queryTable *QueryTable) (Routing, error) { + oldPredicates := queryTable.Predicates + queryTable.Predicates = nil + tr.SeenPredicates = nil + var routing Routing = tr + var err error + for _, pred := range oldPredicates { + rewritten := sqlparser.RewritePredicate(pred) + predicates := sqlparser.SplitAndExpression(nil, rewritten.(sqlparser.Expr)) + for _, predicate := range predicates { + queryTable.Predicates = append(queryTable.Predicates, predicate) + + routing, err = UpdateRoutingLogic(ctx, predicate, routing) + if err != nil { + return nil, err + } + } + } + + // If we have something other than a sharded routing with scatter, we are done + if sr, ok := routing.(*ShardedRouting); !ok || !sr.isScatter() { + return routing, nil + } + + // if we _still_ haven't found a better route, we can run this additional rewrite on any ORs we have + for _, expr := range queryTable.Predicates { + or, ok := expr.(*sqlparser.OrExpr) + if !ok { + continue + } + for _, predicate := range sqlparser.ExtractINFromOR(or) { + routing, err = UpdateRoutingLogic(ctx, predicate, routing) + if err != nil { + return nil, err + } + } + } + + return routing, nil +} + +func (tr *ShardedRouting) UpdateRoutingParams(_ *plancontext.PlanningContext, rp *engine.RoutingParameters) error { + rp.Keyspace = tr.keyspace + if tr.Selected != nil { + rp.Vindex = tr.Selected.FoundVindex + rp.Values = tr.Selected.Values + } + return nil +} + +func (tr *ShardedRouting) Clone() Routing { + var selected *VindexOption + if tr.Selected != nil { + t := *tr.Selected + selected = &t + } + return &ShardedRouting{ + VindexPreds: slices.Clone(tr.VindexPreds), + Selected: selected, + keyspace: tr.keyspace, + RouteOpCode: tr.RouteOpCode, + SeenPredicates: slices.Clone(tr.SeenPredicates), + } +} + +func (tr *ShardedRouting) updateRoutingLogic(ctx *plancontext.PlanningContext, expr sqlparser.Expr) (Routing, error) { + tr.SeenPredicates = append(tr.SeenPredicates, expr) + + newRouting, newVindexFound, err := tr.searchForNewVindexes(ctx, expr) + if err != nil { + return nil, err + } + + if newRouting != nil { + // we found something that we can route with something other than ShardedRouting + return newRouting, nil + } + + // if we didn't open up any new vindex Options, no need to enter here + if newVindexFound { + tr.PickBestAvailableVindex() + } + + return tr, nil +} + +func (tr *ShardedRouting) ResetRoutingLogic(ctx *plancontext.PlanningContext) (Routing, error) { + tr.RouteOpCode = engine.Scatter + tr.Selected = nil + for i, vp := range tr.VindexPreds { + tr.VindexPreds[i] = &VindexPlusPredicates{ColVindex: vp.ColVindex, TableID: vp.TableID} + } + + var routing Routing = tr + for _, predicate := range tr.SeenPredicates { + var err error + routing, err = UpdateRoutingLogic(ctx, predicate, routing) + if err != nil { + return nil, err + } + } + return routing, nil +} + +func (tr *ShardedRouting) searchForNewVindexes(ctx *plancontext.PlanningContext, predicate sqlparser.Expr) (Routing, bool, error) { + newVindexFound := false + switch node := predicate.(type) { + case *sqlparser.ExtractedSubquery: + originalCmp, ok := node.Original.(*sqlparser.ComparisonExpr) + if !ok { + break + } + + // using the node.subquery which is the rewritten version of our subquery + cmp := &sqlparser.ComparisonExpr{ + Left: node.OtherSide, + Right: &sqlparser.Subquery{Select: node.Subquery.Select}, + Operator: originalCmp.Operator, + } + return tr.planComparison(ctx, cmp) + case *sqlparser.ComparisonExpr: + return tr.planComparison(ctx, node) + + case *sqlparser.IsExpr: + found := tr.planIsExpr(ctx, node) + newVindexFound = newVindexFound || found + } + + return nil, newVindexFound, nil +} + +func (tr *ShardedRouting) planComparison(ctx *plancontext.PlanningContext, cmp *sqlparser.ComparisonExpr) (routing Routing, foundNew bool, err error) { + switch cmp.Operator { + case sqlparser.EqualOp: + found := tr.planEqualOp(ctx, cmp) + return nil, found, nil + case sqlparser.InOp: + found := tr.planInOp(ctx, cmp) + return nil, found, nil + case sqlparser.LikeOp: + found := tr.planLikeOp(ctx, cmp) + return nil, found, nil + + } + return nil, false, nil +} + +func (tr *ShardedRouting) planIsExpr(ctx *plancontext.PlanningContext, node *sqlparser.IsExpr) bool { + // we only handle IS NULL correct. IsExpr can contain other expressions as well + if node.Right != sqlparser.IsNullOp { + return false + } + column, ok := node.Left.(*sqlparser.ColName) + if !ok { + return false + } + vdValue := &sqlparser.NullVal{} + val := makeEvalEngineExpr(ctx, vdValue) + if val == nil { + return false + } + opcodeF := func(vindex *vindexes.ColumnVindex) engine.Opcode { + if _, ok := vindex.Vindex.(vindexes.Lookup); ok { + return engine.Scatter + } + return equalOrEqualUnique(vindex) + } + + return tr.haveMatchingVindex(ctx, node, vdValue, column, val, opcodeF, justTheVindex) +} + +func (tr *ShardedRouting) planInOp(ctx *plancontext.PlanningContext, cmp *sqlparser.ComparisonExpr) bool { + switch left := cmp.Left.(type) { + case *sqlparser.ColName: + vdValue := cmp.Right + + valTuple, isTuple := vdValue.(sqlparser.ValTuple) + if isTuple && len(valTuple) == 1 { + return tr.planEqualOp(ctx, &sqlparser.ComparisonExpr{Left: left, Right: valTuple[0], Operator: sqlparser.EqualOp}) + } + + value := makeEvalEngineExpr(ctx, vdValue) + if value == nil { + return false + } + opcode := func(*vindexes.ColumnVindex) engine.Opcode { return engine.IN } + return tr.haveMatchingVindex(ctx, cmp, vdValue, left, value, opcode, justTheVindex) + case sqlparser.ValTuple: + right, rightIsValTuple := cmp.Right.(sqlparser.ValTuple) + if !rightIsValTuple { + return false + } + return tr.planCompositeInOpRecursive(ctx, cmp, left, right, nil) + } + + return false +} + +func (tr *ShardedRouting) planLikeOp(ctx *plancontext.PlanningContext, node *sqlparser.ComparisonExpr) bool { + column, ok := node.Left.(*sqlparser.ColName) + if !ok { + return false + } + + vdValue := node.Right + val := makeEvalEngineExpr(ctx, vdValue) + if val == nil { + return false + } + selectEqual := func(*vindexes.ColumnVindex) engine.Opcode { return engine.Equal } + vdx := func(vindex *vindexes.ColumnVindex) vindexes.Vindex { + if prefixable, ok := vindex.Vindex.(vindexes.Prefixable); ok { + return prefixable.PrefixVindex() + } + + // if we can't use the vindex as a prefix-vindex, we can't use this vindex at all + return nil + } + return tr.haveMatchingVindex(ctx, node, vdValue, column, val, selectEqual, vdx) +} + +func (tr *ShardedRouting) Cost() int { + switch tr.RouteOpCode { + case engine.EqualUnique: + return 1 + case engine.Equal: + return 5 + case engine.IN: + return 10 + case engine.MultiEqual: + return 10 + case engine.Scatter: + return 20 + default: + panic("this switch should be exhaustive") + } +} + +func (tr *ShardedRouting) OpCode() engine.Opcode { + return tr.RouteOpCode +} + +func (tr *ShardedRouting) Keyspace() *vindexes.Keyspace { + return tr.keyspace +} + +// PickBestAvailableVindex goes over the available vindexes for this route and picks the best one available. +func (tr *ShardedRouting) PickBestAvailableVindex() { + for _, v := range tr.VindexPreds { + option := v.bestOption() + if option != nil && (tr.Selected == nil || less(option.Cost, tr.Selected.Cost)) { + tr.Selected = option + tr.RouteOpCode = option.OpCode + } + } +} + +func (tr *ShardedRouting) haveMatchingVindex( + ctx *plancontext.PlanningContext, + node sqlparser.Expr, + valueExpr sqlparser.Expr, + column *sqlparser.ColName, + value evalengine.Expr, + opcode func(*vindexes.ColumnVindex) engine.Opcode, + vfunc func(*vindexes.ColumnVindex) vindexes.Vindex, +) bool { + newVindexFound := false + for _, v := range tr.VindexPreds { + // check that the + if !ctx.SemTable.DirectDeps(column).IsSolvedBy(v.TableID) { + continue + } + switch v.ColVindex.Vindex.(type) { + case vindexes.SingleColumn: + col := v.ColVindex.Columns[0] + if column.Name.Equal(col) { + // single column vindex - just add the option + routeOpcode := opcode(v.ColVindex) + vindex := vfunc(v.ColVindex) + if vindex == nil || routeOpcode == engine.Scatter { + continue + } + v.Options = append(v.Options, &VindexOption{ + Values: []evalengine.Expr{value}, + ValueExprs: []sqlparser.Expr{valueExpr}, + Predicates: []sqlparser.Expr{node}, + OpCode: routeOpcode, + FoundVindex: vindex, + Cost: costFor(v.ColVindex, routeOpcode), + Ready: true, + }) + newVindexFound = true + } + case vindexes.MultiColumn: + colLoweredName := "" + indexOfCol := -1 + for idx, col := range v.ColVindex.Columns { + if column.Name.Equal(col) { + colLoweredName = column.Name.Lowered() + indexOfCol = idx + break + } + } + if colLoweredName == "" { + break + } + + var newOption []*VindexOption + for _, op := range v.Options { + if op.Ready { + continue + } + _, isPresent := op.ColsSeen[colLoweredName] + if isPresent { + continue + } + option := copyOption(op) + optionReady := option.updateWithNewColumn(colLoweredName, valueExpr, indexOfCol, value, node, v.ColVindex, opcode) + if optionReady { + newVindexFound = true + } + newOption = append(newOption, option) + } + v.Options = append(v.Options, newOption...) + + // multi-column vindex - just always add as new option + option := createOption(v.ColVindex, vfunc) + optionReady := option.updateWithNewColumn(colLoweredName, valueExpr, indexOfCol, value, node, v.ColVindex, opcode) + if optionReady { + newVindexFound = true + } + v.Options = append(v.Options, option) + } + } + return newVindexFound +} + +func (tr *ShardedRouting) planEqualOp(ctx *plancontext.PlanningContext, node *sqlparser.ComparisonExpr) bool { + column, ok := node.Left.(*sqlparser.ColName) + other := node.Right + vdValue := other + if !ok { + column, ok = node.Right.(*sqlparser.ColName) + if !ok { + // either the LHS or RHS have to be a column to be useful for the vindex + return false + } + vdValue = node.Left + } + val := makeEvalEngineExpr(ctx, vdValue) + if val == nil { + return false + } + + return tr.haveMatchingVindex(ctx, node, vdValue, column, val, equalOrEqualUnique, justTheVindex) +} + +func (tr *ShardedRouting) planCompositeInOpRecursive( + ctx *plancontext.PlanningContext, + cmp *sqlparser.ComparisonExpr, + left, right sqlparser.ValTuple, + coordinates []int, +) bool { + foundVindex := false + cindex := len(coordinates) + coordinates = append(coordinates, 0) + for i, expr := range left { + coordinates[cindex] = i + switch expr := expr.(type) { + case sqlparser.ValTuple: + ok := tr.planCompositeInOpRecursive(ctx, cmp, expr, right, coordinates) + return ok || foundVindex + case *sqlparser.ColName: + // check if left col is a vindex + if !tr.hasVindex(expr) { + continue + } + + rightVals := make(sqlparser.ValTuple, len(right)) + for j, currRight := range right { + switch currRight := currRight.(type) { + case sqlparser.ValTuple: + val := tupleAccess(currRight, coordinates) + if val == nil { + return false + } + rightVals[j] = val + default: + return false + } + } + newPlanValues := makeEvalEngineExpr(ctx, rightVals) + if newPlanValues == nil { + return false + } + + opcode := func(*vindexes.ColumnVindex) engine.Opcode { return engine.MultiEqual } + newVindex := tr.haveMatchingVindex(ctx, cmp, rightVals, expr, newPlanValues, opcode, justTheVindex) + foundVindex = newVindex || foundVindex + } + } + return foundVindex +} + +func (tr *ShardedRouting) hasVindex(column *sqlparser.ColName) bool { + for _, v := range tr.VindexPreds { + for _, col := range v.ColVindex.Columns { + if column.Name.Equal(col) { + return true + } + } + } + return false +} + +// Reset all vindex predicates on this route and re-build their options from +// the list of seen routing predicates. +func (tr *ShardedRouting) resetRoutingSelections(ctx *plancontext.PlanningContext) error { + tr.RouteOpCode = engine.Scatter + tr.Selected = nil + for i, vp := range tr.VindexPreds { + tr.VindexPreds[i] = &VindexPlusPredicates{ColVindex: vp.ColVindex, TableID: vp.TableID} + } + + var routing Routing = tr + for _, predicate := range tr.SeenPredicates { + var err error + routing, err = UpdateRoutingLogic(ctx, predicate, routing) + if err != nil { + return err + } + } + if routing != tr { + return vterrors.VT13001("uh-oh. we ended up with a different type of routing") + } + return nil +} + +func (tr *ShardedRouting) SelectedVindex() vindexes.Vindex { + if tr.Selected == nil { + return nil + } + return tr.Selected.FoundVindex +} + +func (tr *ShardedRouting) VindexExpressions() []sqlparser.Expr { + if tr.Selected == nil { + return nil + } + return tr.Selected.ValueExprs +} + +func tryMergeShardedRouting(ctx *plancontext.PlanningContext, routeA *Route, routeB *Route, m merger, joinPredicates []sqlparser.Expr) (ops.Operator, error) { + sameKeyspace := routeA.Routing.Keyspace() == routeB.Routing.Keyspace() + tblA := routeA.Routing.(*ShardedRouting) + tblB := routeB.Routing.(*ShardedRouting) + + switch tblA.RouteOpCode { + case engine.EqualUnique: + // If the two routes fully match, they can be merged together. + if tblB.RouteOpCode == engine.EqualUnique { + aVdx := tblA.SelectedVindex() + bVdx := tblB.SelectedVindex() + aExpr := tblA.VindexExpressions() + bExpr := tblB.VindexExpressions() + if aVdx == bVdx && gen4ValuesEqual(ctx, aExpr, bExpr) { + return m.mergeTables(tblA, tblB, routeA, routeB) + } + } + + // If the two routes don't match, fall through to the next case and see if we + // can merge via join predicates instead. + fallthrough + + case engine.Scatter, engine.IN, engine.None: + if len(joinPredicates) == 0 { + // If we are doing two Scatters, we have to make sure that the + // joins are on the correct vindex to allow them to be merged + // no join predicates - no vindex + return nil, nil + } + + if !sameKeyspace { + return nil, vterrors.VT12001("cross-shard correlated subquery") + } + + canMerge := canMergeOnFilters(ctx, routeA, routeB, joinPredicates) + if !canMerge { + return nil, nil + } + return m.mergeTables(tblA, tblB, routeA, routeB) + } + return nil, nil +} + +// makeEvalEngineExpr transforms the given sqlparser.Expr into an evalengine expression +func makeEvalEngineExpr(ctx *plancontext.PlanningContext, n sqlparser.Expr) evalengine.Expr { + if ctx.IsSubQueryToReplace(n) { + return nil + } + + for _, expr := range ctx.SemTable.GetExprAndEqualities(n) { + if subq, isSubq := expr.(*sqlparser.Subquery); isSubq { + extractedSubquery := ctx.SemTable.FindSubqueryReference(subq) + if extractedSubquery == nil { + continue + } + switch engine.PulloutOpcode(extractedSubquery.OpCode) { + case engine.PulloutIn, engine.PulloutNotIn: + expr = sqlparser.NewListArg(extractedSubquery.GetArgName()) + case engine.PulloutValue, engine.PulloutExists: + expr = sqlparser.NewArgument(extractedSubquery.GetArgName()) + } + } + ee, _ := evalengine.Translate(expr, ctx.SemTable) + if ee != nil { + return ee + } + } + + return nil +} diff --git a/go/vt/vtgate/planbuilder/operators/subquery_planning.go b/go/vt/vtgate/planbuilder/operators/subquery_planning.go index 0b095fe887f..6dd5220e389 100644 --- a/go/vt/vtgate/planbuilder/operators/subquery_planning.go +++ b/go/vt/vtgate/planbuilder/operators/subquery_planning.go @@ -20,7 +20,6 @@ import ( "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/engine" - "vitess.io/vitess/go/vt/vtgate/evalengine" "vitess.io/vitess/go/vt/vtgate/planbuilder/operators/ops" "vitess.io/vitess/go/vt/vtgate/planbuilder/operators/rewrite" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" @@ -37,15 +36,12 @@ func optimizeSubQuery(ctx *plancontext.PlanningContext, op *SubQuery, ts semanti var preds []sqlparser.Expr preds, innerOp = unresolvedAndSource(ctx, innerOp) - merger := func(a, b *Route) (*Route, error) { - return mergeSubQueryOp(ctx, a, b, inner) - } newInner := &SubQueryInner{ Inner: inner.Inner, ExtractedSubquery: inner.ExtractedSubquery, } - merged, err := tryMergeSubQueryOp(ctx, outer, innerOp, newInner, preds, merger, ts) + merged, err := tryMergeSubQueryOp(ctx, outer, innerOp, newInner, preds, newSubQueryMerge(ctx, newInner), ts) if err != nil { return nil, rewrite.SameTree, err } @@ -97,16 +93,27 @@ func unresolvedAndSource(ctx *plancontext.PlanningContext, op ops.Operator) ([]s return preds, op } -func mergeSubQueryOp(ctx *plancontext.PlanningContext, outer *Route, inner *Route, subq *SubQueryInner) (*Route, error) { - subq.ExtractedSubquery.NeedsRewrite = true - outer.SysTableTableSchema = append(outer.SysTableTableSchema, inner.SysTableTableSchema...) - for k, v := range inner.SysTableTableName { - if outer.SysTableTableName == nil { - outer.SysTableTableName = map[string]evalengine.Expr{} - } - outer.SysTableTableName[k] = v +func mergeSubQueryOp(ctx *plancontext.PlanningContext, outer *Route, inner *Route, subq *SubQueryInner, mergedRouting Routing) (*Route, error) { + subq.ExtractedSubquery.Merged = true + + switch outerRouting := outer.Routing.(type) { + case *ShardedRouting: + return mergeSubQueryFromTableRouting(ctx, outer, inner, outerRouting, subq) + default: + outer.Routing = mergedRouting } + outer.MergedWith = append(outer.MergedWith, inner) + + return outer, nil +} + +func mergeSubQueryFromTableRouting( + ctx *plancontext.PlanningContext, + outer, inner *Route, + outerRouting *ShardedRouting, + subq *SubQueryInner, +) (*Route, error) { // When merging an inner query with its outer query, we can remove the // inner query from the list of predicates that can influence routing of // the outer query. @@ -114,9 +121,9 @@ func mergeSubQueryOp(ctx *plancontext.PlanningContext, outer *Route, inner *Rout // Note that not all inner queries necessarily are part of the routing // predicates list, so this might be a no-op. subQueryWasPredicate := false - for i, predicate := range outer.SeenPredicates { + for i, predicate := range outerRouting.SeenPredicates { if ctx.SemTable.EqualsExpr(predicate, subq.ExtractedSubquery) { - outer.SeenPredicates = append(outer.SeenPredicates[:i], outer.SeenPredicates[i+1:]...) + outerRouting.SeenPredicates = append(outerRouting.SeenPredicates[:i], outerRouting.SeenPredicates[i+1:]...) subQueryWasPredicate = true @@ -126,23 +133,22 @@ func mergeSubQueryOp(ctx *plancontext.PlanningContext, outer *Route, inner *Rout } } - err := outer.resetRoutingSelections(ctx) + err := outerRouting.resetRoutingSelections(ctx) if err != nil { return nil, err } if subQueryWasPredicate { - // Copy Vindex predicates from the inner route to the upper route. - // If we can route based on some of these predicates, the routing can improve - outer.VindexPreds = append(outer.VindexPreds, inner.VindexPreds...) + if innerTR, isTR := inner.Routing.(*ShardedRouting); isTR { + // Copy Vindex predicates from the inner route to the upper route. + // If we can route based on some of these predicates, the routing can improve + outerRouting.VindexPreds = append(outerRouting.VindexPreds, innerTR.VindexPreds...) + } - if inner.RouteOpCode == engine.None { - outer.setSelectNoneOpcode() + if inner.Routing.OpCode() == engine.None { + outer.Routing = &NoneRouting{keyspace: outerRouting.keyspace} } } - - outer.MergedWith = append(outer.MergedWith, inner) - return outer, nil } @@ -191,7 +197,7 @@ func tryMergeSubQueryOp( outer, subq ops.Operator, subQueryInner *SubQueryInner, joinPredicates []sqlparser.Expr, - merger mergeFunc, + merger merger, lhs semantics.TableSet, // these are the tables made available because we are on the RHS of a join ) (ops.Operator, error) { switch outerOp := outer.(type) { @@ -216,7 +222,7 @@ func tryMergeSubqueryWithRoute( subq ops.Operator, outerOp *Route, joinPredicates []sqlparser.Expr, - merger mergeFunc, + merger merger, subQueryInner *SubQueryInner, lhs semantics.TableSet, // these are the tables made available because we are on the RHS of a join ) (ops.Operator, error) { @@ -225,7 +231,7 @@ func tryMergeSubqueryWithRoute( return nil, nil } - if outerOp.RouteOpCode == engine.Reference && !subqueryRoute.IsSingleShard() { + if outerOp.Routing.OpCode() == engine.Reference && !subqueryRoute.IsSingleShard() { return nil, nil } @@ -235,7 +241,7 @@ func tryMergeSubqueryWithRoute( return nil, nil } - merged, err := tryMerge(ctx, outerOp, subq, joinPredicates, merger) + merged, err := Merge(ctx, outerOp, subq, joinPredicates, merger) if err != nil { return nil, err } @@ -249,31 +255,23 @@ func tryMergeSubqueryWithRoute( return nil, nil } - // Special case: Inner query won't return any results / is not routable. - if subqueryRoute.RouteOpCode == engine.None { - merged, err := merger(outerOp, subqueryRoute) - if err != nil { - return nil, err - } - return merged, err - } - // Inner subqueries can be merged with the outer subquery as long as // the inner query is a single column selection, and that single column has a matching // vindex on the outer query's operand. if canMergeSubqueryOnColumnSelection(ctx, outerOp, subqueryRoute, subQueryInner.ExtractedSubquery) { - merged, err := merger(outerOp, subqueryRoute) - - if err != nil { - return nil, err + // TODO: clean up. All this casting is not pretty + outerRouting, ok := outerOp.Routing.(*ShardedRouting) + if !ok { + return nil, nil } - - if merged != nil { - // since we inlined the subquery into the outer query, new vindex options might have been enabled, - // so we go over our current options to check if anything better has come up. - merged.PickBestAvailableVindex() - return merged, err + innerRouting := subqueryRoute.Routing.(*ShardedRouting) + if !ok { + return nil, nil } + merged, err := merger.mergeTables(outerRouting, innerRouting, outerOp, subqueryRoute) + mergedRouting := merged.Routing.(*ShardedRouting) + mergedRouting.PickBestAvailableVindex() + return merged, err } return nil, nil } @@ -283,7 +281,7 @@ func tryMergeSubqueryWithJoin( subq ops.Operator, outerOp *ApplyJoin, joinPredicates []sqlparser.Expr, - merger mergeFunc, + merger merger, subQueryInner *SubQueryInner, lhs semantics.TableSet, // these are the tables made available because we are on the RHS of a join ) (ops.PhysicalOperator, error) { @@ -292,13 +290,13 @@ func tryMergeSubqueryWithJoin( if outerOp.LeftJoin { return nil, nil } - newMergefunc := func(a, b *Route) (*Route, error) { - rt, err := merger(a, b) - if err != nil { - return nil, err - } - outerOp.RHS, err = rewriteColumnsInSubqueryOpForJoin(ctx, outerOp.RHS, outerOp, subQueryInner) - return rt, err + newMergefunc := &mergeDecorator{ + inner: merger, + f: func() error { + var err error + outerOp.RHS, err = rewriteColumnsInSubqueryOpForJoin(ctx, outerOp.RHS, outerOp, subQueryInner) + return err + }, } merged, err := tryMergeSubQueryOp(ctx, outerOp.LHS, subq, subQueryInner, joinPredicates, newMergefunc, lhs) if err != nil { @@ -309,14 +307,12 @@ func tryMergeSubqueryWithJoin( return outerOp, nil } - newMergefunc = func(a, b *Route) (*Route, error) { - rt, err := merger(a, b) - if err != nil { - return nil, err - } - outerOp.LHS, err = rewriteColumnsInSubqueryOpForJoin(ctx, outerOp.LHS, outerOp, subQueryInner) - return rt, err + newMergefunc.f = func() error { + var err error + outerOp.RHS, err = rewriteColumnsInSubqueryOpForJoin(ctx, outerOp.LHS, outerOp, subQueryInner) + return err } + merged, err = tryMergeSubQueryOp(ctx, outerOp.RHS, subq, subQueryInner, joinPredicates, newMergefunc, lhs.Merge(TableID(outerOp.LHS))) if err != nil { return nil, err diff --git a/go/vt/vtgate/planbuilder/operators/system_tables.go b/go/vt/vtgate/planbuilder/operators/system_tables.go deleted file mode 100644 index 8486805a853..00000000000 --- a/go/vt/vtgate/planbuilder/operators/system_tables.go +++ /dev/null @@ -1,142 +0,0 @@ -/* -Copyright 2022 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package operators - -import ( - "strings" - - "vitess.io/vitess/go/mysql/collations" - "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/vt/sqlparser" - "vitess.io/vitess/go/vt/vterrors" - "vitess.io/vitess/go/vt/vtgate/evalengine" -) - -func (r *Route) findSysInfoRoutingPredicatesGen4(predicates []sqlparser.Expr, reservedVars *sqlparser.ReservedVars) error { - for _, pred := range predicates { - isTableSchema, bvName, out, err := extractInfoSchemaRoutingPredicate(pred, reservedVars) - if err != nil { - return err - } - if out == nil { - // we didn't find a predicate to use for routing, continue to look for next predicate - continue - } - - if r.SysTableTableName == nil { - r.SysTableTableName = map[string]evalengine.Expr{} - } - - if isTableSchema { - r.SysTableTableSchema = append(r.SysTableTableSchema, out) - } else { - r.SysTableTableName[bvName] = out - } - } - return nil -} - -func extractInfoSchemaRoutingPredicate( - in sqlparser.Expr, - reservedVars *sqlparser.ReservedVars, -) (isSchemaName bool, name string, evalExpr evalengine.Expr, err error) { - cmp, ok := in.(*sqlparser.ComparisonExpr) - if !ok || cmp.Operator != sqlparser.EqualOp { - return - } - - isSchemaName, col := isTableOrSchemaRouteable(cmp) - if col == nil || !shouldRewrite(cmp.Right) { - return - } - - evalExpr, err = evalengine.Translate(cmp.Right, ¬ImplementedSchemaInfoConverter{}) - if err != nil { - if strings.Contains(err.Error(), evalengine.ErrTranslateExprNotSupported) { - // This just means we can't rewrite this particular expression, - // not that we have to exit altogether - err = nil - return - } - return - } - if isSchemaName { - name = sqltypes.BvSchemaName - } else { - name = reservedVars.ReserveColName(col) - } - cmp.Right = sqlparser.NewArgument(name) - return isSchemaName, name, evalExpr, nil -} - -// isTableOrSchemaRouteable searches for a comparison where one side is a table or schema name column. -// if it finds the correct column name being used, -// it also makes sure that the LHS of the comparison contains the column, and the RHS the value sought after -func isTableOrSchemaRouteable(cmp *sqlparser.ComparisonExpr) ( - isSchema bool, // tells if we are dealing with a table or a schema name comparator - col *sqlparser.ColName, // which is the colName we are comparing against -) { - if col, schema, table := isTableSchemaOrName(cmp.Left); schema || table { - return schema, col - } - if col, schema, table := isTableSchemaOrName(cmp.Right); schema || table { - // to make the rest of the code easier, we shuffle these around so the ColName is always on the LHS - cmp.Right, cmp.Left = cmp.Left, cmp.Right - return schema, col - } - - return false, nil -} - -func shouldRewrite(e sqlparser.Expr) bool { - switch node := e.(type) { - case *sqlparser.FuncExpr: - // we should not rewrite database() calls against information_schema - return !(node.Name.EqualString("database") || node.Name.EqualString("schema")) - } - return true -} - -func isTableSchemaOrName(e sqlparser.Expr) (col *sqlparser.ColName, isTableSchema bool, isTableName bool) { - col, ok := e.(*sqlparser.ColName) - if !ok { - return nil, false, false - } - return col, isDbNameCol(col), isTableNameCol(col) -} - -func isDbNameCol(col *sqlparser.ColName) bool { - return col.Name.EqualString("table_schema") || col.Name.EqualString("constraint_schema") || col.Name.EqualString("schema_name") || col.Name.EqualString("routine_schema") -} - -func isTableNameCol(col *sqlparser.ColName) bool { - return col.Name.EqualString("table_name") -} - -type notImplementedSchemaInfoConverter struct{} - -func (f *notImplementedSchemaInfoConverter) ColumnLookup(*sqlparser.ColName) (int, error) { - return 0, vterrors.VT12001("comparing table schema name with a column name") -} - -func (f *notImplementedSchemaInfoConverter) CollationForExpr(sqlparser.Expr) collations.ID { - return collations.Unknown -} - -func (f *notImplementedSchemaInfoConverter) DefaultCollation() collations.ID { - return collations.Default() -} diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index 02c094f9f2e..cc823587408 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -336,6 +336,15 @@ func TestOneWithTPCHVSchema(t *testing.T) { testFile(t, "onecase.json", "", vschema, false) } +func TestOneWith57Version(t *testing.T) { + // first we move everything to use 5.7 logic + servenv.SetMySQLServerVersionForTest("5.7") + defer servenv.SetMySQLServerVersionForTest("") + vschema := &vschemaWrapper{v: loadSchema(t, "vschemas/schema.json", true)} + + testFile(t, "onecase.json", "", vschema, false) +} + func TestRubyOnRailsQueries(t *testing.T) { vschemaWrapper := &vschemaWrapper{ v: loadSchema(t, "vschemas/rails_schema.json", true), diff --git a/go/vt/vtgate/planbuilder/plancontext/planning_context.go b/go/vt/vtgate/planbuilder/plancontext/planning_context.go index a57c9528f01..653691a4697 100644 --- a/go/vt/vtgate/planbuilder/plancontext/planning_context.go +++ b/go/vt/vtgate/planbuilder/plancontext/planning_context.go @@ -55,7 +55,7 @@ func (c PlanningContext) IsSubQueryToReplace(e sqlparser.Expr) bool { return false } for _, extractedSubq := range c.SemTable.GetSubqueryNeedingRewrite() { - if extractedSubq.NeedsRewrite && c.SemTable.EqualsExpr(extractedSubq.Subquery, ext) { + if extractedSubq.Merged && c.SemTable.EqualsExpr(extractedSubq.Subquery, ext) { return true } } diff --git a/go/vt/vtgate/planbuilder/rewrite.go b/go/vt/vtgate/planbuilder/rewrite.go index 93df3ba63c3..93884ecf863 100644 --- a/go/vt/vtgate/planbuilder/rewrite.go +++ b/go/vt/vtgate/planbuilder/rewrite.go @@ -88,6 +88,8 @@ func (r *rewriter) rewriteDown(cursor *sqlparser.Cursor) bool { // replace the table name with the original table tableName.Name = vindexTable.Name node.Expr = tableName + case *sqlparser.ExtractedSubquery: + return false case *sqlparser.Subquery: err := rewriteSubquery(cursor, r, node) if err != nil { @@ -111,9 +113,9 @@ func rewriteInSubquery(cursor *sqlparser.Cursor, r *rewriter, node *sqlparser.Co return nil } - semTableSQ, found := r.semTable.SubqueryRef[subq] - if !found { - return vterrors.VT13001("got subquery that was not in the subq map") + semTableSQ, err := r.getSubQueryRef(subq) + if err != nil { + return err } r.inSubquery++ @@ -125,9 +127,9 @@ func rewriteInSubquery(cursor *sqlparser.Cursor, r *rewriter, node *sqlparser.Co } func rewriteSubquery(cursor *sqlparser.Cursor, r *rewriter, node *sqlparser.Subquery) error { - semTableSQ, found := r.semTable.SubqueryRef[node] - if !found { - return vterrors.VT13001("got subquery that was not in the subq map") + semTableSQ, err := r.getSubQueryRef(node) + if err != nil { + return err } if semTableSQ.GetArgName() != "" || engine.PulloutOpcode(semTableSQ.OpCode) != engine.PulloutValue { return nil @@ -140,9 +142,9 @@ func rewriteSubquery(cursor *sqlparser.Cursor, r *rewriter, node *sqlparser.Subq } func (r *rewriter) rewriteExistsSubquery(cursor *sqlparser.Cursor, node *sqlparser.ExistsExpr) error { - semTableSQ, found := r.semTable.SubqueryRef[node.Subquery] - if !found { - return vterrors.VT13001("got subquery that was not in the subq map") + semTableSQ, err := r.getSubQueryRef(node.Subquery) + if err != nil { + return err } r.inSubquery++ @@ -152,6 +154,14 @@ func (r *rewriter) rewriteExistsSubquery(cursor *sqlparser.Cursor, node *sqlpars return nil } +func (r *rewriter) getSubQueryRef(sq *sqlparser.Subquery) (*sqlparser.ExtractedSubquery, error) { + semTableSQ, found := r.semTable.SubqueryRef[sq] + if !found { + return nil, vterrors.VT13001("got subquery that was not in the subq map") + } + return semTableSQ, nil +} + func rewriteHavingClause(node *sqlparser.Select) { if node.Having == nil { return diff --git a/go/vt/vtgate/planbuilder/subquery_op.go b/go/vt/vtgate/planbuilder/subquery_op.go index 060c0ecfebd..93596a5c55e 100644 --- a/go/vt/vtgate/planbuilder/subquery_op.go +++ b/go/vt/vtgate/planbuilder/subquery_op.go @@ -74,7 +74,7 @@ func mergeSubQueryOpPlan(ctx *plancontext.PlanningContext, inner, outer logicalP if canMergeSubqueryPlans(ctx, iroute, oroute) { // n.extracted is an expression that lives in oroute.Select. // Instead of looking for it in the AST, we have a copy in the subquery tree that we can update - n.Extracted.NeedsRewrite = true + n.Extracted.Merged = true replaceSubQuery(ctx, oroute.Select) return mergeSystemTableInformation(oroute, iroute) } diff --git a/go/vt/vtgate/planbuilder/testdata/filter_cases.json b/go/vt/vtgate/planbuilder/testdata/filter_cases.json index c706c78a8d4..96ebc8dda10 100644 --- a/go/vt/vtgate/planbuilder/testdata/filter_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/filter_cases.json @@ -1150,7 +1150,7 @@ "Original": "select Id from user where 1 in ('aa', 'bb')", "Instructions": { "OperatorType": "Route", - "Variant": "Scatter", + "Variant": "None", "Keyspace": { "Name": "user", "Sharded": true diff --git a/go/vt/vtgate/planbuilder/testdata/info_schema57_cases.json b/go/vt/vtgate/planbuilder/testdata/info_schema57_cases.json index c3798bbc2fe..141c1de81f6 100644 --- a/go/vt/vtgate/planbuilder/testdata/info_schema57_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/info_schema57_cases.json @@ -279,7 +279,7 @@ "Sharded": false }, "FieldQuery": "select RC.CONSTRAINT_NAME, ORDINAL_POSITION from INFORMATION_SCHEMA.KEY_COLUMN_USAGE as KCU, INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS as RC where 1 != 1", - "Query": "select RC.CONSTRAINT_NAME, ORDINAL_POSITION from INFORMATION_SCHEMA.KEY_COLUMN_USAGE as KCU, INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS as RC where KCU.TABLE_SCHEMA = :__vtschemaname and KCU.TABLE_NAME = :KCU_TABLE_NAME and KCU.COLUMN_NAME = 'id' and KCU.REFERENCED_TABLE_SCHEMA = 'test' and KCU.CONSTRAINT_NAME = 'data_type_table_id_fkey' and KCU.CONSTRAINT_NAME = RC.CONSTRAINT_NAME order by KCU.CONSTRAINT_NAME asc, KCU.COLUMN_NAME asc", + "Query": "select RC.CONSTRAINT_NAME, ORDINAL_POSITION from INFORMATION_SCHEMA.KEY_COLUMN_USAGE as KCU, INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS as RC where KCU.TABLE_SCHEMA = :__vtschemaname and KCU.TABLE_NAME = :KCU_TABLE_NAME and KCU.COLUMN_NAME = 'id' and KCU.REFERENCED_TABLE_SCHEMA = :__vtschemaname and KCU.CONSTRAINT_NAME = 'data_type_table_id_fkey' and KCU.CONSTRAINT_NAME = RC.CONSTRAINT_NAME order by KCU.CONSTRAINT_NAME asc, KCU.COLUMN_NAME asc", "SysTableTableName": "[KCU_TABLE_NAME:VARCHAR(\"data_type_table\")]", "SysTableTableSchema": "[VARCHAR(\"test\")]", "Table": "INFORMATION_SCHEMA.KEY_COLUMN_USAGE, INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS" @@ -340,7 +340,7 @@ "FieldQuery": "select KCU.TABLE_NAME, S.TABLE_NAME from INFORMATION_SCHEMA.KEY_COLUMN_USAGE as KCU, INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS as RC, INFORMATION_SCHEMA.`TABLES` as S where 1 != 1", "Query": "select KCU.TABLE_NAME, S.TABLE_NAME from INFORMATION_SCHEMA.KEY_COLUMN_USAGE as KCU, INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS as RC, INFORMATION_SCHEMA.`TABLES` as S where S.TABLE_SCHEMA = :__vtschemaname and S.TABLE_NAME = :S_TABLE_NAME and KCU.TABLE_SCHEMA = :__vtschemaname and KCU.TABLE_NAME = :KCU_TABLE_NAME and KCU.CONSTRAINT_NAME = RC.CONSTRAINT_NAME order by KCU.CONSTRAINT_NAME asc, KCU.COLUMN_NAME asc", "SysTableTableName": "[KCU_TABLE_NAME:VARCHAR(\"data_type_table\"), S_TABLE_NAME:VARCHAR(\"sc\")]", - "SysTableTableSchema": "[VARCHAR(\"test\"), VARCHAR(\"test\")]", + "SysTableTableSchema": "[VARCHAR(\"test\")]", "Table": "INFORMATION_SCHEMA.KEY_COLUMN_USAGE, INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS, INFORMATION_SCHEMA.`TABLES`" } } @@ -408,16 +408,40 @@ "QueryType": "SELECT", "Original": "SELECT kcu.constraint_name constraint_name, kcu.column_name column_name, kcu.referenced_table_name referenced_table_name, kcu.referenced_column_name referenced_column_name, kcu.ordinal_position ordinal_position, kcu.table_name table_name, rc.delete_rule delete_rule, rc.update_rule update_rule FROM information_schema.key_column_usage AS kcu INNER JOIN information_schema.referential_constraints AS rc ON kcu.constraint_name = rc.constraint_name WHERE kcu.table_schema = ? AND rc.constraint_schema = ? AND kcu.referenced_column_name IS NOT NULL ORDER BY ordinal_position", "Instructions": { - "OperatorType": "Route", - "Variant": "DBA", - "Keyspace": { - "Name": "main", - "Sharded": false + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:1,L:2,L:3,L:4,L:5,L:6,R:0,R:1", + "JoinVars": { + "kcu_constraint_name": 0 }, - "FieldQuery": "select kcu.constraint_name as constraint_name, kcu.column_name as column_name, kcu.referenced_table_name as referenced_table_name, kcu.referenced_column_name as referenced_column_name, kcu.ordinal_position as ordinal_position, kcu.table_name as table_name, rc.delete_rule as delete_rule, rc.update_rule as update_rule from information_schema.key_column_usage as kcu, information_schema.referential_constraints as rc where 1 != 1", - "Query": "select kcu.constraint_name as constraint_name, kcu.column_name as column_name, kcu.referenced_table_name as referenced_table_name, kcu.referenced_column_name as referenced_column_name, kcu.ordinal_position as ordinal_position, kcu.table_name as table_name, rc.delete_rule as delete_rule, rc.update_rule as update_rule from information_schema.key_column_usage as kcu, information_schema.referential_constraints as rc where kcu.table_schema = :__vtschemaname and kcu.referenced_column_name is not null and rc.constraint_schema = :__vtschemaname and kcu.constraint_name = rc.constraint_name order by ordinal_position asc", - "SysTableTableSchema": "[:v1, :v2]", - "Table": "information_schema.key_column_usage, information_schema.referential_constraints" + "TableName": "information_schema.key_column_usage_information_schema.referential_constraints", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "DBA", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select kcu.constraint_name, kcu.constraint_name as constraint_name, kcu.column_name as column_name, kcu.referenced_table_name as referenced_table_name, kcu.referenced_column_name as referenced_column_name, kcu.ordinal_position as ordinal_position, kcu.table_name as table_name from information_schema.key_column_usage as kcu where 1 != 1", + "OrderBy": "5 ASC", + "Query": "select kcu.constraint_name, kcu.constraint_name as constraint_name, kcu.column_name as column_name, kcu.referenced_table_name as referenced_table_name, kcu.referenced_column_name as referenced_column_name, kcu.ordinal_position as ordinal_position, kcu.table_name as table_name from information_schema.key_column_usage as kcu where kcu.table_schema = :__vtschemaname and kcu.referenced_column_name is not null order by ordinal_position asc", + "SysTableTableSchema": "[:v1]", + "Table": "information_schema.key_column_usage" + }, + { + "OperatorType": "Route", + "Variant": "DBA", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select rc.delete_rule as delete_rule, rc.update_rule as update_rule from information_schema.referential_constraints as rc where 1 != 1", + "Query": "select rc.delete_rule as delete_rule, rc.update_rule as update_rule from information_schema.referential_constraints as rc where rc.constraint_schema = :__vtschemaname and rc.constraint_name = :kcu_constraint_name", + "SysTableTableSchema": "[:v2]", + "Table": "information_schema.referential_constraints" + } + ] } } }, @@ -547,7 +571,7 @@ "FieldQuery": "select fk.referenced_table_name as to_table, fk.referenced_column_name as primary_key, fk.column_name as `column`, fk.constraint_name as `name`, rc.update_rule as on_update, rc.delete_rule as on_delete from information_schema.referential_constraints as rc, information_schema.key_column_usage as fk where 1 != 1", "Query": "select fk.referenced_table_name as to_table, fk.referenced_column_name as primary_key, fk.column_name as `column`, fk.constraint_name as `name`, rc.update_rule as on_update, rc.delete_rule as on_delete from information_schema.referential_constraints as rc, information_schema.key_column_usage as fk where rc.constraint_schema = :__vtschemaname and rc.table_name = :rc_table_name and fk.referenced_column_name is not null and fk.table_schema = :__vtschemaname and fk.table_name = :fk_table_name", "SysTableTableName": "[fk_table_name:VARCHAR(\"table_name\"), rc_table_name:VARCHAR(\"table_name\")]", - "SysTableTableSchema": "[VARCHAR(\"table_schema\"), VARCHAR(\"table_schema\")]", + "SysTableTableSchema": "[VARCHAR(\"table_schema\")]", "Table": "information_schema.key_column_usage, information_schema.referential_constraints" } } diff --git a/go/vt/vtgate/planbuilder/testdata/info_schema80_cases.json b/go/vt/vtgate/planbuilder/testdata/info_schema80_cases.json index 919ce017add..4ccb74dd117 100644 --- a/go/vt/vtgate/planbuilder/testdata/info_schema80_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/info_schema80_cases.json @@ -279,7 +279,7 @@ "Sharded": false }, "FieldQuery": "select RC.CONSTRAINT_NAME, ORDINAL_POSITION from INFORMATION_SCHEMA.KEY_COLUMN_USAGE as KCU, INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS as RC where 1 != 1", - "Query": "select RC.CONSTRAINT_NAME, ORDINAL_POSITION from INFORMATION_SCHEMA.KEY_COLUMN_USAGE as KCU, INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS as RC where KCU.TABLE_SCHEMA = :__vtschemaname and KCU.TABLE_NAME = :KCU_TABLE_NAME and KCU.COLUMN_NAME = 'id' and KCU.REFERENCED_TABLE_SCHEMA = 'test' and KCU.CONSTRAINT_NAME = 'data_type_table_id_fkey' and KCU.CONSTRAINT_NAME = RC.CONSTRAINT_NAME order by KCU.CONSTRAINT_NAME asc, KCU.COLUMN_NAME asc", + "Query": "select RC.CONSTRAINT_NAME, ORDINAL_POSITION from INFORMATION_SCHEMA.KEY_COLUMN_USAGE as KCU, INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS as RC where KCU.TABLE_SCHEMA = :__vtschemaname and KCU.TABLE_NAME = :KCU_TABLE_NAME and KCU.COLUMN_NAME = 'id' and KCU.REFERENCED_TABLE_SCHEMA = :__vtschemaname and KCU.CONSTRAINT_NAME = 'data_type_table_id_fkey' and KCU.CONSTRAINT_NAME = RC.CONSTRAINT_NAME order by KCU.CONSTRAINT_NAME asc, KCU.COLUMN_NAME asc", "SysTableTableName": "[KCU_TABLE_NAME:VARCHAR(\"data_type_table\")]", "SysTableTableSchema": "[VARCHAR(\"test\")]", "Table": "INFORMATION_SCHEMA.KEY_COLUMN_USAGE, INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS" @@ -340,7 +340,7 @@ "FieldQuery": "select KCU.TABLE_NAME, S.TABLE_NAME from INFORMATION_SCHEMA.KEY_COLUMN_USAGE as KCU, INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS as RC, INFORMATION_SCHEMA.`TABLES` as S where 1 != 1", "Query": "select KCU.TABLE_NAME, S.TABLE_NAME from INFORMATION_SCHEMA.KEY_COLUMN_USAGE as KCU, INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS as RC, INFORMATION_SCHEMA.`TABLES` as S where S.TABLE_SCHEMA = :__vtschemaname and S.TABLE_NAME = :S_TABLE_NAME and KCU.TABLE_SCHEMA = :__vtschemaname and KCU.TABLE_NAME = :KCU_TABLE_NAME and KCU.CONSTRAINT_NAME = RC.CONSTRAINT_NAME order by KCU.CONSTRAINT_NAME asc, KCU.COLUMN_NAME asc", "SysTableTableName": "[KCU_TABLE_NAME:VARCHAR(\"data_type_table\"), S_TABLE_NAME:VARCHAR(\"sc\")]", - "SysTableTableSchema": "[VARCHAR(\"test\"), VARCHAR(\"test\")]", + "SysTableTableSchema": "[VARCHAR(\"test\")]", "Table": "INFORMATION_SCHEMA.KEY_COLUMN_USAGE, INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS, INFORMATION_SCHEMA.`TABLES`" } } @@ -386,7 +386,7 @@ } }, { - "comment": "information_schema referential contraints", + "comment": "information_schema referential contraints - cant merge without knowing values", "query": "SELECT kcu.constraint_name constraint_name, kcu.column_name column_name, kcu.referenced_table_name referenced_table_name, kcu.referenced_column_name referenced_column_name, kcu.ordinal_position ordinal_position, kcu.table_name table_name, rc.delete_rule delete_rule, rc.update_rule update_rule FROM information_schema.key_column_usage AS kcu INNER JOIN information_schema.referential_constraints AS rc ON kcu.constraint_name = rc.constraint_name WHERE kcu.table_schema = ? AND rc.constraint_schema = ? AND kcu.referenced_column_name IS NOT NULL ORDER BY ordinal_position", "v3-plan": { "QueryType": "SELECT", @@ -408,16 +408,40 @@ "QueryType": "SELECT", "Original": "SELECT kcu.constraint_name constraint_name, kcu.column_name column_name, kcu.referenced_table_name referenced_table_name, kcu.referenced_column_name referenced_column_name, kcu.ordinal_position ordinal_position, kcu.table_name table_name, rc.delete_rule delete_rule, rc.update_rule update_rule FROM information_schema.key_column_usage AS kcu INNER JOIN information_schema.referential_constraints AS rc ON kcu.constraint_name = rc.constraint_name WHERE kcu.table_schema = ? AND rc.constraint_schema = ? AND kcu.referenced_column_name IS NOT NULL ORDER BY ordinal_position", "Instructions": { - "OperatorType": "Route", - "Variant": "DBA", - "Keyspace": { - "Name": "main", - "Sharded": false + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:1,L:2,L:3,L:4,L:5,L:6,R:0,R:1", + "JoinVars": { + "kcu_constraint_name": 0 }, - "FieldQuery": "select kcu.constraint_name as constraint_name, kcu.column_name as column_name, kcu.referenced_table_name as referenced_table_name, kcu.referenced_column_name as referenced_column_name, kcu.ordinal_position as ordinal_position, kcu.table_name as table_name, rc.delete_rule as delete_rule, rc.update_rule as update_rule from information_schema.key_column_usage as kcu, information_schema.referential_constraints as rc where 1 != 1", - "Query": "select kcu.constraint_name as constraint_name, kcu.column_name as column_name, kcu.referenced_table_name as referenced_table_name, kcu.referenced_column_name as referenced_column_name, kcu.ordinal_position as ordinal_position, kcu.table_name as table_name, rc.delete_rule as delete_rule, rc.update_rule as update_rule from information_schema.key_column_usage as kcu, information_schema.referential_constraints as rc where kcu.table_schema = :__vtschemaname and kcu.referenced_column_name is not null and rc.constraint_schema = :__vtschemaname and kcu.constraint_name = rc.constraint_name order by ordinal_position asc", - "SysTableTableSchema": "[:v1, :v2]", - "Table": "information_schema.key_column_usage, information_schema.referential_constraints" + "TableName": "information_schema.key_column_usage_information_schema.referential_constraints", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "DBA", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select kcu.constraint_name, kcu.constraint_name as constraint_name, kcu.column_name as column_name, kcu.referenced_table_name as referenced_table_name, kcu.referenced_column_name as referenced_column_name, kcu.ordinal_position as ordinal_position, kcu.table_name as table_name from information_schema.key_column_usage as kcu where 1 != 1", + "OrderBy": "5 ASC", + "Query": "select kcu.constraint_name, kcu.constraint_name as constraint_name, kcu.column_name as column_name, kcu.referenced_table_name as referenced_table_name, kcu.referenced_column_name as referenced_column_name, kcu.ordinal_position as ordinal_position, kcu.table_name as table_name from information_schema.key_column_usage as kcu where kcu.table_schema = :__vtschemaname and kcu.referenced_column_name is not null order by ordinal_position asc", + "SysTableTableSchema": "[:v1]", + "Table": "information_schema.key_column_usage" + }, + { + "OperatorType": "Route", + "Variant": "DBA", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select rc.delete_rule as delete_rule, rc.update_rule as update_rule from information_schema.referential_constraints as rc where 1 != 1", + "Query": "select rc.delete_rule as delete_rule, rc.update_rule as update_rule from information_schema.referential_constraints as rc where rc.constraint_schema = :__vtschemaname and rc.constraint_name = :kcu_constraint_name", + "SysTableTableSchema": "[:v2]", + "Table": "information_schema.referential_constraints" + } + ] } } }, @@ -547,7 +571,7 @@ "FieldQuery": "select fk.referenced_table_name as to_table, fk.referenced_column_name as primary_key, fk.column_name as `column`, fk.constraint_name as `name`, rc.update_rule as on_update, rc.delete_rule as on_delete from information_schema.referential_constraints as rc, information_schema.key_column_usage as fk where 1 != 1", "Query": "select fk.referenced_table_name as to_table, fk.referenced_column_name as primary_key, fk.column_name as `column`, fk.constraint_name as `name`, rc.update_rule as on_update, rc.delete_rule as on_delete from information_schema.referential_constraints as rc, information_schema.key_column_usage as fk where rc.constraint_schema = :__vtschemaname and rc.table_name = :rc_table_name and fk.referenced_column_name is not null and fk.table_schema = :__vtschemaname and fk.table_name = :fk_table_name", "SysTableTableName": "[fk_table_name:VARCHAR(\"table_name\"), rc_table_name:VARCHAR(\"table_name\")]", - "SysTableTableSchema": "[VARCHAR(\"table_schema\"), VARCHAR(\"table_schema\")]", + "SysTableTableSchema": "[VARCHAR(\"table_schema\")]", "Table": "information_schema.key_column_usage, information_schema.referential_constraints" } } @@ -576,17 +600,37 @@ "QueryType": "SELECT", "Original": "SELECT cc.constraint_name AS 'name', cc.check_clause AS 'expression' FROM information_schema.check_constraints cc JOIN information_schema.table_constraints tc USING (constraint_schema, constraint_name) WHERE tc.table_schema = 'table_schema' AND tc.table_name = 'table_name' AND cc.constraint_schema = 'constraint_schema'", "Instructions": { - "OperatorType": "Route", - "Variant": "DBA", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select cc.constraint_name as `name`, cc.check_clause as expression from information_schema.check_constraints as cc, information_schema.table_constraints as tc where 1 != 1", - "Query": "select cc.constraint_name as `name`, cc.check_clause as expression from information_schema.check_constraints as cc, information_schema.table_constraints as tc where cc.constraint_schema = :__vtschemaname and tc.table_schema = :__vtschemaname and tc.table_name = :tc_table_name", - "SysTableTableName": "[tc_table_name:VARCHAR(\"table_name\")]", - "SysTableTableSchema": "[VARCHAR(\"constraint_schema\"), VARCHAR(\"table_schema\")]", - "Table": "information_schema.check_constraints, information_schema.table_constraints" + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,L:1", + "TableName": "information_schema.check_constraints_information_schema.table_constraints", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "DBA", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select cc.constraint_name as `name`, cc.check_clause as expression from information_schema.check_constraints as cc where 1 != 1", + "Query": "select cc.constraint_name as `name`, cc.check_clause as expression from information_schema.check_constraints as cc where cc.constraint_schema = :__vtschemaname", + "SysTableTableSchema": "[VARCHAR(\"constraint_schema\")]", + "Table": "information_schema.check_constraints" + }, + { + "OperatorType": "Route", + "Variant": "DBA", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select 1 from information_schema.table_constraints as tc where 1 != 1", + "Query": "select 1 from information_schema.table_constraints as tc where tc.table_schema = :__vtschemaname and tc.table_name = :tc_table_name", + "SysTableTableName": "[tc_table_name:VARCHAR(\"table_name\")]", + "SysTableTableSchema": "[VARCHAR(\"table_schema\")]", + "Table": "information_schema.table_constraints" + } + ] } } }, diff --git a/go/vt/vtgate/planbuilder/testdata/reference_cases.json b/go/vt/vtgate/planbuilder/testdata/reference_cases.json index 375cbf9cb57..a5078bd89d1 100644 --- a/go/vt/vtgate/planbuilder/testdata/reference_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/reference_cases.json @@ -22,7 +22,7 @@ "Original": "select * from ambiguous_ref_with_source", "Instructions": { "OperatorType": "Route", - "Variant": "Reference", + "Variant": "Unsharded", "Keyspace": { "Name": "main", "Sharded": false @@ -116,7 +116,7 @@ "Original": "select r1.col from ambiguous_ref_with_source r1 join ambiguous_ref_with_source", "Instructions": { "OperatorType": "Route", - "Variant": "Reference", + "Variant": "Unsharded", "Keyspace": { "Name": "main", "Sharded": false @@ -406,7 +406,7 @@ "Original": "update user.ambiguous_ref_with_source set col = 1", "Instructions": { "OperatorType": "Update", - "Variant": "Scatter", + "Variant": "Unsharded", "Keyspace": { "Name": "main", "Sharded": false @@ -453,7 +453,7 @@ "Original": "delete from user.ambiguous_ref_with_source where col = 1", "Instructions": { "OperatorType": "Delete", - "Variant": "Scatter", + "Variant": "Unsharded", "Keyspace": { "Name": "main", "Sharded": false diff --git a/go/vt/vtgate/planbuilder/testdata/select_cases.json b/go/vt/vtgate/planbuilder/testdata/select_cases.json index acdf329e92b..3b1a17099d8 100644 --- a/go/vt/vtgate/planbuilder/testdata/select_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/select_cases.json @@ -2791,7 +2791,7 @@ "Original": "select 42 from dual where false", "Instructions": { "OperatorType": "Route", - "Variant": "Reference", + "Variant": "None", "Keyspace": { "Name": "main", "Sharded": false diff --git a/go/vt/vtgate/planbuilder/testdata/sysschema_default.json b/go/vt/vtgate/planbuilder/testdata/sysschema_default.json index 2d12dd815cf..3dcf08b102b 100644 --- a/go/vt/vtgate/planbuilder/testdata/sysschema_default.json +++ b/go/vt/vtgate/planbuilder/testdata/sysschema_default.json @@ -67,7 +67,7 @@ }, "FieldQuery": "select t.table_schema, t.table_name, c.column_name, c.column_type from information_schema.`tables` as t, information_schema.`columns` as c where 1 != 1", "Query": "select t.table_schema, t.table_name, c.column_name, c.column_type from information_schema.`tables` as t, information_schema.`columns` as c where t.table_schema = :__vtschemaname and c.table_schema = :__vtschemaname and c.table_schema = t.table_schema and c.table_name = t.table_name order by t.table_schema asc, t.table_name asc, c.column_name asc", - "SysTableTableSchema": "[VARCHAR(\"user\"), VARCHAR(\"user\")]", + "SysTableTableSchema": "[VARCHAR(\"user\")]", "Table": "information_schema.`columns`, information_schema.`tables`" } } @@ -147,4 +147,4 @@ } } } -] \ No newline at end of file +] diff --git a/go/vt/vtgate/planbuilder/vindex_op.go b/go/vt/vtgate/planbuilder/vindex_op.go index 7e81647aa7d..665ed70c4ca 100644 --- a/go/vt/vtgate/planbuilder/vindex_op.go +++ b/go/vt/vtgate/planbuilder/vindex_op.go @@ -23,7 +23,6 @@ import ( "vitess.io/vitess/go/vt/vtgate/evalengine" "vitess.io/vitess/go/vt/vtgate/planbuilder/operators" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" - "vitess.io/vitess/go/vt/vtgate/vindexes" ) diff --git a/go/vt/vtgate/semantics/semantic_state.go b/go/vt/vtgate/semantics/semantic_state.go index 49fcf715d2e..4cd38b835ff 100644 --- a/go/vt/vtgate/semantics/semantic_state.go +++ b/go/vt/vtgate/semantics/semantic_state.go @@ -343,7 +343,7 @@ func (st *SemTable) FindSubqueryReference(subquery *sqlparser.Subquery) *sqlpars func (st *SemTable) GetSubqueryNeedingRewrite() []*sqlparser.ExtractedSubquery { var res []*sqlparser.ExtractedSubquery for _, extractedSubquery := range st.SubqueryRef { - if extractedSubquery.NeedsRewrite { + if extractedSubquery.Merged { res = append(res, extractedSubquery) } } diff --git a/go/vt/vtgate/testdata/executorVSchema.json b/go/vt/vtgate/testdata/executorVSchema.json new file mode 100644 index 00000000000..da12a3b9946 --- /dev/null +++ b/go/vt/vtgate/testdata/executorVSchema.json @@ -0,0 +1,351 @@ +{ + "sharded": true, + "vindexes": { + "hash_index": { + "type": "hash" + }, + "music_user_map": { + "type": "lookup_hash_unique", + "owner": "music", + "params": { + "table": "music_user_map", + "from": "music_id", + "to": "user_id" + } + }, + "name_user_map": { + "type": "lookup_hash", + "owner": "user", + "params": { + "table": "name_user_map", + "from": "name", + "to": "user_id" + } + }, + "name_lastname_keyspace_id_map": { + "type": "lookup", + "owner": "user2", + "params": { + "table": "name_lastname_keyspace_id_map", + "from": "name,lastname", + "to": "keyspace_id" + } + }, + "insert_ignore_idx": { + "type": "lookup_hash", + "owner": "insert_ignore_test", + "params": { + "table": "ins_lookup", + "from": "fromcol", + "to": "tocol" + } + }, + "idx1": { + "type": "hash" + }, + "idx_noauto": { + "type": "hash", + "owner": "noauto_table" + }, + "keyspace_id": { + "type": "numeric" + }, + "krcol_unique_vdx": { + "type": "keyrange_lookuper_unique" + }, + "krcol_vdx": { + "type": "keyrange_lookuper" + }, + "t1_lkp_vdx": { + "type": "consistent_lookup_unique", + "params": { + "table": "t1_lkp_idx", + "from": "unq_col", + "to": "keyspace_id" + }, + "owner": "t1" + }, + "t2_wo_lu_vdx": { + "type": "lookup_unique", + "params": { + "table": "TestUnsharded.wo_lu_idx", + "from": "wo_lu_col", + "to": "keyspace_id", + "write_only": "true" + }, + "owner": "t2_lookup" + }, + "t2_erl_lu_vdx": { + "type": "lookup_unique", + "params": { + "table": "TestUnsharded.erl_lu_idx", + "from": "erl_lu_col", + "to": "keyspace_id", + "read_lock": "exclusive" + }, + "owner": "t2_lookup" + }, + "t2_srl_lu_vdx": { + "type": "lookup_unique", + "params": { + "table": "TestUnsharded.srl_lu_idx", + "from": "srl_lu_col", + "to": "keyspace_id", + "read_lock": "shared" + }, + "owner": "t2_lookup" + }, + "t2_nrl_lu_vdx": { + "type": "lookup_unique", + "params": { + "table": "TestUnsharded.nrl_lu_idx", + "from": "nrl_lu_col", + "to": "keyspace_id", + "read_lock": "none" + }, + "owner": "t2_lookup" + }, + "t2_nv_lu_vdx": { + "type": "lookup_unique", + "params": { + "table": "TestUnsharded.nv_lu_idx", + "from": "nv_lu_col", + "to": "keyspace_id", + "no_verify": "true" + }, + "owner": "t2_lookup" + }, + "t2_lu_vdx": { + "type": "lookup_hash_unique", + "params": { + "table": "TestUnsharded.lu_idx", + "from": "lu_col", + "to": "keyspace_id" + }, + "owner": "t2_lookup" + }, + "regional_vdx": { + "type": "region_experimental", + "params": { + "region_bytes": "1" + } + }, + "cfc": { + "type": "cfc" + } + }, + "tables": { + "user": { + "column_vindexes": [ + { + "column": "Id", + "name": "hash_index" + }, + { + "column": "name", + "name": "name_user_map" + } + ], + "auto_increment": { + "column": "id", + "sequence": "user_seq" + }, + "columns": [ + { + "name": "textcol", + "type": "VARCHAR" + } + ] + }, + "user2": { + "column_vindexes": [ + { + "column": "id", + "name": "hash_index" + }, + { + "columns": ["name", "lastname"], + "name": "name_lastname_keyspace_id_map" + } + ] + }, + "user_extra": { + "column_vindexes": [ + { + "column": "user_id", + "name": "hash_index" + } + ] + }, + "sharded_user_msgs": { + "column_vindexes": [ + { + "column": "user_id", + "name": "hash_index" + } + ] + }, + "music": { + "column_vindexes": [ + { + "column": "user_id", + "name": "hash_index" + }, + { + "column": "id", + "name": "music_user_map" + } + ], + "auto_increment": { + "column": "id", + "sequence": "user_seq" + } + }, + "music_extra": { + "column_vindexes": [ + { + "column": "user_id", + "name": "hash_index" + }, + { + "column": "music_id", + "name": "music_user_map" + } + ] + }, + "music_extra_reversed": { + "column_vindexes": [ + { + "column": "music_id", + "name": "music_user_map" + }, + { + "column": "user_id", + "name": "hash_index" + } + ] + }, + "insert_ignore_test": { + "column_vindexes": [ + { + "column": "pv", + "name": "music_user_map" + }, + { + "column": "owned", + "name": "insert_ignore_idx" + }, + { + "column": "verify", + "name": "hash_index" + } + ] + }, + "noauto_table": { + "column_vindexes": [ + { + "column": "id", + "name": "idx_noauto" + } + ] + }, + "keyrange_table": { + "column_vindexes": [ + { + "column": "krcol_unique", + "name": "krcol_unique_vdx" + }, + { + "column": "krcol", + "name": "krcol_vdx" + } + ] + }, + "ksid_table": { + "column_vindexes": [ + { + "column": "keyspace_id", + "name": "keyspace_id" + } + ] + }, + "t1": { + "column_vindexes": [ + { + "column": "id", + "name": "hash_index" + }, + { + "column": "unq_col", + "name": "t1_lkp_vdx" + } + ] + }, + "t1_lkp_idx": { + "column_vindexes": [ + { + "column": "unq_col", + "name": "hash_index" + } + ] + }, + "t2_lookup": { + "column_vindexes": [ + { + "column": "id", + "name": "hash_index" + }, + { + "column": "wo_lu_col", + "name": "t2_wo_lu_vdx" + }, + { + "column": "erl_lu_col", + "name": "t2_erl_lu_vdx" + }, + { + "column": "srl_lu_col", + "name": "t2_srl_lu_vdx" + }, + { + "column": "nrl_lu_col", + "name": "t2_nrl_lu_vdx" + }, + { + "column": "nv_lu_col", + "name": "t2_nv_lu_vdx" + }, + { + "column": "lu_col", + "name": "t2_lu_vdx" + } + ] + }, + "user_region": { + "column_vindexes": [ + { + "columns": ["cola","colb"], + "name": "regional_vdx" + } + ] + }, + "tbl_cfc": { + "column_vindexes": [ + { + "column": "c1", + "name": "cfc" + } + ], + "columns": [ + { + "name": "c2", + "type": "VARCHAR" + } + ] + }, + "zip_detail": { + "type": "reference", + "source": "TestUnsharded.zip_detail" + } + } +} diff --git a/go/vt/vtgate/testdata/unshardedVschema.json b/go/vt/vtgate/testdata/unshardedVschema.json new file mode 100644 index 00000000000..311cdda0c4f --- /dev/null +++ b/go/vt/vtgate/testdata/unshardedVschema.json @@ -0,0 +1,27 @@ +{ + "sharded": false, + "tables": { + "user_seq": { + "type": "sequence" + }, + "music_user_map": {}, + "name_user_map": {}, + "name_lastname_keyspace_id_map": {}, + "user_msgs": {}, + "ins_lookup": {}, + "main1": { + "auto_increment": { + "column": "id", + "sequence": "user_seq" + } + }, + "wo_lu_idx": {}, + "erl_lu_idx": {}, + "srl_lu_idx": {}, + "nrl_lu_idx": {}, + "nv_lu_idx": {}, + "lu_idx": {}, + "simple": {}, + "zip_detail": {} + } +} \ No newline at end of file