diff --git a/go/vt/sqlparser/comments.go b/go/vt/sqlparser/comments.go index 0167e58dad5..7952aaab71e 100644 --- a/go/vt/sqlparser/comments.go +++ b/go/vt/sqlparser/comments.go @@ -291,6 +291,30 @@ func (d CommentDirectives) GetString(key string, defaultVal string) string { return stringVal } +// MultiShardAutocommitDirective returns true if multishard autocommit directive is set to true in query. +func MultiShardAutocommitDirective(stmt Statement) bool { + switch stmt := stmt.(type) { + case *Insert: + directives := ExtractCommentDirectives(stmt.Comments) + if directives.IsSet(DirectiveMultiShardAutocommit) { + return true + } + case *Update: + directives := ExtractCommentDirectives(stmt.Comments) + if directives.IsSet(DirectiveMultiShardAutocommit) { + return true + } + case *Delete: + directives := ExtractCommentDirectives(stmt.Comments) + if directives.IsSet(DirectiveMultiShardAutocommit) { + return true + } + default: + return false + } + return false +} + // SkipQueryPlanCacheDirective returns true if skip query plan cache directive is set to true in query. func SkipQueryPlanCacheDirective(stmt Statement) bool { switch stmt := stmt.(type) { diff --git a/go/vt/vtgate/engine/send.go b/go/vt/vtgate/engine/send.go index 9c2d7f33022..f42411b2666 100644 --- a/go/vt/vtgate/engine/send.go +++ b/go/vt/vtgate/engine/send.go @@ -48,6 +48,9 @@ type Send struct { // ShardNameNeeded specified that the shard name is added to the bind variables ShardNameNeeded bool + // MultishardAutocommit specifies that a multishard transaction query can autocommit + MultishardAutocommit bool + noInputs } @@ -108,7 +111,7 @@ func (s *Send) Execute(vcursor VCursor, bindVars map[string]*querypb.BindVariabl canAutocommit := false if s.IsDML { - canAutocommit = len(rss) == 1 && vcursor.AutocommitApproval() + canAutocommit = (len(rss) == 1 || s.MultishardAutocommit) && vcursor.AutocommitApproval() } rollbackOnError := s.IsDML // for non-dml queries, there's no need to do a rollback @@ -180,6 +183,9 @@ func (s *Send) description() PrimitiveDescription { if s.ShardNameNeeded { other["ShardNameNeeded"] = true } + if s.MultishardAutocommit { + other["MultishardAutocommit"] = true + } return PrimitiveDescription{ OperatorType: "Send", Keyspace: s.Keyspace, diff --git a/go/vt/vtgate/engine/send_test.go b/go/vt/vtgate/engine/send_test.go index 536e35fc24d..1d0d7a4cbf0 100644 --- a/go/vt/vtgate/engine/send_test.go +++ b/go/vt/vtgate/engine/send_test.go @@ -31,14 +31,15 @@ import ( func TestSendTable(t *testing.T) { type testCase struct { - testName string - sharded bool - shards []string - destination key.Destination - expectedQueryLog []string - expectedError string - isDML bool - singleShardOnly bool + testName string + sharded bool + shards []string + destination key.Destination + expectedQueryLog []string + expectedError string + isDML bool + singleShardOnly bool + multiShardAutocommit bool } singleShard := []string{"0"} @@ -53,7 +54,8 @@ func TestSendTable(t *testing.T) { `ResolveDestinations ks [] Destinations:DestinationAllShards()`, `ExecuteMultiShard ks.0: dummy_query {} false false`, }, - isDML: false, + isDML: false, + multiShardAutocommit: false, }, { testName: "sharded with no autocommit", @@ -64,7 +66,8 @@ func TestSendTable(t *testing.T) { `ResolveDestinations ks [] Destinations:DestinationShard(20-)`, `ExecuteMultiShard ks.DestinationShard(20-): dummy_query {} false false`, }, - isDML: false, + isDML: false, + multiShardAutocommit: false, }, { testName: "unsharded", @@ -75,7 +78,8 @@ func TestSendTable(t *testing.T) { `ResolveDestinations ks [] Destinations:DestinationAllShards()`, `ExecuteMultiShard ks.0: dummy_query {} true true`, }, - isDML: true, + isDML: true, + multiShardAutocommit: false, }, { testName: "sharded with single shard destination", @@ -86,7 +90,8 @@ func TestSendTable(t *testing.T) { `ResolveDestinations ks [] Destinations:DestinationShard(20-)`, `ExecuteMultiShard ks.DestinationShard(20-): dummy_query {} true true`, }, - isDML: true, + isDML: true, + multiShardAutocommit: false, }, { testName: "sharded with multi shard destination", @@ -97,7 +102,20 @@ func TestSendTable(t *testing.T) { `ResolveDestinations ks [] Destinations:DestinationAllShards()`, `ExecuteMultiShard ks.-20: dummy_query {} ks.20-: dummy_query {} true false`, }, - isDML: true, + isDML: true, + multiShardAutocommit: false, + }, + { + testName: "sharded with multi shard destination and autocommit", + sharded: true, + shards: twoShards, + destination: key.DestinationAllShards{}, + expectedQueryLog: []string{ + `ResolveDestinations ks [] Destinations:DestinationAllShards()`, + `ExecuteMultiShard ks.-20: dummy_query {} ks.20-: dummy_query {} true true`, + }, + isDML: true, + multiShardAutocommit: true, }, { testName: "sharded with multi shard destination", @@ -107,9 +125,10 @@ func TestSendTable(t *testing.T) { expectedQueryLog: []string{ `ResolveDestinations ks [] Destinations:DestinationAllShards()`, }, - expectedError: "Unexpected error, DestinationKeyspaceID mapping to multiple shards: dummy_query, got: DestinationAllShards()", - isDML: true, - singleShardOnly: true, + expectedError: "Unexpected error, DestinationKeyspaceID mapping to multiple shards: dummy_query, got: DestinationAllShards()", + isDML: true, + singleShardOnly: true, + multiShardAutocommit: false, }, } @@ -120,10 +139,11 @@ func TestSendTable(t *testing.T) { Name: "ks", Sharded: tc.sharded, }, - Query: "dummy_query", - TargetDestination: tc.destination, - IsDML: tc.isDML, - SingleShardOnly: tc.singleShardOnly, + Query: "dummy_query", + TargetDestination: tc.destination, + IsDML: tc.isDML, + SingleShardOnly: tc.singleShardOnly, + MultishardAutocommit: tc.multiShardAutocommit, } vc := &loggingVCursor{shards: tc.shards} _, err := send.Execute(vc, map[string]*querypb.BindVariable{}, false) diff --git a/go/vt/vtgate/planbuilder/bypass.go b/go/vt/vtgate/planbuilder/bypass.go index abc42459abf..a13cb2ce841 100644 --- a/go/vt/vtgate/planbuilder/bypass.go +++ b/go/vt/vtgate/planbuilder/bypass.go @@ -37,10 +37,11 @@ func buildPlanForBypass(stmt sqlparser.Statement, _ *sqlparser.ReservedVars, vsc return nil, err } return &engine.Send{ - Keyspace: keyspace, - TargetDestination: vschema.Destination(), - Query: sqlparser.String(stmt), - IsDML: sqlparser.IsDMLStatement(stmt), - SingleShardOnly: false, + Keyspace: keyspace, + TargetDestination: vschema.Destination(), + Query: sqlparser.String(stmt), + IsDML: sqlparser.IsDMLStatement(stmt), + SingleShardOnly: false, + MultishardAutocommit: sqlparser.MultiShardAutocommitDirective(stmt), }, nil } diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index bd78144b732..fd600cb2588 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -217,10 +217,29 @@ func TestOne(t *testing.T) { testFile(t, "onecase.txt", "", vschema, true) } -func TestBypassPlanningFromFile(t *testing.T) { +func TestBypassPlanningShardTargetFromFile(t *testing.T) { testOutputTempDir, err := ioutil.TempDir("", "plan_test") require.NoError(t, err) defer os.RemoveAll(testOutputTempDir) + + vschema := &vschemaWrapper{ + v: loadSchema(t, "schema_test.json"), + keyspace: &vindexes.Keyspace{ + Name: "main", + Sharded: false, + }, + tabletType: topodatapb.TabletType_MASTER, + dest: key.DestinationShard("-80")} + + testFile(t, "bypass_shard_cases.txt", testOutputTempDir, vschema, true) +} +func TestBypassPlanningKeyrangeTargetFromFile(t *testing.T) { + testOutputTempDir, err := ioutil.TempDir("", "plan_test") + require.NoError(t, err) + defer os.RemoveAll(testOutputTempDir) + + keyRange, _ := key.ParseShardingSpec("-") + vschema := &vschemaWrapper{ v: loadSchema(t, "schema_test.json"), keyspace: &vindexes.Keyspace{ @@ -228,10 +247,10 @@ func TestBypassPlanningFromFile(t *testing.T) { Sharded: false, }, tabletType: topodatapb.TabletType_MASTER, - dest: key.DestinationShard("-80"), + dest: key.DestinationExactKeyRange{KeyRange: keyRange[0]}, } - testFile(t, "bypass_cases.txt", testOutputTempDir, vschema, true) + testFile(t, "bypass_keyrange_cases.txt", testOutputTempDir, vschema, true) } func TestWithDefaultKeyspaceFromFile(t *testing.T) { diff --git a/go/vt/vtgate/planbuilder/testdata/bypass_keyrange_cases.txt b/go/vt/vtgate/planbuilder/testdata/bypass_keyrange_cases.txt new file mode 100644 index 00000000000..a9bb3e93249 --- /dev/null +++ b/go/vt/vtgate/planbuilder/testdata/bypass_keyrange_cases.txt @@ -0,0 +1,163 @@ +# select bypass +"select count(*), col from unsharded" +{ + "QueryType": "SELECT", + "Original": "select count(*), col from unsharded", + "Instructions": { + "OperatorType": "Send", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetDestination": "ExactKeyRange(-)", + "Query": "select count(*), col from unsharded" + } +} +Gen4 plan same as above + +# update bypass +"update user set val = 1 where id = 18446744073709551616 and id = 1" +{ + "QueryType": "UPDATE", + "Original": "update user set val = 1 where id = 18446744073709551616 and id = 1", + "Instructions": { + "OperatorType": "Send", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetDestination": "ExactKeyRange(-)", + "IsDML": true, + "Query": "update `user` set val = 1 where id = 18446744073709551616 and id = 1" + } +} +Gen4 plan same as above + +# update bypass autocommit +"update /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ user set val = 1 where id = 18446744073709551616 and id = 1" +{ + "QueryType": "UPDATE", + "Original": "update /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ user set val = 1 where id = 18446744073709551616 and id = 1", + "Instructions": { + "OperatorType": "Send", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetDestination": "ExactKeyRange(-)", + "IsDML": true, + "MultishardAutocommit": true, + "Query": "update /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ `user` set val = 1 where id = 18446744073709551616 and id = 1" + } +} +Gen4 plan same as above + +# delete bypass +"DELETE FROM USER WHERE ID = 42" +{ + "QueryType": "DELETE", + "Original": "DELETE FROM USER WHERE ID = 42", + "Instructions": { + "OperatorType": "Send", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetDestination": "ExactKeyRange(-)", + "IsDML": true, + "Query": "delete from `USER` where ID = 42" + } +} +Gen4 plan same as above + +# insert bypass: not supported +"INSERT INTO USER (ID, NAME) VALUES (42, 'ms X')" +"INSERT not supported when targeting a key range: targetString" +Gen4 plan same as above + +# bypass query for into outfile s3 +"select count(*), col from unsharded into outfile S3 'x.txt'" +{ + "QueryType": "SELECT", + "Original": "select count(*), col from unsharded into outfile S3 'x.txt'", + "Instructions": { + "OperatorType": "Send", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetDestination": "ExactKeyRange(-)", + "Query": "select count(*), col from unsharded into outfile s3 'x.txt'" + } +} +Gen4 plan same as above + +# Select outfile +"select * from user into outfile S3 'x.txt'" +{ + "QueryType": "SELECT", + "Original": "select * from user into outfile S3 'x.txt'", + "Instructions": { + "OperatorType": "Send", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetDestination": "ExactKeyRange(-)", + "Query": "select * from `user` into outfile s3 'x.txt'" + } +} +Gen4 plan same as above + +"load data from s3 'x.txt' into table x" +{ + "QueryType": "OTHER", + "Original": "load data from s3 'x.txt' into table x", + "Instructions": { + "OperatorType": "Send", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetDestination": "ExactKeyRange(-)", + "IsDML": true, + "Query": "load data from s3 'x.txt' into table x", + "SingleShardOnly": true + } +} +Gen4 plan same as above + +"load data from s3 'x.txt'" +{ + "QueryType": "OTHER", + "Original": "load data from s3 'x.txt'", + "Instructions": { + "OperatorType": "Send", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetDestination": "ExactKeyRange(-)", + "IsDML": true, + "Query": "load data from s3 'x.txt'", + "SingleShardOnly": true + } +} +Gen4 plan same as above + +# create table +"create /* test */ table t1(id bigint, primary key(id)) /* comments */" +{ + "QueryType": "DDL", + "Original": "create /* test */ table t1(id bigint, primary key(id)) /* comments */", + "Instructions": { + "OperatorType": "Send", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetDestination": "ExactKeyRange(-)", + "Query": "create /* test */ table t1(id bigint, primary key(id)) /* comments */" + } +} +Gen4 plan same as above diff --git a/go/vt/vtgate/planbuilder/testdata/bypass_cases.txt b/go/vt/vtgate/planbuilder/testdata/bypass_shard_cases.txt similarity index 100% rename from go/vt/vtgate/planbuilder/testdata/bypass_cases.txt rename to go/vt/vtgate/planbuilder/testdata/bypass_shard_cases.txt