From c218a1ff4b0667635917212dd77f8da15794d7c9 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Mon, 8 Mar 2021 18:28:35 +0100 Subject: [PATCH 1/5] VRep JSON Columns: cast json column values during in an update statement Signed-off-by: Rohit Nayak --- go.mod | 1 + go.sum | 2 ++ go/test/endtoend/vreplication/config.go | 2 +- .../vreplication/unsharded_init_data.sql | 4 ++-- .../endtoend/vreplication/vreplication_test.go | 2 ++ .../vreplication/table_plan_builder.go | 6 +++++- .../vreplication/vplayer_flaky_test.go | 18 ++++++++++++++---- 7 files changed, 27 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 6840b146fca..07fc32afff5 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,7 @@ require ( github.com/martini-contrib/render v0.0.0-20150707142108-ec18f8345a11 github.com/mattn/go-sqlite3 v1.14.0 github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1 + github.com/mitchellh/go-ps v1.0.0 // indirect github.com/mitchellh/go-testing-interface v1.14.0 // indirect github.com/mitchellh/mapstructure v1.2.3 // indirect github.com/montanaflynn/stats v0.6.3 diff --git a/go.sum b/go.sum index 226248912b4..99a0751dba6 100644 --- a/go.sum +++ b/go.sum @@ -484,6 +484,8 @@ github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXx github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= +github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.14.0 h1:/x0XQ6h+3U3nAyk1yx+bHPURrKa9sVVvYbuqZ7pIAtI= github.com/mitchellh/go-testing-interface v1.14.0/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= diff --git a/go/test/endtoend/vreplication/config.go b/go/test/endtoend/vreplication/config.go index 8bb6150c0c5..6d73e4db255 100644 --- a/go/test/endtoend/vreplication/config.go +++ b/go/test/endtoend/vreplication/config.go @@ -3,7 +3,7 @@ package vreplication var ( initialProductSchema = ` create table product(pid int, description varbinary(128), primary key(pid)); -create table customer(cid int, name varbinary(128), typ enum('individual','soho','enterprise'), sport set('football','cricket','baseball'),ts timestamp not null default current_timestamp, primary key(cid)); +create table customer(cid int, name varbinary(128), meta json default null, typ enum('individual','soho','enterprise'), sport set('football','cricket','baseball'),ts timestamp not null default current_timestamp, primary key(cid)) CHARSET=utf8mb4; create table customer_seq(id int, next_id bigint, cache bigint, primary key(id)) comment 'vitess_sequence'; create table merchant(mname varchar(128), category varchar(128), primary key(mname)) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; create table orders(oid int, cid int, pid int, mname varchar(128), price int, primary key(oid)); diff --git a/go/test/endtoend/vreplication/unsharded_init_data.sql b/go/test/endtoend/vreplication/unsharded_init_data.sql index f8e0cc5d86f..15e2405eb0a 100644 --- a/go/test/endtoend/vreplication/unsharded_init_data.sql +++ b/go/test/endtoend/vreplication/unsharded_init_data.sql @@ -1,5 +1,5 @@ -insert into customer(cid, name, typ, sport) values(1, 'john',1,'football,baseball'); -insert into customer(cid, name, typ, sport) values(2, 'paul','soho','cricket'); +insert into customer(cid, name, typ, sport, meta) values(1, 'john',1,'football,baseball','{}'); +insert into customer(cid, name, typ, sport, meta) values(2, 'paul','soho','cricket',convert(x'7b7d' using utf8mb4)); insert into customer(cid, name, typ, sport) values(3, 'ringo','enterprise',''); insert into merchant(mname, category) values('monoprice', 'electronics'); insert into merchant(mname, category) values('newegg', 'electronics'); diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index 899273b9192..dfdb39d7af6 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -284,6 +284,8 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl insertQuery2 = "insert into customer(name, cid) values('tempCustomer4', 102)" //ID 102, hence due to reverse_bits in shard -80 require.True(t, validateThatQueryExecutesOnTablet(t, vtgateConn, customerTab1, "customer", insertQuery2, matchInsertQuery2)) + + execVtgateQuery(t, vtgateConn, "customer", "update customer set meta = convert(x'7b7d' using utf8mb4) where cid = 1") reverseKsWorkflow := "product.p2c_reverse" if testReverse { //Reverse Replicate diff --git a/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go b/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go index d02c372469a..c3325c118d7 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go +++ b/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go @@ -626,7 +626,11 @@ func (tpb *tablePlanBuilder) generateUpdateStatement() *sqlparser.ParsedQuery { switch cexpr.operation { case opExpr: bvf.mode = bvAfter - buf.Myprintf("%v", cexpr.expr) + if cexpr.colType == querypb.Type_JSON { + buf.Myprintf("convert(%v using utf8mb4)", cexpr.expr) + } else { + buf.Myprintf("%v", cexpr.expr) + } case opCount: buf.Myprintf("%v", cexpr.colName) case opSum: diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go index adf55a3f80f..08c797788da 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go @@ -1270,7 +1270,10 @@ func TestPlayerRowMove(t *testing.T) { func TestPlayerTypes(t *testing.T) { log.Errorf("TestPlayerTypes: flavor is %s", env.Flavor) - + enableJSONColumnTesting := false + if strings.EqualFold(env.Flavor, "mysql56") || strings.EqualFold(env.Flavor, "mysql57") { + enableJSONColumnTesting = true + } defer deleteTablet(addTablet(100)) execStatements(t, []string{ @@ -1303,7 +1306,7 @@ func TestPlayerTypes(t *testing.T) { "drop table binary_pk", fmt.Sprintf("drop table %s.binary_pk", vrepldb), }) - if strings.Contains(env.Flavor, "mysql57") { + if enableJSONColumnTesting { execStatements(t, []string{ "create table vitess_json(id int auto_increment, val1 json, val2 json, val3 json, val4 json, val5 json, primary key(id))", fmt.Sprintf("create table %s.vitess_json(id int, val1 json, val2 json, val3 json, val4 json, val5 json, primary key(id))", vrepldb), @@ -1386,8 +1389,7 @@ func TestPlayerTypes(t *testing.T) { {"a\x00\x00\x00", "bbb"}, }, }} - - if strings.Contains(env.Flavor, "mysql57") { + if enableJSONColumnTesting { testcases = append(testcases, testcase{ input: "insert into vitess_json(val1,val2,val3,val4,val5) values (null,'{}','123','{\"a\":[42,100]}', '{\"foo\":\"bar\"}')", output: "insert into vitess_json(id,val1,val2,val3,val4,val5) values (1," + @@ -1398,6 +1400,14 @@ func TestPlayerTypes(t *testing.T) { {"1", "", "{}", "123", `{"a": [42, 100]}`, `{"foo": "bar"}`}, }, }) + testcases = append(testcases, testcase{ + input: "update vitess_json set val4 = '{\"a\": [98, 123]}', val5 = convert(x'7b7d' using utf8mb4)", + output: "update vitess_json set val1=convert(null using utf8mb4), val2=convert('{}' using utf8mb4), val3=convert('123' using utf8mb4), val4=convert('{\\\"a\\\":[98,123]}' using utf8mb4), val5=convert('{}' using utf8mb4) where id=1", + table: "vitess_json", + data: [][]string{ + {"1", "", "{}", "123", `{"a": [98, 123]}`, `{}`}, + }, + }) } for _, tcases := range testcases { From 89bedc1388ab37fd1871daddffcccd85e0add6d3 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Mon, 8 Mar 2021 18:49:10 +0100 Subject: [PATCH 2/5] Attempt to disable json test for percona. go mod tidy-ed Signed-off-by: Rohit Nayak --- go.mod | 1 - go.sum | 2 -- go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go | 2 +- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 07fc32afff5..6840b146fca 100644 --- a/go.mod +++ b/go.mod @@ -60,7 +60,6 @@ require ( github.com/martini-contrib/render v0.0.0-20150707142108-ec18f8345a11 github.com/mattn/go-sqlite3 v1.14.0 github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1 - github.com/mitchellh/go-ps v1.0.0 // indirect github.com/mitchellh/go-testing-interface v1.14.0 // indirect github.com/mitchellh/mapstructure v1.2.3 // indirect github.com/montanaflynn/stats v0.6.3 diff --git a/go.sum b/go.sum index 99a0751dba6..226248912b4 100644 --- a/go.sum +++ b/go.sum @@ -484,8 +484,6 @@ github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXx github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= -github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.14.0 h1:/x0XQ6h+3U3nAyk1yx+bHPURrKa9sVVvYbuqZ7pIAtI= github.com/mitchellh/go-testing-interface v1.14.0/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go index 08c797788da..bfd4eff329d 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go @@ -1271,7 +1271,7 @@ func TestPlayerRowMove(t *testing.T) { func TestPlayerTypes(t *testing.T) { log.Errorf("TestPlayerTypes: flavor is %s", env.Flavor) enableJSONColumnTesting := false - if strings.EqualFold(env.Flavor, "mysql56") || strings.EqualFold(env.Flavor, "mysql57") { + if strings.EqualFold(env.Flavor, "mysql57") { enableJSONColumnTesting = true } defer deleteTablet(addTablet(100)) From 94d9496be9598887cc654772f7383c2ab7e42c22 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Tue, 9 Mar 2021 09:18:24 +0100 Subject: [PATCH 3/5] Try all flavors to filter out flavors that don't support JSON columns in CI unit tests Signed-off-by: Rohit Nayak --- .../vreplication/vplayer_flaky_test.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go index bfd4eff329d..7f4dba45096 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go @@ -1270,9 +1270,17 @@ func TestPlayerRowMove(t *testing.T) { func TestPlayerTypes(t *testing.T) { log.Errorf("TestPlayerTypes: flavor is %s", env.Flavor) - enableJSONColumnTesting := false - if strings.EqualFold(env.Flavor, "mysql57") { - enableJSONColumnTesting = true + enableJSONColumnTesting := true + // some unit test + supportedFlavorsForJSONColumnTesting := []string{"mysql57", "mysql80"} + for _, flavor := range supportedFlavorsForJSONColumnTesting { + if strings.EqualFold(env.Flavor, flavor) { + log.Warningf("Enabling testing JSON columns for flavor: %s", env.Flavor) + enableJSONColumnTesting = true + break + } else { + log.Warningf("Not testing JSON columns for flavor: %s", env.Flavor) + } } defer deleteTablet(addTablet(100)) From aec340f02cc33c38f47f0172915fc7b1d904d158 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Tue, 9 Mar 2021 10:17:14 +0100 Subject: [PATCH 4/5] Don't run json test in CI for percona/mariadb flavors, which either don't have json support or have not yet been enabled Signed-off-by: Rohit Nayak --- .../vreplication/vplayer_flaky_test.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go index 7f4dba45096..33d5f381b26 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go @@ -1271,16 +1271,9 @@ func TestPlayerRowMove(t *testing.T) { func TestPlayerTypes(t *testing.T) { log.Errorf("TestPlayerTypes: flavor is %s", env.Flavor) enableJSONColumnTesting := true - // some unit test - supportedFlavorsForJSONColumnTesting := []string{"mysql57", "mysql80"} - for _, flavor := range supportedFlavorsForJSONColumnTesting { - if strings.EqualFold(env.Flavor, flavor) { - log.Warningf("Enabling testing JSON columns for flavor: %s", env.Flavor) - enableJSONColumnTesting = true - break - } else { - log.Warningf("Not testing JSON columns for flavor: %s", env.Flavor) - } + flavor := strings.ToLower(env.Flavor) + if strings.Contains(flavor, "percona") || strings.Contains(flavor, "mariadb") { + enableJSONColumnTesting = false } defer deleteTablet(addTablet(100)) From a0f7933ed09007e9808ecdff8c3bd65c3bf61dbd Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Tue, 9 Mar 2021 11:14:26 +0100 Subject: [PATCH 5/5] Explicitly enable json tests for specific flavors which are supported in CI Signed-off-by: Rohit Nayak --- .../tabletmanager/vreplication/vplayer_flaky_test.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go index 33d5f381b26..24e2a1773c9 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go @@ -1270,10 +1270,15 @@ func TestPlayerRowMove(t *testing.T) { func TestPlayerTypes(t *testing.T) { log.Errorf("TestPlayerTypes: flavor is %s", env.Flavor) - enableJSONColumnTesting := true + enableJSONColumnTesting := false flavor := strings.ToLower(env.Flavor) - if strings.Contains(flavor, "percona") || strings.Contains(flavor, "mariadb") { - enableJSONColumnTesting = false + // Disable tests on percona (which identifies as mysql56) and mariadb platforms in CI since they + // either don't support JSON or JSON support is not enabled by default + if strings.Contains(flavor, "mysql57") || strings.Contains(flavor, "mysql80") { + log.Infof("Running JSON column type tests on flavor %s", flavor) + enableJSONColumnTesting = true + } else { + log.Warningf("Not running JSON column type tests on flavor %s", flavor) } defer deleteTablet(addTablet(100))