diff --git a/go/sqltypes/arithmetic.go b/go/sqltypes/arithmetic.go index b91cdf9a489..9b02ec56048 100644 --- a/go/sqltypes/arithmetic.go +++ b/go/sqltypes/arithmetic.go @@ -248,7 +248,7 @@ func ToNative(v Value) (interface{}, error) { return ToUint64(v) case v.IsFloat(): return ToFloat64(v) - case v.IsQuoted() || v.Type() == Decimal: + case v.IsQuoted() || v.Type() == Bit || v.Type() == Decimal: out = v.val case v.Type() == Expression: err = vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "%v cannot be converted to a go type", v) diff --git a/go/sqltypes/type.go b/go/sqltypes/type.go index f2032da33bb..cf1bed67a42 100644 --- a/go/sqltypes/type.go +++ b/go/sqltypes/type.go @@ -66,7 +66,7 @@ func IsFloat(t querypb.Type) bool { // IsQuoted returns true if querypb.Type is a quoted text or binary. // If you have a Value object, use its member function. func IsQuoted(t querypb.Type) bool { - return int(t)&flagIsQuoted == flagIsQuoted + return (int(t)&flagIsQuoted == flagIsQuoted) && t != Bit } // IsText returns true if querypb.Type is a text. @@ -95,7 +95,7 @@ func isNumber(t querypb.Type) bool { // instead. // The following conditions are non-overlapping // and cover all types: IsSigned(), IsUnsigned(), -// IsFloat(), IsQuoted(), Null, Decimal, Expression. +// IsFloat(), IsQuoted(), Null, Decimal, Expression, Bit // Also, IsIntegral() == (IsSigned()||IsUnsigned()). // TestCategory needs to be updated accordingly if // you add a new type. diff --git a/go/sqltypes/type_test.go b/go/sqltypes/type_test.go index 74f6cbc1a88..08aed75c81b 100644 --- a/go/sqltypes/type_test.go +++ b/go/sqltypes/type_test.go @@ -192,7 +192,7 @@ func TestCategory(t *testing.T) { } matched = true } - if typ == Null || typ == Decimal || typ == Expression { + if typ == Null || typ == Decimal || typ == Expression || typ == Bit { if matched { t.Errorf("%v matched more than one category", typ) } diff --git a/go/sqltypes/value.go b/go/sqltypes/value.go index 6e64263905b..4cc328decbe 100644 --- a/go/sqltypes/value.go +++ b/go/sqltypes/value.go @@ -74,7 +74,7 @@ func NewValue(typ querypb.Type, val []byte) (v Value, err error) { return NULL, err } return MakeTrusted(typ, val), nil - case IsQuoted(typ) || typ == Null: + case IsQuoted(typ) || typ == Bit || typ == Null: return MakeTrusted(typ, val), nil } // All other types are unsafe or invalid. @@ -205,7 +205,7 @@ func (v Value) String() string { if v.typ == Null { return "NULL" } - if v.IsQuoted() { + if v.IsQuoted() || v.typ == Bit { return fmt.Sprintf("%v(%q)", v.typ, v.val) } return fmt.Sprintf("%v(%s)", v.typ, v.val) @@ -218,6 +218,8 @@ func (v Value) EncodeSQL(b BinWriter) { b.Write(nullstr) case v.IsQuoted(): encodeBytesSQL(v.val, b) + case v.typ == Bit: + encodeBytesSQLBits(v.val, b) default: b.Write(v.val) } @@ -228,7 +230,7 @@ func (v Value) EncodeASCII(b BinWriter) { switch { case v.typ == Null: b.Write(nullstr) - case v.IsQuoted(): + case v.IsQuoted() || v.typ == Bit: encodeBytesASCII(v.val, b) default: b.Write(v.val) @@ -279,7 +281,7 @@ func (v Value) IsBinary() bool { // It's not a complete implementation. func (v Value) MarshalJSON() ([]byte, error) { switch { - case v.IsQuoted(): + case v.IsQuoted() || v.typ == Bit: return json.Marshal(v.ToString()) case v.typ == Null: return nullstr, nil @@ -333,6 +335,15 @@ func encodeBytesSQL(val []byte, b BinWriter) { b.Write(buf.Bytes()) } +func encodeBytesSQLBits(val []byte, b BinWriter) { + fmt.Fprintf(b, "%c", 'b') + fmt.Fprintf(b, "%c", '\'') + for _, ch := range val { + fmt.Fprintf(b, "%08b", ch) + } + fmt.Fprintf(b, "%c", '\'') +} + func encodeBytesASCII(val []byte, b BinWriter) { buf := &bytes2.Buffer{} buf.WriteByte('\'') diff --git a/go/sqltypes/value_test.go b/go/sqltypes/value_test.go index 73df210c50b..2a76058bb33 100644 --- a/go/sqltypes/value_test.go +++ b/go/sqltypes/value_test.go @@ -257,7 +257,7 @@ func TestIntegralValue(t *testing.T) { } } -func TestInerfaceValue(t *testing.T) { +func TestInterfaceValue(t *testing.T) { testcases := []struct { in interface{} out Value @@ -382,6 +382,10 @@ func TestEncode(t *testing.T) { in: TestValue(VarChar, "\x00'\"\b\n\r\t\x1A\\"), outSQL: "'\\0\\'\\\"\\b\\n\\r\\t\\Z\\\\'", outASCII: "'ACciCAoNCRpc'", + }, { + in: TestValue(Bit, "a"), + outSQL: "b'01100001'", + outASCII: "'YQ=='", }} for _, tcase := range testcases { buf := &bytes.Buffer{} diff --git a/go/vt/sqlparser/analyzer.go b/go/vt/sqlparser/analyzer.go index 010d63aafda..90d996378a8 100644 --- a/go/vt/sqlparser/analyzer.go +++ b/go/vt/sqlparser/analyzer.go @@ -175,7 +175,7 @@ func IsValue(node Expr) bool { switch v := node.(type) { case *SQLVal: switch v.Type { - case StrVal, HexVal, IntVal, ValArg: + case StrVal, HexVal, IntVal, BitVal, ValArg: return true } } @@ -224,6 +224,8 @@ func NewPlanValue(node Expr) (sqltypes.PlanValue, error) { return sqltypes.PlanValue{Value: n}, nil case StrVal: return sqltypes.PlanValue{Value: sqltypes.MakeTrusted(sqltypes.VarBinary, node.Val)}, nil + case BitVal: + return sqltypes.PlanValue{Value: sqltypes.MakeTrusted(sqltypes.Bit, node.Val)}, nil case HexVal: v, err := node.HexDecode() if err != nil { diff --git a/go/vt/sqlparser/normalizer_test.go b/go/vt/sqlparser/normalizer_test.go index 7514b9a75cb..a72b67917f2 100644 --- a/go/vt/sqlparser/normalizer_test.go +++ b/go/vt/sqlparser/normalizer_test.go @@ -117,6 +117,16 @@ func TestNormalize(t *testing.T) { in: "update a set v1 = 0x1234", outstmt: "update a set v1 = 0x1234", outbv: map[string]*querypb.BindVariable{}, + }, { + // Bin value does not convert + in: "select * from t where v1 = b'11'", + outstmt: "select * from t where v1 = B'11'", + outbv: map[string]*querypb.BindVariable{}, + }, { + // Bin value does not convert for DMLs + in: "update a set v1 = b'11'", + outstmt: "update a set v1 = B'11'", + outbv: map[string]*querypb.BindVariable{}, }, { // Values up to len 256 will reuse. in: fmt.Sprintf("select * from t where v1 = '%256s' and v2 = '%256s'", "a", "a"), diff --git a/go/vt/vttablet/endtoend/queries_test.go b/go/vt/vttablet/endtoend/queries_test.go index f321de17dab..56021b831da 100644 --- a/go/vt/vttablet/endtoend/queries_test.go +++ b/go/vt/vttablet/endtoend/queries_test.go @@ -1679,7 +1679,7 @@ func TestQueries(t *testing.T) { Query: "insert into vitess_misc(id, b, d, dt, t) select 2, b, d, dt, t from vitess_misc", Rewritten: []string{ "select 2, b, d, dt, t from vitess_misc limit 10001", - "insert into vitess_misc(id, b, d, dt, t) values (2, '\x01', '2012-01-01', '2012-01-01 15:45:45', '15:45:45') /* _stream vitess_misc (id ) (2 )", + "insert into vitess_misc(id, b, d, dt, t) values (2, b'00000001', '2012-01-01', '2012-01-01 15:45:45', '15:45:45') /* _stream vitess_misc (id ) (2 )", }, }, framework.TestQuery("commit"), @@ -1798,8 +1798,11 @@ func TestQueries(t *testing.T) { } func TestBitDefault(t *testing.T) { + // Default values for bit fields that are PKs are not supported + // Does not make sense to use a bit field as PK client := framework.NewClient() + expectedError := "bit default value: Execute failed: could not create default row for insert without row values: cannot convert value BIT(\"\\x05\") to AST (CallerID: dev)" testCases := []framework.Testable{ &framework.MultiCase{ Name: "bit default value", @@ -1824,8 +1827,9 @@ func TestBitDefault(t *testing.T) { }, } for _, tcase := range testCases { - if err := tcase.Test("", client); err != nil { - t.Error(err) + err := tcase.Test("", client) + if err == nil || err.Error() != expectedError { + t.Errorf("TestBitDefault result: \n%q\nexpecting\n%q", err.Error(), expectedError) } } } diff --git a/test/resharding.py b/test/resharding.py index 00359621dd0..8d1379855f0 100755 --- a/test/resharding.py +++ b/test/resharding.py @@ -202,6 +202,14 @@ def _create_schema(self): parent_id bigint not null, primary key (parent_id, id), index by_msg (msg) +) Engine=InnoDB''' + create_table_bindata_template = '''create table %s( +custom_ksid_col ''' + t + ''' not null, +id bigint not null, +parent_id bigint not null, +msg bit(8), +primary key (parent_id, id), +index by_msg (msg) ) Engine=InnoDB''' create_view_template = ( 'create view %s' @@ -236,6 +244,10 @@ def _create_schema(self): '-sql=' + create_table_template % ('resharding2'), 'test_keyspace'], auto_log=True) + utils.run_vtctl(['ApplySchema', + '-sql=' + create_table_bindata_template % ('resharding3'), + 'test_keyspace'], + auto_log=True) utils.run_vtctl(['ApplySchema', '-sql=' + create_view_template % ('view1', 'resharding1'), 'test_keyspace'], @@ -259,6 +271,12 @@ def _insert_startup_values(self): 0x9000000000000000) self._insert_value(shard_1_master, 'resharding1', 3, 'msg3', 0xD000000000000000) + self._insert_value(shard_0_master, 'resharding3', 1, 'a', + 0x1000000000000000) + self._insert_value(shard_1_master, 'resharding3', 2, 'b', + 0x9000000000000000) + self._insert_value(shard_1_master, 'resharding3', 3, 'c', + 0xD000000000000000) if base_sharding.use_rbr: self._insert_value(shard_1_master, 'no_pk', 1, 'msg1', 0xA000000000000000) @@ -270,16 +288,22 @@ def _check_startup_values(self): # check first value is in the right shard for t in shard_2_tablets: self._check_value(t, 'resharding1', 2, 'msg2', 0x9000000000000000) + self._check_value(t, 'resharding3', 2, 'b', 0x9000000000000000) for t in shard_3_tablets: self._check_value(t, 'resharding1', 2, 'msg2', 0x9000000000000000, should_be_here=False) + self._check_value(t, 'resharding3', 2, 'b', 0x9000000000000000, + should_be_here=False) # check second value is in the right shard too for t in shard_2_tablets: self._check_value(t, 'resharding1', 3, 'msg3', 0xD000000000000000, should_be_here=False) + self._check_value(t, 'resharding3', 3, 'c', 0xD000000000000000, + should_be_here=False) for t in shard_3_tablets: self._check_value(t, 'resharding1', 3, 'msg3', 0xD000000000000000) + self._check_value(t, 'resharding3', 3, 'c', 0xD000000000000000) if base_sharding.use_rbr: for t in shard_2_tablets: @@ -314,6 +338,7 @@ def _exec_multi_shard_dmls(self): keyspace_ids = [0x9000000000000000, 0xD000000000000000, 0xE000000000000000] self._insert_multi_value(shard_1_master, 'resharding1', mids, msg_ids, keyspace_ids) + # This update targets two shards. self._exec_non_annotated_update(shard_1_master, 'resharding1', [10000011, 10000012], 'update1') @@ -326,12 +351,51 @@ def _exec_multi_shard_dmls(self): keyspace_ids = [0x9000000000000000, 0xD000000000000000, 0xE000000000000000] self._insert_multi_value(shard_1_master, 'resharding1', mids, msg_ids, keyspace_ids) + # This delete targets two shards. self._exec_non_annotated_delete(shard_1_master, 'resharding1', [10000014, 10000015]) + # This delete targets one shard. self._exec_non_annotated_delete(shard_1_master, 'resharding1', [10000016]) + # repeat DMLs for table with msg as bit(8) + mids = [10000001, 10000002, 10000003] + keyspace_ids = [0x9000000000000000, 0xD000000000000000, + 0xE000000000000000] + self._insert_multi_value(shard_1_master, 'resharding3', mids, + ['a','b','c'], keyspace_ids) + + mids = [10000004, 10000005] + keyspace_ids = [0xD000000000000000, 0xE000000000000000] + self._insert_multi_value(shard_1_master, 'resharding3', mids, + ['d', 'e'], keyspace_ids) + mids = [10000011, 10000012, 10000013] + keyspace_ids = [0x9000000000000000, 0xD000000000000000, 0xE000000000000000] + + self._insert_multi_value(shard_1_master, 'resharding3', mids, + ['k', 'l', 'm'], keyspace_ids) + + # This update targets two shards. + self._exec_non_annotated_update(shard_1_master, 'resharding3', + [10000011, 10000012], 'g') + + # This update targets one shard. + self._exec_non_annotated_update(shard_1_master, 'resharding3', + [10000013], 'h') + + mids = [10000014, 10000015, 10000016] + keyspace_ids = [0x9000000000000000, 0xD000000000000000, 0xE000000000000000] + self._insert_multi_value(shard_1_master, 'resharding3', mids, + ['n', 'o', 'p'], keyspace_ids) + + # This delete targets two shards. + self._exec_non_annotated_delete(shard_1_master, 'resharding3', + [10000014, 10000015]) + + # This delete targets one shard. + self._exec_non_annotated_delete(shard_1_master, 'resharding3', [10000016]) + def _check_multi_shard_values(self): self._check_multi_dbs( [shard_2_master, shard_2_replica1, shard_2_replica2], @@ -396,6 +460,70 @@ def _check_multi_shard_values(self): 'resharding1', 10000016, 'msg-id10000016', 0xF000000000000000, should_be_here=False) + # checks for bit(8) table + self._check_multi_dbs( + [shard_2_master, shard_2_replica1, shard_2_replica2], + 'resharding3', 10000001, 'a', 0x9000000000000000) + self._check_multi_dbs( + [shard_2_master, shard_2_replica1, shard_2_replica2], + 'resharding3', 10000002, 'b', 0xD000000000000000, + should_be_here=False) + self._check_multi_dbs( + [shard_2_master, shard_2_replica1, shard_2_replica2], + 'resharding3', 10000003, 'c', 0xE000000000000000, + should_be_here=False) + self._check_multi_dbs( + [shard_3_master, shard_3_replica], + 'resharding3', 10000001, 'a', 0x9000000000000000, + should_be_here=False) + self._check_multi_dbs( + [shard_3_master, shard_3_replica], + 'resharding3', 10000002, 'b', 0xD000000000000000) + self._check_multi_dbs( + [shard_3_master, shard_3_replica], + 'resharding3', 10000003, 'c', 0xE000000000000000) + + self._check_multi_dbs( + [shard_2_master, shard_2_replica1, shard_2_replica2], + 'resharding3', 10000004, 'd', 0xD000000000000000, + should_be_here=False) + self._check_multi_dbs( + [shard_2_master, shard_2_replica1, shard_2_replica2], + 'resharding3', 10000005, 'e', 0xE000000000000000, + should_be_here=False) + self._check_multi_dbs( + [shard_3_master, shard_3_replica], + 'resharding3', 10000004, 'd', 0xD000000000000000) + self._check_multi_dbs( + [shard_3_master, shard_3_replica], + 'resharding3', 10000005, 'e', 0xE000000000000000) + + self._check_multi_dbs( + [shard_2_master, shard_2_replica1, shard_2_replica2], + 'resharding3', 10000011, 'g', 0x9000000000000000) + self._check_multi_dbs( + [shard_3_master, shard_3_replica], + 'resharding3', 10000012, 'g', 0xD000000000000000) + self._check_multi_dbs( + [shard_3_master, shard_3_replica], + 'resharding3', 10000013, 'h', 0xE000000000000000) + + self._check_multi_dbs( + [shard_2_master, shard_2_replica1, shard_2_replica2, + shard_3_master, shard_3_replica], + 'resharding3', 10000014, 'n', 0x9000000000000000, + should_be_here=False) + self._check_multi_dbs( + [shard_2_master, shard_2_replica1, shard_2_replica2, + shard_3_master, shard_3_replica], + 'resharding3', 10000015, 'o', 0xD000000000000000, + should_be_here=False) + self._check_multi_dbs( + [shard_2_master, shard_2_replica1, shard_2_replica2, + shard_3_master, shard_3_replica], + 'resharding3', 10000016, 'p', 0xF000000000000000, + should_be_here=False) + # _check_multi_dbs checks the row in multiple dbs. def _check_multi_dbs(self, dblist, table, mid, msg, keyspace_id, should_be_here=True): @@ -774,11 +902,11 @@ def test_resharding(self): # are smaller. In the second shard, we submitted statements # that affect more than one keyspace id. These will result # in two queries with RBR. So the count there is higher. - self.check_running_binlog_player(shard_2_master, 4018, 2008) - self.check_running_binlog_player(shard_3_master, 4028, 2008) + self.check_running_binlog_player(shard_2_master, 4036, 2016) + self.check_running_binlog_player(shard_3_master, 4056, 2016) else: - self.check_running_binlog_player(shard_2_master, 4022, 2008) - self.check_running_binlog_player(shard_3_master, 4024, 2008) + self.check_running_binlog_player(shard_2_master, 4044, 2016) + self.check_running_binlog_player(shard_3_master, 4048, 2016) # start a thread to insert data into shard_1 in the background # with current time, and monitor the delay