Handle bit types properly so that UPDATE and DELETE work as expected during replication

go/sqltypes/arithmetic.go

@@ -248,7 +248,7 @@ func ToNative(v Value) (interface{}, error) {
 		return ToUint64(v)
 	case v.IsFloat():
 		return ToFloat64(v)
-	case v.IsQuoted() || v.Type() == Decimal:
+	case v.IsQuoted() || v.Type() == Bit || v.Type() == Decimal:
 		out = v.val
 	case v.Type() == Expression:
 		err = vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "%v cannot be converted to a go type", v)

go/sqltypes/type.go

@@ -66,7 +66,7 @@ func IsFloat(t querypb.Type) bool {
 // IsQuoted returns true if querypb.Type is a quoted text or binary.
 // If you have a Value object, use its member function.
 func IsQuoted(t querypb.Type) bool {
-	return int(t)&flagIsQuoted == flagIsQuoted
+	return (int(t)&flagIsQuoted == flagIsQuoted) && t != Bit
 }
 
 // IsText returns true if querypb.Type is a text.
@@ -95,7 +95,7 @@ func isNumber(t querypb.Type) bool {
 // instead.
 // The following conditions are non-overlapping
 // and cover all types: IsSigned(), IsUnsigned(),
-// IsFloat(), IsQuoted(), Null, Decimal, Expression.
+// IsFloat(), IsQuoted(), Null, Decimal, Expression, Bit
 // Also, IsIntegral() == (IsSigned()||IsUnsigned()).
 // TestCategory needs to be updated accordingly if
 // you add a new type.

go/sqltypes/type_test.go

@@ -192,7 +192,7 @@ func TestCategory(t *testing.T) {
 			}
 			matched = true
 		}
-		if typ == Null || typ == Decimal || typ == Expression {
+		if typ == Null || typ == Decimal || typ == Expression || typ == Bit {
 			if matched {
 				t.Errorf("%v matched more than one category", typ)
 			}

go/sqltypes/value.go

@@ -74,7 +74,7 @@ func NewValue(typ querypb.Type, val []byte) (v Value, err error) {
 			return NULL, err
 		}
 		return MakeTrusted(typ, val), nil
-	case IsQuoted(typ) || typ == Null:
+	case IsQuoted(typ) || typ == Bit || typ == Null:
 		return MakeTrusted(typ, val), nil
 	}
 	// All other types are unsafe or invalid.
@@ -205,7 +205,7 @@ func (v Value) String() string {
 	if v.typ == Null {
 		return "NULL"
 	}
-	if v.IsQuoted() {
+	if v.IsQuoted() || v.typ == Bit {
 		return fmt.Sprintf("%v(%q)", v.typ, v.val)
 	}
 	return fmt.Sprintf("%v(%s)", v.typ, v.val)
@@ -218,6 +218,8 @@ func (v Value) EncodeSQL(b BinWriter) {
 		b.Write(nullstr)
 	case v.IsQuoted():
 		encodeBytesSQL(v.val, b)
+	case v.typ == Bit:
+		encodeBytesSQLBits(v.val, b)
 	default:
 		b.Write(v.val)
 	}
@@ -228,7 +230,7 @@ func (v Value) EncodeASCII(b BinWriter) {
 	switch {
 	case v.typ == Null:
 		b.Write(nullstr)
-	case v.IsQuoted():
+	case v.IsQuoted() || v.typ == Bit:
 		encodeBytesASCII(v.val, b)
 	default:
 		b.Write(v.val)
@@ -279,7 +281,7 @@ func (v Value) IsBinary() bool {
 // It's not a complete implementation.
 func (v Value) MarshalJSON() ([]byte, error) {
 	switch {
-	case v.IsQuoted():
+	case v.IsQuoted() || v.typ == Bit:
 		return json.Marshal(v.ToString())
 	case v.typ == Null:
 		return nullstr, nil
@@ -333,6 +335,15 @@ func encodeBytesSQL(val []byte, b BinWriter) {
 	b.Write(buf.Bytes())
 }
 
+func encodeBytesSQLBits(val []byte, b BinWriter) {
+	fmt.Fprintf(b, "%c", 'b')
+	fmt.Fprintf(b, "%c", '\'')
+	for _, ch := range val {
+		fmt.Fprintf(b, "%08b", ch)
+	}
+	fmt.Fprintf(b, "%c", '\'')
+}
+
 func encodeBytesASCII(val []byte, b BinWriter) {
 	buf := &bytes2.Buffer{}
 	buf.WriteByte('\'')

go/sqltypes/value_test.go

@@ -257,7 +257,7 @@ func TestIntegralValue(t *testing.T) {
 	}
 }
 
-func TestInerfaceValue(t *testing.T) {
+func TestInterfaceValue(t *testing.T) {
 	testcases := []struct {
 		in  interface{}
 		out Value
@@ -382,6 +382,10 @@ func TestEncode(t *testing.T) {
 		in:       TestValue(VarChar, "\x00'\"\b\n\r\t\x1A\\"),
 		outSQL:   "'\\0\\'\\\"\\b\\n\\r\\t\\Z\\\\'",
 		outASCII: "'ACciCAoNCRpc'",
+	}, {
+		in:       TestValue(Bit, "a"),
+		outSQL:   "b'01100001'",
+		outASCII: "'YQ=='",
 	}}
 	for _, tcase := range testcases {
 		buf := &bytes.Buffer{}

go/vt/sqlparser/analyzer.go

@@ -175,7 +175,7 @@ func IsValue(node Expr) bool {
 	switch v := node.(type) {
 	case *SQLVal:
 		switch v.Type {
-		case StrVal, HexVal, IntVal, ValArg:
+		case StrVal, HexVal, IntVal, BitVal, ValArg:
 			return true
 		}
 	}
@@ -224,6 +224,8 @@ func NewPlanValue(node Expr) (sqltypes.PlanValue, error) {
 			return sqltypes.PlanValue{Value: n}, nil
 		case StrVal:
 			return sqltypes.PlanValue{Value: sqltypes.MakeTrusted(sqltypes.VarBinary, node.Val)}, nil
+		case BitVal:
+			return sqltypes.PlanValue{Value: sqltypes.MakeTrusted(sqltypes.Bit, node.Val)}, nil
 		case HexVal:
 			v, err := node.HexDecode()
 			if err != nil {

go/vt/sqlparser/normalizer_test.go

@@ -117,6 +117,16 @@ func TestNormalize(t *testing.T) {
 		in:      "update a set v1 = 0x1234",
 		outstmt: "update a set v1 = 0x1234",
 		outbv:   map[string]*querypb.BindVariable{},
+	}, {
+		// Bin value does not convert
+		in:      "select * from t where v1 = b'11'",
+		outstmt: "select * from t where v1 = B'11'",
+		outbv:   map[string]*querypb.BindVariable{},
+	}, {
+		// Bin value does not convert for DMLs
+		in:      "update a set v1 = b'11'",
+		outstmt: "update a set v1 = B'11'",
+		outbv:   map[string]*querypb.BindVariable{},
 	}, {
 		// Values up to len 256 will reuse.
 		in:      fmt.Sprintf("select * from t where v1 = '%256s' and v2 = '%256s'", "a", "a"),

go/vt/vttablet/endtoend/queries_test.go

@@ -1679,7 +1679,7 @@ func TestQueries(t *testing.T) {
 					Query: "insert into vitess_misc(id, b, d, dt, t) select 2, b, d, dt, t from vitess_misc",
 					Rewritten: []string{
 						"select 2, b, d, dt, t from vitess_misc limit 10001",
-						"insert into vitess_misc(id, b, d, dt, t) values (2, '\x01', '2012-01-01', '2012-01-01 15:45:45', '15:45:45') /* _stream vitess_misc (id ) (2 )",
+						"insert into vitess_misc(id, b, d, dt, t) values (2, b'00000001', '2012-01-01', '2012-01-01 15:45:45', '15:45:45') /* _stream vitess_misc (id ) (2 )",
 					},
 				},
 				framework.TestQuery("commit"),
@@ -1798,8 +1798,11 @@ func TestQueries(t *testing.T) {
 }
 
 func TestBitDefault(t *testing.T) {
+	// Default values for bit fields that are PKs are not supported
+	// Does not make sense to use a bit field as PK
 	client := framework.NewClient()
 
+	expectedError := "bit default value: Execute failed: could not create default row for insert without row values: cannot convert value BIT(\"\\x05\") to AST (CallerID: dev)"
 	testCases := []framework.Testable{
 		&framework.MultiCase{
 			Name: "bit default value",
@@ -1824,8 +1827,9 @@ func TestBitDefault(t *testing.T) {
 		},
 	}
 	for _, tcase := range testCases {
-		if err := tcase.Test("", client); err != nil {
-			t.Error(err)
+		err := tcase.Test("", client)
+		if err == nil || err.Error() != expectedError {
+			t.Errorf("TestBitDefault result: \n%q\nexpecting\n%q", err.Error(), expectedError)
 		}
 	}
 }

test/resharding.py

@@ -202,6 +202,14 @@ def _create_schema(self):
 parent_id bigint not null,
 primary key (parent_id, id),
 index by_msg (msg)
+) Engine=InnoDB'''
+    create_table_bindata_template = '''create table %s(
+custom_ksid_col ''' + t + ''' not null,
+id bigint not null,
+parent_id bigint not null,
+msg bit(8),
+primary key (parent_id, id),
+index by_msg (msg)
 ) Engine=InnoDB'''
     create_view_template = (
         'create view %s'
@@ -236,6 +244,10 @@ def _create_schema(self):
                      '-sql=' + create_table_template % ('resharding2'),
                      'test_keyspace'],
                     auto_log=True)
+    utils.run_vtctl(['ApplySchema',
+                     '-sql=' + create_table_bindata_template % ('resharding3'),
+                     'test_keyspace'],
+                    auto_log=True)
     utils.run_vtctl(['ApplySchema',
                      '-sql=' + create_view_template % ('view1', 'resharding1'),
                      'test_keyspace'],
@@ -259,6 +271,12 @@ def _insert_startup_values(self):
                        0x9000000000000000)
     self._insert_value(shard_1_master, 'resharding1', 3, 'msg3',
                        0xD000000000000000)
+    self._insert_value(shard_0_master, 'resharding3', 1, 'a',
+                       0x1000000000000000)
+    self._insert_value(shard_1_master, 'resharding3', 2, 'b',
+                       0x9000000000000000)
+    self._insert_value(shard_1_master, 'resharding3', 3, 'c',
+                       0xD000000000000000)
     if base_sharding.use_rbr:
       self._insert_value(shard_1_master, 'no_pk', 1, 'msg1',
                          0xA000000000000000)
@@ -270,16 +288,22 @@ def _check_startup_values(self):
     # check first value is in the right shard
     for t in shard_2_tablets:
       self._check_value(t, 'resharding1', 2, 'msg2', 0x9000000000000000)
+      self._check_value(t, 'resharding3', 2, 'b', 0x9000000000000000)
     for t in shard_3_tablets:
       self._check_value(t, 'resharding1', 2, 'msg2', 0x9000000000000000,
                         should_be_here=False)
+      self._check_value(t, 'resharding3', 2, 'b', 0x9000000000000000,
+                        should_be_here=False)
 
     # check second value is in the right shard too
     for t in shard_2_tablets:
       self._check_value(t, 'resharding1', 3, 'msg3', 0xD000000000000000,
                         should_be_here=False)
+      self._check_value(t, 'resharding3', 3, 'c', 0xD000000000000000,
+                        should_be_here=False)
     for t in shard_3_tablets:
       self._check_value(t, 'resharding1', 3, 'msg3', 0xD000000000000000)
+      self._check_value(t, 'resharding3', 3, 'c', 0xD000000000000000)
 
     if base_sharding.use_rbr:
       for t in shard_2_tablets:
@@ -314,6 +338,7 @@ def _exec_multi_shard_dmls(self):
     keyspace_ids = [0x9000000000000000, 0xD000000000000000, 0xE000000000000000]
     self._insert_multi_value(shard_1_master, 'resharding1', mids,
                              msg_ids, keyspace_ids)
+
     # This update targets two shards.
     self._exec_non_annotated_update(shard_1_master, 'resharding1',
                                     [10000011, 10000012], 'update1')
@@ -326,12 +351,51 @@ def _exec_multi_shard_dmls(self):
     keyspace_ids = [0x9000000000000000, 0xD000000000000000, 0xE000000000000000]
     self._insert_multi_value(shard_1_master, 'resharding1', mids,
                              msg_ids, keyspace_ids)
+
     # This delete targets two shards.
     self._exec_non_annotated_delete(shard_1_master, 'resharding1',
                                     [10000014, 10000015])
+
     # This delete targets one shard.
     self._exec_non_annotated_delete(shard_1_master, 'resharding1', [10000016])
 
+    # repeat DMLs for table with msg as bit(8)
+    mids = [10000001, 10000002, 10000003]
+    keyspace_ids = [0x9000000000000000, 0xD000000000000000,
+                    0xE000000000000000]
+    self._insert_multi_value(shard_1_master, 'resharding3', mids,
+                             ['a','b','c'], keyspace_ids)
+
+    mids = [10000004, 10000005]
+    keyspace_ids = [0xD000000000000000, 0xE000000000000000]
+    self._insert_multi_value(shard_1_master, 'resharding3', mids,
+                            ['d', 'e'], keyspace_ids)
+    mids = [10000011, 10000012, 10000013]
+    keyspace_ids = [0x9000000000000000, 0xD000000000000000, 0xE000000000000000]
+
+    self._insert_multi_value(shard_1_master, 'resharding3', mids,
+                             ['k', 'l', 'm'], keyspace_ids)
+
+    # This update targets two shards.
+    self._exec_non_annotated_update(shard_1_master, 'resharding3',
+                                    [10000011, 10000012], 'g')
+
+    # This update targets one shard.
+    self._exec_non_annotated_update(shard_1_master, 'resharding3',
+                                    [10000013], 'h')
+
+    mids = [10000014, 10000015, 10000016]
+    keyspace_ids = [0x9000000000000000, 0xD000000000000000, 0xE000000000000000]
+    self._insert_multi_value(shard_1_master, 'resharding3', mids,
+                             ['n', 'o', 'p'], keyspace_ids)
+
+    # This delete targets two shards.
+    self._exec_non_annotated_delete(shard_1_master, 'resharding3',
+                                    [10000014, 10000015])
+
+    # This delete targets one shard.
+    self._exec_non_annotated_delete(shard_1_master, 'resharding3', [10000016])
+
   def _check_multi_shard_values(self):
     self._check_multi_dbs(
         [shard_2_master, shard_2_replica1, shard_2_replica2],
@@ -396,6 +460,70 @@ def _check_multi_shard_values(self):
         'resharding1', 10000016, 'msg-id10000016', 0xF000000000000000,
         should_be_here=False)
 
+  # checks for bit(8) table
+    self._check_multi_dbs(
+        [shard_2_master, shard_2_replica1, shard_2_replica2],
+        'resharding3', 10000001, 'a', 0x9000000000000000)
+    self._check_multi_dbs(
+        [shard_2_master, shard_2_replica1, shard_2_replica2],
+        'resharding3', 10000002, 'b', 0xD000000000000000,
+        should_be_here=False)
+    self._check_multi_dbs(
+        [shard_2_master, shard_2_replica1, shard_2_replica2],
+        'resharding3', 10000003, 'c', 0xE000000000000000,
+        should_be_here=False)
+    self._check_multi_dbs(
+        [shard_3_master, shard_3_replica],
+        'resharding3', 10000001, 'a', 0x9000000000000000,
+        should_be_here=False)
+    self._check_multi_dbs(
+        [shard_3_master, shard_3_replica],
+        'resharding3', 10000002, 'b', 0xD000000000000000)
+    self._check_multi_dbs(
+        [shard_3_master, shard_3_replica],
+        'resharding3', 10000003, 'c', 0xE000000000000000)
+
+    self._check_multi_dbs(
+        [shard_2_master, shard_2_replica1, shard_2_replica2],
+        'resharding3', 10000004, 'd', 0xD000000000000000,
+        should_be_here=False)
+    self._check_multi_dbs(
+        [shard_2_master, shard_2_replica1, shard_2_replica2],
+        'resharding3', 10000005, 'e', 0xE000000000000000,
+        should_be_here=False)
+    self._check_multi_dbs(
+        [shard_3_master, shard_3_replica],
+        'resharding3', 10000004, 'd', 0xD000000000000000)
+    self._check_multi_dbs(
+        [shard_3_master, shard_3_replica],
+        'resharding3', 10000005, 'e', 0xE000000000000000)
+
+    self._check_multi_dbs(
+        [shard_2_master, shard_2_replica1, shard_2_replica2],
+        'resharding3', 10000011, 'g', 0x9000000000000000)
+    self._check_multi_dbs(
+        [shard_3_master, shard_3_replica],
+        'resharding3', 10000012, 'g', 0xD000000000000000)
+    self._check_multi_dbs(
+        [shard_3_master, shard_3_replica],
+        'resharding3', 10000013, 'h', 0xE000000000000000)
+
+    self._check_multi_dbs(
+        [shard_2_master, shard_2_replica1, shard_2_replica2,
+         shard_3_master, shard_3_replica],
+        'resharding3', 10000014, 'n', 0x9000000000000000,
+        should_be_here=False)
+    self._check_multi_dbs(
+        [shard_2_master, shard_2_replica1, shard_2_replica2,
+         shard_3_master, shard_3_replica],
+        'resharding3', 10000015, 'o', 0xD000000000000000,
+        should_be_here=False)
+    self._check_multi_dbs(
+        [shard_2_master, shard_2_replica1, shard_2_replica2,
+         shard_3_master, shard_3_replica],
+        'resharding3', 10000016, 'p', 0xF000000000000000,
+        should_be_here=False)
+
   # _check_multi_dbs checks the row in multiple dbs.
   def _check_multi_dbs(self, dblist, table, mid, msg, keyspace_id,
                        should_be_here=True):
@@ -774,11 +902,11 @@ def test_resharding(self):
       # are smaller. In the second shard, we submitted statements
       # that affect more than one keyspace id. These will result
       # in two queries with RBR. So the count there is higher.
-      self.check_running_binlog_player(shard_2_master, 4018, 2008)
-      self.check_running_binlog_player(shard_3_master, 4028, 2008)
+      self.check_running_binlog_player(shard_2_master, 4036, 2016)
+      self.check_running_binlog_player(shard_3_master, 4056, 2016)
     else:
-      self.check_running_binlog_player(shard_2_master, 4022, 2008)
-      self.check_running_binlog_player(shard_3_master, 4024, 2008)
+      self.check_running_binlog_player(shard_2_master, 4044, 2016)
+      self.check_running_binlog_player(shard_3_master, 4048, 2016)
 
     # start a thread to insert data into shard_1 in the background
     # with current time, and monitor the delay