diff --git a/go/mysql/binlog_event_rbr.go b/go/mysql/binlog_event_rbr.go index 7e61f83b205..a8ec52dd05b 100644 --- a/go/mysql/binlog_event_rbr.go +++ b/go/mysql/binlog_event_rbr.go @@ -853,14 +853,26 @@ func CellValue(data []byte, pos int, typ byte, metadata uint16, styp querypb.Typ max := int((((metadata >> 4) & 0x300) ^ 0x300) + (metadata & 0xff)) // Length is encoded in 1 or 2 bytes. if max > 255 { + // This code path exists due to https://bugs.mysql.com/bug.php?id=37426. + // CHAR types need to allocate 3 bytes per char. So, the length for CHAR(255) + // cannot be represented in 1 byte. This also means that this rule does not + // apply to BINARY data. l := int(uint64(data[pos]) | uint64(data[pos+1])<<8) return sqltypes.MakeTrusted(querypb.Type_VARCHAR, data[pos+2:pos+2+l]), l + 2, nil } l := int(data[pos]) - return sqltypes.MakeTrusted(querypb.Type_VARCHAR, - data[pos+1:pos+1+l]), l + 1, nil + mdata := data[pos+1 : pos+1+l] + if sqltypes.IsBinary(styp) { + // Fixed length binaries have to be padded with zeroes + // up to the length of the field. Otherwise, equality checks + // fail against saved data. See https://github.com/vitessio/vitess/issues/3984. + ret := make([]byte, max) + copy(ret, mdata) + return sqltypes.MakeTrusted(querypb.Type_BINARY, ret), l + 1, nil + } + return sqltypes.MakeTrusted(querypb.Type_VARCHAR, mdata), l + 1, nil case TypeGeometry: l := 0 diff --git a/go/vt/binlog/binlogplayer/binlog_player.go b/go/vt/binlog/binlogplayer/binlog_player.go index b76fa52f42c..419bacf1624 100644 --- a/go/vt/binlog/binlogplayer/binlog_player.go +++ b/go/vt/binlog/binlogplayer/binlog_player.go @@ -170,7 +170,7 @@ func NewBinlogPlayerTables(dbClient DBClient, tablet *topodatapb.Tablet, tables // If an error is encountered, it updates the vreplication state to "Error". // If a stop position was specifed, and reached, the state is updated to "Stopped". func (blp *BinlogPlayer) ApplyBinlogEvents(ctx context.Context) error { - if err := setVReplicationState(blp.dbClient, blp.uid, BlpRunning, ""); err != nil { + if err := SetVReplicationState(blp.dbClient, blp.uid, BlpRunning, ""); err != nil { log.Errorf("Error writing Running state: %v", err) } @@ -180,7 +180,7 @@ func (blp *BinlogPlayer) ApplyBinlogEvents(ctx context.Context) error { Time: time.Now(), Message: msg, }) - if err := setVReplicationState(blp.dbClient, blp.uid, BlpError, msg); err != nil { + if err := SetVReplicationState(blp.dbClient, blp.uid, BlpError, msg); err != nil { log.Errorf("Error writing stop state: %v", err) } return err @@ -191,7 +191,7 @@ func (blp *BinlogPlayer) ApplyBinlogEvents(ctx context.Context) error { // applyEvents returns a recordable status message on termination or an error otherwise. func (blp *BinlogPlayer) applyEvents(ctx context.Context) error { // Read starting values for vreplication. - pos, stopPos, maxTPS, maxReplicationLag, err := readVRSettings(blp.dbClient, blp.uid) + pos, stopPos, maxTPS, maxReplicationLag, err := ReadVRSettings(blp.dbClient, blp.uid) if err != nil { log.Error(err) return err @@ -244,14 +244,14 @@ func (blp *BinlogPlayer) applyEvents(ctx context.Context) error { case blp.position.Equal(blp.stopPosition): msg := fmt.Sprintf("not starting BinlogPlayer, we're already at the desired position %v", blp.stopPosition) log.Info(msg) - if err := setVReplicationState(blp.dbClient, blp.uid, BlpStopped, msg); err != nil { + if err := SetVReplicationState(blp.dbClient, blp.uid, BlpStopped, msg); err != nil { log.Errorf("Error writing stop state: %v", err) } return nil case blp.position.AtLeast(blp.stopPosition): msg := fmt.Sprintf("starting point %v greater than stopping point %v", blp.position, blp.stopPosition) log.Error(msg) - if err := setVReplicationState(blp.dbClient, blp.uid, BlpStopped, msg); err != nil { + if err := SetVReplicationState(blp.dbClient, blp.uid, BlpStopped, msg); err != nil { log.Errorf("Error writing stop state: %v", err) } // Don't return an error. Otherwise, it will keep retrying. @@ -351,7 +351,7 @@ func (blp *BinlogPlayer) applyEvents(ctx context.Context) error { if blp.position.AtLeast(blp.stopPosition) { msg := "Reached stopping position, done playing logs" log.Info(msg) - if err := setVReplicationState(blp.dbClient, blp.uid, BlpStopped, msg); err != nil { + if err := SetVReplicationState(blp.dbClient, blp.uid, BlpStopped, msg); err != nil { log.Errorf("Error writing stop state: %v", err) } return nil @@ -447,7 +447,7 @@ func (blp *BinlogPlayer) writeRecoveryPosition(tx *binlogdatapb.BinlogTransactio } now := time.Now().Unix() - updateRecovery := updateVReplicationPos(blp.uid, position, now, tx.EventToken.Timestamp) + updateRecovery := GenerateUpdatePos(blp.uid, position, now, tx.EventToken.Timestamp) qr, err := blp.exec(updateRecovery) if err != nil { @@ -503,8 +503,8 @@ func CreateVReplicationTable() []string { ) ENGINE=InnoDB`} } -// setVReplicationState updates the state in the _vt.vreplication table. -func setVReplicationState(dbClient DBClient, uid uint32, state, message string) error { +// SetVReplicationState updates the state in the _vt.vreplication table. +func SetVReplicationState(dbClient DBClient, uid uint32, state, message string) error { query := fmt.Sprintf("update _vt.vreplication set state='%v', message=%v where id=%v", state, encodeString(message), uid) if _, err := dbClient.ExecuteFetch(query, 1); err != nil { return fmt.Errorf("could not set state: %v: %v", query, err) @@ -512,9 +512,9 @@ func setVReplicationState(dbClient DBClient, uid uint32, state, message string) return nil } -// readVRSettings retrieves the throttler settings for +// ReadVRSettings retrieves the throttler settings for // vreplication from the checkpoint table. -func readVRSettings(dbClient DBClient, uid uint32) (pos, stopPos string, maxTPS, maxReplicationLag int64, err error) { +func ReadVRSettings(dbClient DBClient, uid uint32) (pos, stopPos string, maxTPS, maxReplicationLag int64, err error) { query := fmt.Sprintf("select pos, stop_pos, max_tps, max_replication_lag from _vt.vreplication where id=%v", uid) qr, err := dbClient.ExecuteFetch(query, 1) if err != nil { @@ -554,9 +554,9 @@ func CreateVReplicationStopped(workflow string, source *binlogdatapb.BinlogSourc encodeString(workflow), encodeString(source.String()), encodeString(position), throttler.MaxRateModuleDisabled, throttler.ReplicationLagModuleDisabled, time.Now().Unix(), BlpStopped) } -// updateVReplicationPos returns a statement to update a value in the +// GenerateUpdatePos returns a statement to update a value in the // _vt.vreplication table. -func updateVReplicationPos(uid uint32, pos mysql.Position, timeUpdated int64, txTimestamp int64) string { +func GenerateUpdatePos(uid uint32, pos mysql.Position, timeUpdated int64, txTimestamp int64) string { if txTimestamp != 0 { return fmt.Sprintf( "update _vt.vreplication set pos=%v, time_updated=%v, transaction_timestamp=%v where id=%v", @@ -601,11 +601,17 @@ func encodeString(in string) string { } // ReadVReplicationPos returns a statement to query the gtid for a -// given shard from the _vt.vreplication table. +// given stream from the _vt.vreplication table. func ReadVReplicationPos(index uint32) string { return fmt.Sprintf("select pos from _vt.vreplication where id=%v", index) } +// ReadVReplicationStatus returns a statement to query the status fields for a +// given stream from the _vt.vreplication table. +func ReadVReplicationStatus(index uint32) string { + return fmt.Sprintf("select pos, state, message from _vt.vreplication where id=%v", index) +} + // StatsHistoryRecord is used to store a Message with timestamp type StatsHistoryRecord struct { Time time.Time diff --git a/go/vt/binlog/binlogplayer/binlog_player_test.go b/go/vt/binlog/binlogplayer/binlog_player_test.go index 8f4376f8212..4493dc33a37 100644 --- a/go/vt/binlog/binlogplayer/binlog_player_test.go +++ b/go/vt/binlog/binlogplayer/binlog_player_test.go @@ -355,7 +355,7 @@ func TestUpdateVReplicationPos(t *testing.T) { "set pos='MariaDB/0-1-8283', time_updated=88822 " + "where id=78522" - got := updateVReplicationPos(78522, mysql.Position{GTIDSet: gtid.GTIDSet()}, 88822, 0) + got := GenerateUpdatePos(78522, mysql.Position{GTIDSet: gtid.GTIDSet()}, 88822, 0) if got != want { t.Errorf("updateVReplicationPos() = %#v, want %#v", got, want) } @@ -367,7 +367,7 @@ func TestUpdateVReplicationTimestamp(t *testing.T) { "set pos='MariaDB/0-2-582', time_updated=88822, transaction_timestamp=481828 " + "where id=78522" - got := updateVReplicationPos(78522, mysql.Position{GTIDSet: gtid.GTIDSet()}, 88822, 481828) + got := GenerateUpdatePos(78522, mysql.Position{GTIDSet: gtid.GTIDSet()}, 88822, 481828) if got != want { t.Errorf("updateVReplicationPos() = %#v, want %#v", got, want) } @@ -377,6 +377,14 @@ func TestReadVReplicationPos(t *testing.T) { want := "select pos from _vt.vreplication where id=482821" got := ReadVReplicationPos(482821) if got != want { - t.Errorf("ReadVReplicationThrottlerSettings(482821) = %#v, want %#v", got, want) + t.Errorf("ReadVReplicationPos(482821) = %#v, want %#v", got, want) + } +} + +func TestReadVReplicationStatus(t *testing.T) { + want := "select pos, state, message from _vt.vreplication where id=482821" + got := ReadVReplicationStatus(482821) + if got != want { + t.Errorf("ReadVReplicationStatus(482821) = %#v, want %#v", got, want) } } diff --git a/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go b/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go index a7d945f06b6..5688624ba5e 100644 --- a/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go +++ b/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go @@ -383,6 +383,16 @@ func (fmd *FakeMysqlDaemon) GetSchema(dbName string, tables, excludeTables []str return tmutils.FilterTables(fmd.Schema, tables, excludeTables, includeViews) } +// GetColumns is part of the MysqlDaemon interface +func (fmd *FakeMysqlDaemon) GetColumns(dbName, table string) ([]string, error) { + return []string{}, nil +} + +// GetPrimaryKeyColumns is part of the MysqlDaemon interface +func (fmd *FakeMysqlDaemon) GetPrimaryKeyColumns(dbName, table string) ([]string, error) { + return []string{}, nil +} + // PreflightSchemaChange is part of the MysqlDaemon interface func (fmd *FakeMysqlDaemon) PreflightSchemaChange(dbName string, changes []string) ([]*tabletmanagerdatapb.SchemaChangeResult, error) { if fmd.PreflightSchemaChangeResult == nil { diff --git a/go/vt/mysqlctl/mysql_daemon.go b/go/vt/mysqlctl/mysql_daemon.go index c8426e8915b..60e59ee3b71 100644 --- a/go/vt/mysqlctl/mysql_daemon.go +++ b/go/vt/mysqlctl/mysql_daemon.go @@ -69,6 +69,8 @@ type MysqlDaemon interface { // Schema related methods GetSchema(dbName string, tables, excludeTables []string, includeViews bool) (*tabletmanagerdatapb.SchemaDefinition, error) + GetColumns(dbName, table string) ([]string, error) + GetPrimaryKeyColumns(dbName, table string) ([]string, error) PreflightSchemaChange(dbName string, changes []string) ([]*tabletmanagerdatapb.SchemaChangeResult, error) ApplySchemaChange(dbName string, change *tmutils.SchemaChange) (*tabletmanagerdatapb.SchemaChangeResult, error) diff --git a/go/vt/proto/binlogdata/binlogdata.pb.go b/go/vt/proto/binlogdata/binlogdata.pb.go index 1a3510d1bc5..2683321fd16 100644 --- a/go/vt/proto/binlogdata/binlogdata.pb.go +++ b/go/vt/proto/binlogdata/binlogdata.pb.go @@ -8,6 +8,7 @@ import fmt "fmt" import math "math" import query "vitess.io/vitess/go/vt/proto/query" import topodata "vitess.io/vitess/go/vt/proto/topodata" +import vtrpc "vitess.io/vitess/go/vt/proto/vtrpc" // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -20,6 +21,36 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +// OnDDLAction lists the possible actions for DDLs. +type OnDDLAction int32 + +const ( + OnDDLAction_IGNORE OnDDLAction = 0 + OnDDLAction_STOP OnDDLAction = 1 + OnDDLAction_EXEC OnDDLAction = 2 + OnDDLAction_EXEC_IGNORE OnDDLAction = 3 +) + +var OnDDLAction_name = map[int32]string{ + 0: "IGNORE", + 1: "STOP", + 2: "EXEC", + 3: "EXEC_IGNORE", +} +var OnDDLAction_value = map[string]int32{ + "IGNORE": 0, + "STOP": 1, + "EXEC": 2, + "EXEC_IGNORE": 3, +} + +func (x OnDDLAction) String() string { + return proto.EnumName(OnDDLAction_name, int32(x)) +} +func (OnDDLAction) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_binlogdata_6d214635eb8c538c, []int{0} +} + // VEventType enumerates the event types. // This list is comprehensive. Many of these types // will not be encountered in RBR mode. @@ -79,7 +110,7 @@ func (x VEventType) String() string { return proto.EnumName(VEventType_name, int32(x)) } func (VEventType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_binlogdata_e1edbb575eea20d0, []int{0} + return fileDescriptor_binlogdata_6d214635eb8c538c, []int{1} } type BinlogTransaction_Statement_Category int32 @@ -127,7 +158,7 @@ func (x BinlogTransaction_Statement_Category) String() string { return proto.EnumName(BinlogTransaction_Statement_Category_name, int32(x)) } func (BinlogTransaction_Statement_Category) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_binlogdata_e1edbb575eea20d0, []int{1, 0, 0} + return fileDescriptor_binlogdata_6d214635eb8c538c, []int{1, 0, 0} } // Charset is the per-statement charset info from a QUERY_EVENT binlog entry. @@ -147,7 +178,7 @@ func (m *Charset) Reset() { *m = Charset{} } func (m *Charset) String() string { return proto.CompactTextString(m) } func (*Charset) ProtoMessage() {} func (*Charset) Descriptor() ([]byte, []int) { - return fileDescriptor_binlogdata_e1edbb575eea20d0, []int{0} + return fileDescriptor_binlogdata_6d214635eb8c538c, []int{0} } func (m *Charset) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Charset.Unmarshal(m, b) @@ -204,7 +235,7 @@ func (m *BinlogTransaction) Reset() { *m = BinlogTransaction{} } func (m *BinlogTransaction) String() string { return proto.CompactTextString(m) } func (*BinlogTransaction) ProtoMessage() {} func (*BinlogTransaction) Descriptor() ([]byte, []int) { - return fileDescriptor_binlogdata_e1edbb575eea20d0, []int{1} + return fileDescriptor_binlogdata_6d214635eb8c538c, []int{1} } func (m *BinlogTransaction) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BinlogTransaction.Unmarshal(m, b) @@ -254,7 +285,7 @@ func (m *BinlogTransaction_Statement) Reset() { *m = BinlogTransaction_S func (m *BinlogTransaction_Statement) String() string { return proto.CompactTextString(m) } func (*BinlogTransaction_Statement) ProtoMessage() {} func (*BinlogTransaction_Statement) Descriptor() ([]byte, []int) { - return fileDescriptor_binlogdata_e1edbb575eea20d0, []int{1, 0} + return fileDescriptor_binlogdata_6d214635eb8c538c, []int{1, 0} } func (m *BinlogTransaction_Statement) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BinlogTransaction_Statement.Unmarshal(m, b) @@ -312,7 +343,7 @@ func (m *StreamKeyRangeRequest) Reset() { *m = StreamKeyRangeRequest{} } func (m *StreamKeyRangeRequest) String() string { return proto.CompactTextString(m) } func (*StreamKeyRangeRequest) ProtoMessage() {} func (*StreamKeyRangeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_binlogdata_e1edbb575eea20d0, []int{2} + return fileDescriptor_binlogdata_6d214635eb8c538c, []int{2} } func (m *StreamKeyRangeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StreamKeyRangeRequest.Unmarshal(m, b) @@ -365,7 +396,7 @@ func (m *StreamKeyRangeResponse) Reset() { *m = StreamKeyRangeResponse{} func (m *StreamKeyRangeResponse) String() string { return proto.CompactTextString(m) } func (*StreamKeyRangeResponse) ProtoMessage() {} func (*StreamKeyRangeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_binlogdata_e1edbb575eea20d0, []int{3} + return fileDescriptor_binlogdata_6d214635eb8c538c, []int{3} } func (m *StreamKeyRangeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StreamKeyRangeResponse.Unmarshal(m, b) @@ -409,7 +440,7 @@ func (m *StreamTablesRequest) Reset() { *m = StreamTablesRequest{} } func (m *StreamTablesRequest) String() string { return proto.CompactTextString(m) } func (*StreamTablesRequest) ProtoMessage() {} func (*StreamTablesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_binlogdata_e1edbb575eea20d0, []int{4} + return fileDescriptor_binlogdata_6d214635eb8c538c, []int{4} } func (m *StreamTablesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StreamTablesRequest.Unmarshal(m, b) @@ -462,7 +493,7 @@ func (m *StreamTablesResponse) Reset() { *m = StreamTablesResponse{} } func (m *StreamTablesResponse) String() string { return proto.CompactTextString(m) } func (*StreamTablesResponse) ProtoMessage() {} func (*StreamTablesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_binlogdata_e1edbb575eea20d0, []int{5} + return fileDescriptor_binlogdata_6d214635eb8c538c, []int{5} } func (m *StreamTablesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StreamTablesResponse.Unmarshal(m, b) @@ -507,7 +538,7 @@ func (m *Rule) Reset() { *m = Rule{} } func (m *Rule) String() string { return proto.CompactTextString(m) } func (*Rule) ProtoMessage() {} func (*Rule) Descriptor() ([]byte, []int) { - return fileDescriptor_binlogdata_e1edbb575eea20d0, []int{6} + return fileDescriptor_binlogdata_6d214635eb8c538c, []int{6} } func (m *Rule) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Rule.Unmarshal(m, b) @@ -554,7 +585,7 @@ func (m *Filter) Reset() { *m = Filter{} } func (m *Filter) String() string { return proto.CompactTextString(m) } func (*Filter) ProtoMessage() {} func (*Filter) Descriptor() ([]byte, []int) { - return fileDescriptor_binlogdata_e1edbb575eea20d0, []int{7} + return fileDescriptor_binlogdata_6d214635eb8c538c, []int{7} } func (m *Filter) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Filter.Unmarshal(m, b) @@ -597,17 +628,19 @@ type BinlogSource struct { Tables []string `protobuf:"bytes,5,rep,name=tables,proto3" json:"tables,omitempty"` // filter is set if we're using the generalized representation // for the filter. - Filter *Filter `protobuf:"bytes,6,opt,name=filter,proto3" json:"filter,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Filter *Filter `protobuf:"bytes,6,opt,name=filter,proto3" json:"filter,omitempty"` + // on_ddl specifies the action to be taken when a DDL is encountered. + OnDdl OnDDLAction `protobuf:"varint,7,opt,name=on_ddl,json=onDdl,proto3,enum=binlogdata.OnDDLAction" json:"on_ddl,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *BinlogSource) Reset() { *m = BinlogSource{} } func (m *BinlogSource) String() string { return proto.CompactTextString(m) } func (*BinlogSource) ProtoMessage() {} func (*BinlogSource) Descriptor() ([]byte, []int) { - return fileDescriptor_binlogdata_e1edbb575eea20d0, []int{8} + return fileDescriptor_binlogdata_6d214635eb8c538c, []int{8} } func (m *BinlogSource) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BinlogSource.Unmarshal(m, b) @@ -669,6 +702,13 @@ func (m *BinlogSource) GetFilter() *Filter { return nil } +func (m *BinlogSource) GetOnDdl() OnDDLAction { + if m != nil { + return m.OnDdl + } + return OnDDLAction_IGNORE +} + // RowChange represents one row change type RowChange struct { Before *query.Row `protobuf:"bytes,1,opt,name=before,proto3" json:"before,omitempty"` @@ -682,7 +722,7 @@ func (m *RowChange) Reset() { *m = RowChange{} } func (m *RowChange) String() string { return proto.CompactTextString(m) } func (*RowChange) ProtoMessage() {} func (*RowChange) Descriptor() ([]byte, []int) { - return fileDescriptor_binlogdata_e1edbb575eea20d0, []int{9} + return fileDescriptor_binlogdata_6d214635eb8c538c, []int{9} } func (m *RowChange) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RowChange.Unmarshal(m, b) @@ -729,7 +769,7 @@ func (m *RowEvent) Reset() { *m = RowEvent{} } func (m *RowEvent) String() string { return proto.CompactTextString(m) } func (*RowEvent) ProtoMessage() {} func (*RowEvent) Descriptor() ([]byte, []int) { - return fileDescriptor_binlogdata_e1edbb575eea20d0, []int{10} + return fileDescriptor_binlogdata_6d214635eb8c538c, []int{10} } func (m *RowEvent) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RowEvent.Unmarshal(m, b) @@ -775,7 +815,7 @@ func (m *FieldEvent) Reset() { *m = FieldEvent{} } func (m *FieldEvent) String() string { return proto.CompactTextString(m) } func (*FieldEvent) ProtoMessage() {} func (*FieldEvent) Descriptor() ([]byte, []int) { - return fileDescriptor_binlogdata_e1edbb575eea20d0, []int{11} + return fileDescriptor_binlogdata_6d214635eb8c538c, []int{11} } func (m *FieldEvent) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FieldEvent.Unmarshal(m, b) @@ -812,10 +852,11 @@ func (m *FieldEvent) GetFields() []*query.Field { // VEvent represents a vstream event type VEvent struct { Type VEventType `protobuf:"varint,1,opt,name=type,proto3,enum=binlogdata.VEventType" json:"type,omitempty"` - Gtid string `protobuf:"bytes,2,opt,name=gtid,proto3" json:"gtid,omitempty"` - Ddl string `protobuf:"bytes,3,opt,name=ddl,proto3" json:"ddl,omitempty"` - RowEvent *RowEvent `protobuf:"bytes,4,opt,name=row_event,json=rowEvent,proto3" json:"row_event,omitempty"` - FieldEvent *FieldEvent `protobuf:"bytes,5,opt,name=field_event,json=fieldEvent,proto3" json:"field_event,omitempty"` + Timestamp int64 `protobuf:"varint,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + Gtid string `protobuf:"bytes,3,opt,name=gtid,proto3" json:"gtid,omitempty"` + Ddl string `protobuf:"bytes,4,opt,name=ddl,proto3" json:"ddl,omitempty"` + RowEvent *RowEvent `protobuf:"bytes,5,opt,name=row_event,json=rowEvent,proto3" json:"row_event,omitempty"` + FieldEvent *FieldEvent `protobuf:"bytes,6,opt,name=field_event,json=fieldEvent,proto3" json:"field_event,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -825,7 +866,7 @@ func (m *VEvent) Reset() { *m = VEvent{} } func (m *VEvent) String() string { return proto.CompactTextString(m) } func (*VEvent) ProtoMessage() {} func (*VEvent) Descriptor() ([]byte, []int) { - return fileDescriptor_binlogdata_e1edbb575eea20d0, []int{12} + return fileDescriptor_binlogdata_6d214635eb8c538c, []int{12} } func (m *VEvent) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VEvent.Unmarshal(m, b) @@ -852,6 +893,13 @@ func (m *VEvent) GetType() VEventType { return VEventType_UNKNOWN } +func (m *VEvent) GetTimestamp() int64 { + if m != nil { + return m.Timestamp + } + return 0 +} + func (m *VEvent) GetGtid() string { if m != nil { return m.Gtid @@ -882,18 +930,21 @@ func (m *VEvent) GetFieldEvent() *FieldEvent { // VStreamRequest is the payload for VStream type VStreamRequest struct { - Position string `protobuf:"bytes,1,opt,name=position,proto3" json:"position,omitempty"` - Filter *Filter `protobuf:"bytes,2,opt,name=filter,proto3" json:"filter,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId,proto3" json:"effective_caller_id,omitempty"` + ImmediateCallerId *query.VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId,proto3" json:"immediate_caller_id,omitempty"` + Target *query.Target `protobuf:"bytes,3,opt,name=target,proto3" json:"target,omitempty"` + Position string `protobuf:"bytes,4,opt,name=position,proto3" json:"position,omitempty"` + Filter *Filter `protobuf:"bytes,5,opt,name=filter,proto3" json:"filter,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *VStreamRequest) Reset() { *m = VStreamRequest{} } func (m *VStreamRequest) String() string { return proto.CompactTextString(m) } func (*VStreamRequest) ProtoMessage() {} func (*VStreamRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_binlogdata_e1edbb575eea20d0, []int{13} + return fileDescriptor_binlogdata_6d214635eb8c538c, []int{13} } func (m *VStreamRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VStreamRequest.Unmarshal(m, b) @@ -913,6 +964,27 @@ func (m *VStreamRequest) XXX_DiscardUnknown() { var xxx_messageInfo_VStreamRequest proto.InternalMessageInfo +func (m *VStreamRequest) GetEffectiveCallerId() *vtrpc.CallerID { + if m != nil { + return m.EffectiveCallerId + } + return nil +} + +func (m *VStreamRequest) GetImmediateCallerId() *query.VTGateCallerID { + if m != nil { + return m.ImmediateCallerId + } + return nil +} + +func (m *VStreamRequest) GetTarget() *query.Target { + if m != nil { + return m.Target + } + return nil +} + func (m *VStreamRequest) GetPosition() string { if m != nil { return m.Position @@ -929,7 +1001,7 @@ func (m *VStreamRequest) GetFilter() *Filter { // VStreamResponse is the response from VStream type VStreamResponse struct { - Event []*VEvent `protobuf:"bytes,1,rep,name=event,proto3" json:"event,omitempty"` + Events []*VEvent `protobuf:"bytes,1,rep,name=events,proto3" json:"events,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -939,7 +1011,7 @@ func (m *VStreamResponse) Reset() { *m = VStreamResponse{} } func (m *VStreamResponse) String() string { return proto.CompactTextString(m) } func (*VStreamResponse) ProtoMessage() {} func (*VStreamResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_binlogdata_e1edbb575eea20d0, []int{14} + return fileDescriptor_binlogdata_6d214635eb8c538c, []int{14} } func (m *VStreamResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VStreamResponse.Unmarshal(m, b) @@ -959,9 +1031,9 @@ func (m *VStreamResponse) XXX_DiscardUnknown() { var xxx_messageInfo_VStreamResponse proto.InternalMessageInfo -func (m *VStreamResponse) GetEvent() []*VEvent { +func (m *VStreamResponse) GetEvents() []*VEvent { if m != nil { - return m.Event + return m.Events } return nil } @@ -983,76 +1055,87 @@ func init() { proto.RegisterType((*VEvent)(nil), "binlogdata.VEvent") proto.RegisterType((*VStreamRequest)(nil), "binlogdata.VStreamRequest") proto.RegisterType((*VStreamResponse)(nil), "binlogdata.VStreamResponse") + proto.RegisterEnum("binlogdata.OnDDLAction", OnDDLAction_name, OnDDLAction_value) proto.RegisterEnum("binlogdata.VEventType", VEventType_name, VEventType_value) proto.RegisterEnum("binlogdata.BinlogTransaction_Statement_Category", BinlogTransaction_Statement_Category_name, BinlogTransaction_Statement_Category_value) } -func init() { proto.RegisterFile("binlogdata.proto", fileDescriptor_binlogdata_e1edbb575eea20d0) } - -var fileDescriptor_binlogdata_e1edbb575eea20d0 = []byte{ - // 1017 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x56, 0xdb, 0x6e, 0xdb, 0x46, - 0x13, 0x0e, 0x45, 0x8a, 0x22, 0x87, 0x8e, 0xbd, 0x5e, 0x1f, 0x20, 0x18, 0x08, 0x60, 0x10, 0x3f, - 0xfe, 0xb8, 0x06, 0x2a, 0xa5, 0xea, 0xe9, 0xa2, 0x57, 0x96, 0x44, 0xbb, 0x8a, 0x69, 0xc9, 0x59, - 0xd3, 0x49, 0x91, 0x1b, 0x82, 0x96, 0xd6, 0xb2, 0x60, 0x89, 0x94, 0xc9, 0x95, 0x5d, 0x3d, 0x47, - 0x9f, 0xa2, 0x7d, 0x90, 0xbc, 0x49, 0xef, 0xfa, 0x10, 0xc5, 0x1e, 0x48, 0x49, 0x0e, 0x90, 0xaa, - 0x17, 0xbd, 0x9b, 0xd3, 0x7e, 0x3b, 0xf3, 0xcd, 0x70, 0x87, 0x80, 0x6e, 0x46, 0xf1, 0x38, 0x19, - 0x0e, 0x22, 0x16, 0xd5, 0xa6, 0x69, 0xc2, 0x12, 0x0c, 0x0b, 0xcb, 0x81, 0xf3, 0x30, 0xa3, 0xe9, - 0x5c, 0x3a, 0x0e, 0x36, 0x59, 0x32, 0x4d, 0x16, 0x81, 0xee, 0x05, 0x54, 0x5a, 0x77, 0x51, 0x9a, - 0x51, 0x86, 0xf7, 0xc1, 0xec, 0x8f, 0x47, 0x34, 0x66, 0x55, 0xed, 0x50, 0x3b, 0x2a, 0x13, 0xa5, - 0x61, 0x0c, 0x46, 0x3f, 0x89, 0xe3, 0x6a, 0x49, 0x58, 0x85, 0xcc, 0x63, 0x33, 0x9a, 0x3e, 0xd2, - 0xb4, 0xaa, 0xcb, 0x58, 0xa9, 0xb9, 0x7f, 0xea, 0xb0, 0xdd, 0x14, 0x57, 0x07, 0x69, 0x14, 0x67, - 0x51, 0x9f, 0x8d, 0x92, 0x18, 0x9f, 0x01, 0x64, 0x2c, 0x62, 0x74, 0x42, 0x63, 0x96, 0x55, 0xb5, - 0x43, 0xfd, 0xc8, 0x69, 0xbc, 0xae, 0x2d, 0x25, 0xfd, 0xd9, 0x91, 0xda, 0x55, 0x1e, 0x4f, 0x96, - 0x8e, 0xe2, 0x06, 0x38, 0xf4, 0x91, 0xc6, 0x2c, 0x64, 0xc9, 0x3d, 0x8d, 0xab, 0xc6, 0xa1, 0x76, - 0xe4, 0x34, 0xb6, 0x6b, 0xb2, 0x40, 0x8f, 0x7b, 0x02, 0xee, 0x20, 0x40, 0x0b, 0xf9, 0xe0, 0x53, - 0x09, 0xec, 0x02, 0x0d, 0xfb, 0x60, 0xf5, 0x23, 0x46, 0x87, 0x49, 0x3a, 0x17, 0x65, 0x6e, 0x36, - 0xde, 0xac, 0x99, 0x48, 0xad, 0xa5, 0xce, 0x91, 0x02, 0x01, 0x7f, 0x0d, 0x95, 0xbe, 0x64, 0x4f, - 0xb0, 0xe3, 0x34, 0x76, 0x96, 0xc1, 0x14, 0xb1, 0x24, 0x8f, 0xc1, 0x08, 0xf4, 0xec, 0x61, 0x2c, - 0x28, 0xdb, 0x20, 0x5c, 0x74, 0x7f, 0xd7, 0xc0, 0xca, 0x71, 0xf1, 0x0e, 0x6c, 0x35, 0xfd, 0xf0, - 0xba, 0x4b, 0xbc, 0x56, 0xef, 0xac, 0xdb, 0xf9, 0xe8, 0xb5, 0xd1, 0x0b, 0xbc, 0x01, 0x56, 0xd3, - 0x0f, 0x9b, 0xde, 0x59, 0xa7, 0x8b, 0x34, 0xfc, 0x12, 0xec, 0xa6, 0x1f, 0xb6, 0x7a, 0x17, 0x17, - 0x9d, 0x00, 0x95, 0xf0, 0x16, 0x38, 0x4d, 0x3f, 0x24, 0x3d, 0xdf, 0x6f, 0x9e, 0xb4, 0xce, 0x91, - 0x8e, 0xf7, 0x60, 0xbb, 0xe9, 0x87, 0xed, 0x0b, 0x3f, 0x6c, 0x7b, 0x97, 0xc4, 0x6b, 0x9d, 0x04, - 0x5e, 0x1b, 0x19, 0x18, 0xc0, 0xe4, 0xe6, 0xb6, 0x8f, 0xca, 0x4a, 0xbe, 0xf2, 0x02, 0x64, 0x2a, - 0xb8, 0x4e, 0xf7, 0xca, 0x23, 0x01, 0xaa, 0x28, 0xf5, 0xfa, 0xb2, 0x7d, 0x12, 0x78, 0xc8, 0x52, - 0x6a, 0xdb, 0xf3, 0xbd, 0xc0, 0x43, 0xf6, 0x5b, 0xc3, 0x2a, 0x21, 0xfd, 0xad, 0x61, 0xe9, 0xc8, - 0x70, 0x7f, 0xd3, 0x60, 0xef, 0x8a, 0xa5, 0x34, 0x9a, 0x9c, 0xd3, 0x39, 0x89, 0xe2, 0x21, 0x25, - 0xf4, 0x61, 0x46, 0x33, 0x86, 0x0f, 0xc0, 0x9a, 0x26, 0xd9, 0x88, 0x73, 0x27, 0x08, 0xb6, 0x49, - 0xa1, 0xe3, 0x3a, 0xd8, 0xf7, 0x74, 0x1e, 0xa6, 0x3c, 0x5e, 0x11, 0x86, 0x6b, 0xc5, 0x40, 0x16, - 0x48, 0xd6, 0xbd, 0x92, 0x96, 0xf9, 0xd5, 0xff, 0x99, 0x5f, 0xf7, 0x16, 0xf6, 0x9f, 0x27, 0x95, - 0x4d, 0x93, 0x38, 0xa3, 0xd8, 0x07, 0x2c, 0x0f, 0x86, 0x6c, 0xd1, 0x5b, 0x91, 0x9f, 0xd3, 0x78, - 0xf5, 0xc5, 0x01, 0x20, 0xdb, 0x37, 0xcf, 0x4d, 0xee, 0xaf, 0xb0, 0x23, 0xef, 0x09, 0xa2, 0x9b, - 0x31, 0xcd, 0xd6, 0x29, 0x7d, 0x1f, 0x4c, 0x26, 0x82, 0xab, 0xa5, 0x43, 0xfd, 0xc8, 0x26, 0x4a, - 0xfb, 0xb7, 0x15, 0x0e, 0x60, 0x77, 0xf5, 0xe6, 0xff, 0xa4, 0xbe, 0xef, 0xc0, 0x20, 0xb3, 0x31, - 0xc5, 0xbb, 0x50, 0x9e, 0x44, 0xac, 0x7f, 0xa7, 0xaa, 0x91, 0x0a, 0x2f, 0xe5, 0x76, 0x34, 0x66, - 0x34, 0x15, 0x2d, 0xb4, 0x89, 0xd2, 0xdc, 0x37, 0x60, 0x9e, 0x0a, 0x09, 0xff, 0x1f, 0xca, 0xe9, - 0x8c, 0xd7, 0x2a, 0x3f, 0x75, 0xb4, 0x9c, 0x00, 0x07, 0x26, 0xd2, 0xed, 0xfe, 0xa5, 0xc1, 0x86, - 0x4c, 0xe8, 0x2a, 0x99, 0xa5, 0x7d, 0xca, 0x19, 0xbc, 0xa7, 0xf3, 0x6c, 0x1a, 0xf5, 0x69, 0xce, - 0x60, 0xae, 0xf3, 0x64, 0xb2, 0xbb, 0x28, 0x1d, 0xa8, 0x5b, 0xa5, 0x82, 0xbf, 0x07, 0x47, 0x30, - 0xc9, 0x42, 0x36, 0x9f, 0x52, 0xc1, 0xe1, 0x66, 0x63, 0x77, 0x31, 0x54, 0x82, 0x27, 0x16, 0xcc, - 0xa7, 0x94, 0x00, 0x2b, 0xe4, 0xd5, 0x49, 0x34, 0xd6, 0x98, 0xc4, 0x45, 0xff, 0xca, 0x2b, 0xfd, - 0x3b, 0x2e, 0xc8, 0x30, 0x15, 0xca, 0x52, 0xad, 0x92, 0x8e, 0x82, 0xa0, 0x77, 0x60, 0x93, 0xe4, - 0xa9, 0x75, 0x27, 0x00, 0x5d, 0x30, 0x6f, 0xe8, 0x6d, 0x92, 0x52, 0xd5, 0x25, 0x50, 0xaf, 0x18, - 0x49, 0x9e, 0x88, 0xf2, 0xe0, 0x43, 0x28, 0x47, 0xb7, 0x39, 0xd1, 0xab, 0x21, 0xd2, 0xe1, 0x46, - 0x60, 0x91, 0xe4, 0x49, 0xbc, 0x7c, 0xf8, 0x15, 0xc8, 0x0a, 0xc3, 0x38, 0x9a, 0xe4, 0xf4, 0xd9, - 0xc2, 0xd2, 0x8d, 0x26, 0x14, 0xff, 0x00, 0x4e, 0x9a, 0x3c, 0x85, 0x7d, 0x71, 0xbd, 0x1c, 0x43, - 0xa7, 0xb1, 0xb7, 0xd2, 0x9a, 0x3c, 0x39, 0x02, 0x69, 0x2e, 0x66, 0xee, 0x3b, 0x80, 0xd3, 0x11, - 0x1d, 0x0f, 0xd6, 0xba, 0xe4, 0x7f, 0x9c, 0x0e, 0x3a, 0x1e, 0xe4, 0xf8, 0x1b, 0x2a, 0x65, 0x81, - 0x40, 0x94, 0xcf, 0xfd, 0xa4, 0x81, 0xf9, 0x5e, 0xe2, 0x1d, 0x83, 0x21, 0x1a, 0x27, 0xdf, 0xe2, - 0xfd, 0xe5, 0x74, 0x64, 0x84, 0x68, 0x9d, 0x88, 0xe1, 0x8b, 0x68, 0xc8, 0x46, 0xf9, 0x00, 0x08, - 0x99, 0x3f, 0xa9, 0x83, 0x81, 0x7c, 0x52, 0x6d, 0xc2, 0x45, 0xfc, 0x0d, 0xd8, 0xbc, 0x4e, 0xb1, - 0x01, 0x54, 0x6b, 0x77, 0x9f, 0x55, 0x29, 0x80, 0x89, 0x95, 0xe6, 0xcc, 0xfd, 0x08, 0x8e, 0xc8, - 0x4c, 0x1d, 0x2a, 0x8b, 0x43, 0xfb, 0xab, 0x9d, 0xcc, 0x19, 0x20, 0x70, 0x5b, 0xc8, 0xee, 0x2f, - 0xb0, 0xf9, 0x5e, 0x7e, 0x8f, 0xeb, 0xbc, 0x01, 0xc7, 0x2b, 0x1f, 0xce, 0x97, 0x67, 0xe5, 0x27, - 0xd8, 0x2a, 0x90, 0xd5, 0x37, 0x7e, 0x04, 0x65, 0x99, 0x9f, 0xfc, 0xaa, 0xf0, 0xe7, 0x5c, 0x11, - 0x19, 0x70, 0xfc, 0x87, 0x06, 0xb0, 0x60, 0x0f, 0x3b, 0x50, 0xb9, 0xee, 0x9e, 0x77, 0x7b, 0x1f, - 0xba, 0xe8, 0x05, 0xb6, 0xc0, 0x38, 0x0b, 0x3a, 0x6d, 0xa4, 0x61, 0x1b, 0xca, 0x72, 0xad, 0x94, - 0xf8, 0x4e, 0x50, 0x3b, 0x45, 0xe7, 0x0b, 0xa7, 0x58, 0x28, 0x06, 0xae, 0x80, 0x5e, 0xac, 0x0d, - 0xb5, 0x27, 0x4c, 0x0e, 0x48, 0xbc, 0x4b, 0xff, 0xa4, 0xe5, 0xa1, 0x0a, 0x77, 0x14, 0x1b, 0x03, - 0xc0, 0xcc, 0xd7, 0x05, 0x3f, 0xc9, 0x97, 0x0c, 0xf0, 0x7b, 0x7a, 0xc1, 0xcf, 0x1e, 0x41, 0x0e, - 0xb7, 0x91, 0xde, 0x07, 0xb4, 0xc1, 0x6d, 0xa7, 0x1d, 0xcf, 0x6f, 0xa3, 0x97, 0xcd, 0xaf, 0x3e, - 0xbe, 0x7e, 0x1c, 0x31, 0x9a, 0x65, 0xb5, 0x51, 0x52, 0x97, 0x52, 0x7d, 0x98, 0xd4, 0x1f, 0x59, - 0x5d, 0xfc, 0xa1, 0xd4, 0x17, 0x55, 0xde, 0x98, 0xc2, 0xf2, 0xed, 0xdf, 0x01, 0x00, 0x00, 0xff, - 0xff, 0xda, 0xbf, 0xad, 0x34, 0xf0, 0x08, 0x00, 0x00, +func init() { proto.RegisterFile("binlogdata.proto", fileDescriptor_binlogdata_6d214635eb8c538c) } + +var fileDescriptor_binlogdata_6d214635eb8c538c = []byte{ + // 1184 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x56, 0x5b, 0x6e, 0xdb, 0x56, + 0x13, 0x8e, 0x44, 0x8a, 0x12, 0x87, 0x8e, 0x4d, 0x1f, 0x5f, 0x7e, 0xc1, 0xf8, 0x03, 0x18, 0x44, + 0xdb, 0xb8, 0x06, 0x2a, 0xa7, 0xea, 0xed, 0xa9, 0x2d, 0x2c, 0x91, 0x71, 0x95, 0xd0, 0x92, 0x73, + 0xcc, 0x24, 0x45, 0x5e, 0x08, 0x9a, 0x3c, 0xb2, 0x09, 0x53, 0xa4, 0x4c, 0x1e, 0xcb, 0xd5, 0x0a, + 0xba, 0x80, 0xbe, 0x76, 0x03, 0xed, 0x42, 0xba, 0x92, 0x76, 0x1f, 0xc5, 0xb9, 0x90, 0x92, 0x1c, + 0xa0, 0x71, 0x1f, 0xfa, 0x36, 0xf7, 0x33, 0xf3, 0xcd, 0x70, 0x86, 0x60, 0x5e, 0xc4, 0x69, 0x92, + 0x5d, 0x46, 0x01, 0x0d, 0x3a, 0xd3, 0x3c, 0xa3, 0x19, 0x82, 0x85, 0x64, 0xcf, 0x98, 0xd1, 0x7c, + 0x1a, 0x0a, 0xc5, 0x9e, 0x71, 0x73, 0x4b, 0xf2, 0xb9, 0x64, 0xd6, 0x69, 0x36, 0xcd, 0x16, 0x5e, + 0xd6, 0x29, 0x34, 0xfb, 0x57, 0x41, 0x5e, 0x10, 0x8a, 0x76, 0x41, 0x0b, 0x93, 0x98, 0xa4, 0xb4, + 0x5d, 0xdb, 0xaf, 0x1d, 0x34, 0xb0, 0xe4, 0x10, 0x02, 0x35, 0xcc, 0xd2, 0xb4, 0x5d, 0xe7, 0x52, + 0x4e, 0x33, 0xdb, 0x82, 0xe4, 0x33, 0x92, 0xb7, 0x15, 0x61, 0x2b, 0x38, 0xeb, 0x2f, 0x05, 0x36, + 0x7b, 0x3c, 0x0f, 0x2f, 0x0f, 0xd2, 0x22, 0x08, 0x69, 0x9c, 0xa5, 0xe8, 0x04, 0xa0, 0xa0, 0x01, + 0x25, 0x13, 0x92, 0xd2, 0xa2, 0x5d, 0xdb, 0x57, 0x0e, 0x8c, 0xee, 0xd3, 0xce, 0x52, 0x05, 0xef, + 0xb9, 0x74, 0xce, 0x4b, 0x7b, 0xbc, 0xe4, 0x8a, 0xba, 0x60, 0x90, 0x19, 0x49, 0xa9, 0x4f, 0xb3, + 0x6b, 0x92, 0xb6, 0xd5, 0xfd, 0xda, 0x81, 0xd1, 0xdd, 0xec, 0x88, 0x02, 0x1d, 0xa6, 0xf1, 0x98, + 0x02, 0x03, 0xa9, 0xe8, 0xbd, 0x3f, 0xea, 0xa0, 0x57, 0xd1, 0x90, 0x0b, 0xad, 0x30, 0xa0, 0xe4, + 0x32, 0xcb, 0xe7, 0xbc, 0xcc, 0xf5, 0xee, 0xb3, 0x07, 0x26, 0xd2, 0xe9, 0x4b, 0x3f, 0x5c, 0x45, + 0x40, 0x9f, 0x41, 0x33, 0x14, 0xe8, 0x71, 0x74, 0x8c, 0xee, 0xd6, 0x72, 0x30, 0x09, 0x2c, 0x2e, + 0x6d, 0x90, 0x09, 0x4a, 0x71, 0x93, 0x70, 0xc8, 0xd6, 0x30, 0x23, 0xad, 0xdf, 0x6a, 0xd0, 0x2a, + 0xe3, 0xa2, 0x2d, 0xd8, 0xe8, 0xb9, 0xfe, 0xeb, 0x21, 0x76, 0xfa, 0xa3, 0x93, 0xe1, 0xe0, 0x9d, + 0x63, 0x9b, 0x8f, 0xd0, 0x1a, 0xb4, 0x7a, 0xae, 0xdf, 0x73, 0x4e, 0x06, 0x43, 0xb3, 0x86, 0x1e, + 0x83, 0xde, 0x73, 0xfd, 0xfe, 0xe8, 0xf4, 0x74, 0xe0, 0x99, 0x75, 0xb4, 0x01, 0x46, 0xcf, 0xf5, + 0xf1, 0xc8, 0x75, 0x7b, 0xc7, 0xfd, 0x97, 0xa6, 0x82, 0x76, 0x60, 0xb3, 0xe7, 0xfa, 0xf6, 0xa9, + 0xeb, 0xdb, 0xce, 0x19, 0x76, 0xfa, 0xc7, 0x9e, 0x63, 0x9b, 0x2a, 0x02, 0xd0, 0x98, 0xd8, 0x76, + 0xcd, 0x86, 0xa4, 0xcf, 0x1d, 0xcf, 0xd4, 0x64, 0xb8, 0xc1, 0xf0, 0xdc, 0xc1, 0x9e, 0xd9, 0x94, + 0xec, 0xeb, 0x33, 0xfb, 0xd8, 0x73, 0xcc, 0x96, 0x64, 0x6d, 0xc7, 0x75, 0x3c, 0xc7, 0xd4, 0x5f, + 0xa8, 0xad, 0xba, 0xa9, 0xbc, 0x50, 0x5b, 0x8a, 0xa9, 0x5a, 0xbf, 0xd4, 0x60, 0xe7, 0x9c, 0xe6, + 0x24, 0x98, 0xbc, 0x24, 0x73, 0x1c, 0xa4, 0x97, 0x04, 0x93, 0x9b, 0x5b, 0x52, 0x50, 0xb4, 0x07, + 0xad, 0x69, 0x56, 0xc4, 0x0c, 0x3b, 0x0e, 0xb0, 0x8e, 0x2b, 0x1e, 0x1d, 0x81, 0x7e, 0x4d, 0xe6, + 0x7e, 0xce, 0xec, 0x25, 0x60, 0xa8, 0x53, 0x0d, 0x64, 0x15, 0xa9, 0x75, 0x2d, 0xa9, 0x65, 0x7c, + 0x95, 0x0f, 0xe3, 0x6b, 0x8d, 0x61, 0xf7, 0x7e, 0x52, 0xc5, 0x34, 0x4b, 0x0b, 0x82, 0x5c, 0x40, + 0xc2, 0xd1, 0xa7, 0x8b, 0xde, 0xf2, 0xfc, 0x8c, 0xee, 0x93, 0x7f, 0x1c, 0x00, 0xbc, 0x79, 0x71, + 0x5f, 0x64, 0xfd, 0x04, 0x5b, 0xe2, 0x1d, 0x2f, 0xb8, 0x48, 0x48, 0xf1, 0x90, 0xd2, 0x77, 0x41, + 0xa3, 0xdc, 0xb8, 0x5d, 0xdf, 0x57, 0x0e, 0x74, 0x2c, 0xb9, 0x7f, 0x5b, 0x61, 0x04, 0xdb, 0xab, + 0x2f, 0xff, 0x27, 0xf5, 0x7d, 0x09, 0x2a, 0xbe, 0x4d, 0x08, 0xda, 0x86, 0xc6, 0x24, 0xa0, 0xe1, + 0x95, 0xac, 0x46, 0x30, 0xac, 0x94, 0x71, 0x9c, 0x50, 0x92, 0xf3, 0x16, 0xea, 0x58, 0x72, 0xd6, + 0x33, 0xd0, 0x9e, 0x73, 0x0a, 0x7d, 0x02, 0x8d, 0xfc, 0x96, 0xd5, 0x2a, 0x3e, 0x75, 0x73, 0x39, + 0x01, 0x16, 0x18, 0x0b, 0xb5, 0xf5, 0x6b, 0x1d, 0xd6, 0x44, 0x42, 0xe7, 0xd9, 0x6d, 0x1e, 0x12, + 0x86, 0xe0, 0x35, 0x99, 0x17, 0xd3, 0x20, 0x24, 0x25, 0x82, 0x25, 0xcf, 0x92, 0x29, 0xae, 0x82, + 0x3c, 0x92, 0xaf, 0x0a, 0x06, 0x7d, 0x05, 0x06, 0x47, 0x92, 0xfa, 0x74, 0x3e, 0x25, 0x1c, 0xc3, + 0xf5, 0xee, 0xf6, 0x62, 0xa8, 0x38, 0x4e, 0xd4, 0x9b, 0x4f, 0x09, 0x06, 0x5a, 0xd1, 0xab, 0x93, + 0xa8, 0x3e, 0x60, 0x12, 0x17, 0xfd, 0x6b, 0xac, 0xf4, 0xef, 0xb0, 0x02, 0x43, 0x93, 0x51, 0x96, + 0x6a, 0x15, 0x70, 0x94, 0x00, 0xa1, 0x0e, 0x68, 0x59, 0xea, 0x47, 0x51, 0xd2, 0x6e, 0xf2, 0x34, + 0xff, 0xb7, 0x6c, 0x3b, 0x4a, 0x6d, 0xdb, 0x3d, 0x16, 0x2d, 0x69, 0x64, 0xa9, 0x1d, 0x25, 0xd6, + 0x2b, 0xd0, 0x71, 0x76, 0xd7, 0xbf, 0xe2, 0x09, 0x58, 0xa0, 0x5d, 0x90, 0x71, 0x96, 0x13, 0xd9, + 0x55, 0x90, 0x5b, 0x0f, 0x67, 0x77, 0x58, 0x6a, 0xd0, 0x3e, 0x34, 0x82, 0x71, 0xd9, 0x98, 0x55, + 0x13, 0xa1, 0xb0, 0x02, 0x68, 0xe1, 0xec, 0x8e, 0x6f, 0x4a, 0xf4, 0x04, 0x04, 0x22, 0x7e, 0x1a, + 0x4c, 0x4a, 0xb8, 0x75, 0x2e, 0x19, 0x06, 0x13, 0x82, 0xbe, 0x06, 0x23, 0xcf, 0xee, 0xfc, 0x90, + 0x3f, 0x2f, 0xc6, 0xd6, 0xe8, 0xee, 0xac, 0xb4, 0xb2, 0x4c, 0x0e, 0x43, 0x5e, 0x92, 0x85, 0xf5, + 0x0a, 0xe0, 0x79, 0x4c, 0x92, 0xe8, 0x41, 0x8f, 0x7c, 0xc4, 0xe0, 0x23, 0x49, 0x54, 0xc6, 0x5f, + 0x93, 0x29, 0xf3, 0x08, 0x58, 0xea, 0xac, 0x3f, 0x6b, 0xa0, 0xbd, 0x11, 0xf1, 0x0e, 0x41, 0xe5, + 0x8d, 0x16, 0xbb, 0x7b, 0x77, 0x39, 0x1d, 0x61, 0xc1, 0x5b, 0xcd, 0x6d, 0xd0, 0xff, 0x41, 0xa7, + 0xf1, 0x84, 0x14, 0x34, 0x98, 0x4c, 0x39, 0x24, 0x0a, 0x5e, 0x08, 0xd8, 0x59, 0xbb, 0xa4, 0x71, + 0xc4, 0x47, 0x46, 0xc7, 0x9c, 0x66, 0x0b, 0x9a, 0xb5, 0x47, 0xe5, 0x22, 0x46, 0xa2, 0xcf, 0x41, + 0x67, 0x28, 0xf0, 0x7b, 0xd2, 0x6e, 0x70, 0x58, 0xb7, 0xef, 0x61, 0xc0, 0x9f, 0xc5, 0xad, 0xbc, + 0xc4, 0xf5, 0x1b, 0x30, 0x78, 0xde, 0xd2, 0x49, 0xcc, 0xc5, 0xee, 0xea, 0x5c, 0x94, 0xf8, 0x60, + 0x18, 0x57, 0xb4, 0xf5, 0x73, 0x1d, 0xd6, 0xdf, 0x88, 0xcf, 0xbb, 0x5c, 0x29, 0xdf, 0xc3, 0x16, + 0x19, 0x8f, 0x49, 0x48, 0xe3, 0x19, 0xf1, 0xc3, 0x20, 0x49, 0x48, 0xee, 0xc7, 0x91, 0x1c, 0x81, + 0x8d, 0x8e, 0x38, 0xf3, 0x7d, 0x2e, 0x1f, 0xd8, 0x78, 0xb3, 0xb2, 0x95, 0xa2, 0x08, 0x39, 0xb0, + 0x15, 0x4f, 0x26, 0x24, 0x8a, 0x03, 0xba, 0x1c, 0x40, 0x0c, 0xc8, 0x8e, 0x44, 0xfb, 0x8d, 0x77, + 0x12, 0x50, 0xb2, 0x08, 0x53, 0x79, 0x54, 0x61, 0x3e, 0x66, 0xe3, 0x9f, 0x5f, 0x56, 0x5b, 0xea, + 0xb1, 0xf4, 0xf4, 0xb8, 0x10, 0x4b, 0xe5, 0xca, 0x06, 0x54, 0xef, 0x6d, 0xc0, 0xc5, 0x97, 0xd2, + 0xf8, 0xd0, 0x97, 0x62, 0x7d, 0x0b, 0x1b, 0x15, 0x10, 0x72, 0xc3, 0x1d, 0x82, 0xc6, 0xf1, 0x2c, + 0x97, 0x0a, 0x7a, 0xbf, 0xf5, 0x58, 0x5a, 0x1c, 0x7e, 0x07, 0xc6, 0xd2, 0xe7, 0xc4, 0x2e, 0xde, + 0xe0, 0x64, 0x38, 0xc2, 0x8e, 0xf9, 0x08, 0xb5, 0x40, 0x3d, 0xf7, 0x46, 0x67, 0x66, 0x8d, 0x51, + 0xce, 0x8f, 0x4e, 0x5f, 0x5c, 0x51, 0x46, 0xf9, 0xd2, 0x48, 0x39, 0xfc, 0xbd, 0x06, 0xb0, 0x98, + 0x26, 0x64, 0x40, 0xf3, 0xf5, 0xf0, 0xe5, 0x70, 0xf4, 0x76, 0x28, 0x02, 0x9c, 0x78, 0x03, 0xdb, + 0xac, 0x21, 0x1d, 0x1a, 0xe2, 0x2c, 0xd7, 0xd9, 0x0b, 0xf2, 0x26, 0x2b, 0xec, 0x60, 0x57, 0x07, + 0x59, 0x45, 0x4d, 0x50, 0xaa, 0xb3, 0x2b, 0xef, 0xac, 0xc6, 0x02, 0x62, 0xe7, 0xcc, 0x3d, 0xee, + 0x3b, 0x66, 0x93, 0x29, 0xaa, 0x8b, 0x0b, 0xa0, 0x95, 0xe7, 0x96, 0x79, 0xb2, 0x23, 0x0d, 0xec, + 0x9d, 0x91, 0xf7, 0x83, 0x83, 0x4d, 0x83, 0xc9, 0xf0, 0xe8, 0xad, 0xb9, 0xc6, 0x64, 0xcf, 0x07, + 0x8e, 0x6b, 0x9b, 0x8f, 0x7b, 0x9f, 0xbe, 0x7b, 0x3a, 0x8b, 0x29, 0x29, 0x8a, 0x4e, 0x9c, 0x1d, + 0x09, 0xea, 0xe8, 0x32, 0x3b, 0x9a, 0xd1, 0x23, 0xfe, 0x87, 0x77, 0xb4, 0x80, 0xe9, 0x42, 0xe3, + 0x92, 0x2f, 0xfe, 0x0e, 0x00, 0x00, 0xff, 0xff, 0x68, 0xbd, 0x20, 0x05, 0x3d, 0x0a, 0x00, 0x00, } diff --git a/go/vt/proto/queryservice/queryservice.pb.go b/go/vt/proto/queryservice/queryservice.pb.go index dc4bf2f2ba8..7a0bce4f401 100644 --- a/go/vt/proto/queryservice/queryservice.pb.go +++ b/go/vt/proto/queryservice/queryservice.pb.go @@ -6,6 +6,7 @@ package queryservice // import "vitess.io/vitess/go/vt/proto/queryservice" import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" +import binlogdata "vitess.io/vitess/go/vt/proto/binlogdata" import query "vitess.io/vitess/go/vt/proto/query" import ( @@ -85,6 +86,8 @@ type QueryClient interface { StreamHealth(ctx context.Context, in *query.StreamHealthRequest, opts ...grpc.CallOption) (Query_StreamHealthClient, error) // UpdateStream asks the server to return a stream of the updates that have been applied to its database. UpdateStream(ctx context.Context, in *query.UpdateStreamRequest, opts ...grpc.CallOption) (Query_UpdateStreamClient, error) + // VStream streams vreplication events. + VStream(ctx context.Context, in *binlogdata.VStreamRequest, opts ...grpc.CallOption) (Query_VStreamClient, error) } type queryClient struct { @@ -376,6 +379,38 @@ func (x *queryUpdateStreamClient) Recv() (*query.UpdateStreamResponse, error) { return m, nil } +func (c *queryClient) VStream(ctx context.Context, in *binlogdata.VStreamRequest, opts ...grpc.CallOption) (Query_VStreamClient, error) { + stream, err := c.cc.NewStream(ctx, &_Query_serviceDesc.Streams[4], "/queryservice.Query/VStream", opts...) + if err != nil { + return nil, err + } + x := &queryVStreamClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type Query_VStreamClient interface { + Recv() (*binlogdata.VStreamResponse, error) + grpc.ClientStream +} + +type queryVStreamClient struct { + grpc.ClientStream +} + +func (x *queryVStreamClient) Recv() (*binlogdata.VStreamResponse, error) { + m := new(binlogdata.VStreamResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + // QueryServer is the server API for Query service. type QueryServer interface { // Execute executes the specified SQL query (might be in a @@ -427,6 +462,8 @@ type QueryServer interface { StreamHealth(*query.StreamHealthRequest, Query_StreamHealthServer) error // UpdateStream asks the server to return a stream of the updates that have been applied to its database. UpdateStream(*query.UpdateStreamRequest, Query_UpdateStreamServer) error + // VStream streams vreplication events. + VStream(*binlogdata.VStreamRequest, Query_VStreamServer) error } func RegisterQueryServer(s *grpc.Server, srv QueryServer) { @@ -823,6 +860,27 @@ func (x *queryUpdateStreamServer) Send(m *query.UpdateStreamResponse) error { return x.ServerStream.SendMsg(m) } +func _Query_VStream_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(binlogdata.VStreamRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(QueryServer).VStream(m, &queryVStreamServer{stream}) +} + +type Query_VStreamServer interface { + Send(*binlogdata.VStreamResponse) error + grpc.ServerStream +} + +type queryVStreamServer struct { + grpc.ServerStream +} + +func (x *queryVStreamServer) Send(m *binlogdata.VStreamResponse) error { + return x.ServerStream.SendMsg(m) +} + var _Query_serviceDesc = grpc.ServiceDesc{ ServiceName: "queryservice.Query", HandlerType: (*QueryServer)(nil), @@ -917,45 +975,51 @@ var _Query_serviceDesc = grpc.ServiceDesc{ Handler: _Query_UpdateStream_Handler, ServerStreams: true, }, + { + StreamName: "VStream", + Handler: _Query_VStream_Handler, + ServerStreams: true, + }, }, Metadata: "queryservice.proto", } -func init() { proto.RegisterFile("queryservice.proto", fileDescriptor_queryservice_bb9da33c43234b51) } +func init() { proto.RegisterFile("queryservice.proto", fileDescriptor_queryservice_17509881eb07629d) } -var fileDescriptor_queryservice_bb9da33c43234b51 = []byte{ - // 519 bytes of a gzipped FileDescriptorProto +var fileDescriptor_queryservice_17509881eb07629d = []byte{ + // 544 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x95, 0xdf, 0x6b, 0xd4, 0x40, - 0x10, 0xc7, 0xf5, 0xa1, 0xad, 0x4c, 0xe3, 0xaf, 0xad, 0x55, 0x9b, 0xd6, 0xb6, 0xf6, 0x4d, 0x84, - 0x46, 0x54, 0x10, 0x0a, 0x3e, 0xf4, 0x82, 0xa2, 0x14, 0x7f, 0xdd, 0x59, 0x10, 0x1f, 0x84, 0x6d, - 0x6e, 0x38, 0x43, 0x73, 0x49, 0xba, 0xbb, 0x77, 0xe8, 0x5f, 0xe4, 0xbf, 0x29, 0x66, 0x33, 0x93, - 0xdd, 0xbd, 0xc4, 0xb7, 0xce, 0xf7, 0x3b, 0xf3, 0x61, 0x6e, 0xa7, 0x33, 0x01, 0x71, 0xb5, 0x40, - 0xf5, 0x5b, 0xa3, 0x5a, 0xe6, 0x19, 0x1e, 0xd7, 0xaa, 0x32, 0x95, 0x88, 0x5c, 0x2d, 0xde, 0x6c, - 0x22, 0x6b, 0x3d, 0xff, 0x13, 0xc1, 0xda, 0x97, 0x7f, 0xb1, 0x38, 0x81, 0x8d, 0x37, 0xbf, 0x30, - 0x5b, 0x18, 0x14, 0xdb, 0xc7, 0x36, 0xa5, 0x8d, 0xc7, 0x78, 0xb5, 0x40, 0x6d, 0xe2, 0xfb, 0xa1, - 0xac, 0xeb, 0xaa, 0xd4, 0x78, 0x74, 0x4d, 0xbc, 0x87, 0xa8, 0x15, 0x47, 0xd2, 0x64, 0x3f, 0x45, - 0xec, 0x67, 0x36, 0x22, 0x51, 0x76, 0x7b, 0x3d, 0x46, 0x7d, 0x84, 0x9b, 0x13, 0xa3, 0x50, 0xce, - 0xa9, 0x19, 0xca, 0xf7, 0x54, 0x82, 0xed, 0xf5, 0x9b, 0x44, 0x7b, 0x76, 0x5d, 0xbc, 0x84, 0xb5, - 0x11, 0xce, 0xf2, 0x52, 0x6c, 0xb5, 0xa9, 0x4d, 0x44, 0xf5, 0xf7, 0x7c, 0x91, 0xbb, 0x78, 0x05, - 0xeb, 0x69, 0x35, 0x9f, 0xe7, 0x46, 0x50, 0x86, 0x0d, 0xa9, 0x6e, 0x3b, 0x50, 0xb9, 0xf0, 0x35, - 0xdc, 0x18, 0x57, 0x45, 0x71, 0x21, 0xb3, 0x4b, 0x41, 0xef, 0x45, 0x02, 0x15, 0x3f, 0x58, 0xd1, - 0xb9, 0xfc, 0x04, 0x36, 0x3e, 0x2b, 0xac, 0xa5, 0xea, 0x86, 0xd0, 0xc6, 0xe1, 0x10, 0x58, 0xe6, - 0xda, 0x4f, 0x70, 0xcb, 0xb6, 0xd3, 0x5a, 0x53, 0xb1, 0xe7, 0x75, 0x49, 0x32, 0x91, 0x1e, 0x0d, - 0xb8, 0x0c, 0x3c, 0x87, 0x3b, 0xd4, 0x22, 0x23, 0xf7, 0x83, 0xde, 0x43, 0xe8, 0xc1, 0xa0, 0xcf, - 0xd8, 0x6f, 0x70, 0x37, 0x55, 0x28, 0x0d, 0x7e, 0x55, 0xb2, 0xd4, 0x32, 0x33, 0x79, 0x55, 0x0a, - 0xaa, 0x5b, 0x71, 0x08, 0x7c, 0x38, 0x9c, 0xc0, 0xe4, 0xb7, 0xb0, 0x39, 0x31, 0x52, 0x99, 0x76, - 0x74, 0x3b, 0xfc, 0xcf, 0xc1, 0x1a, 0xd1, 0xe2, 0x3e, 0xcb, 0xe3, 0xa0, 0xe1, 0x39, 0x32, 0xa7, - 0xd3, 0x56, 0x38, 0xae, 0xc5, 0x9c, 0x1f, 0xb0, 0x95, 0x56, 0x65, 0x56, 0x2c, 0xa6, 0xde, 0x6f, - 0x7d, 0xcc, 0x0f, 0xbf, 0xe2, 0x11, 0xf7, 0xe8, 0x7f, 0x29, 0xcc, 0x1f, 0xc3, 0xed, 0x31, 0xca, - 0xa9, 0xcb, 0xa6, 0xa1, 0x06, 0x3a, 0x71, 0xf7, 0x87, 0x6c, 0x77, 0x95, 0x9b, 0x65, 0xa0, 0xf5, - 0x8b, 0xdd, 0x0d, 0x09, 0xb6, 0x6f, 0xb7, 0xd7, 0x73, 0x07, 0xed, 0x3a, 0xf6, 0x34, 0x1c, 0xf4, - 0xd4, 0x78, 0xf7, 0xe1, 0x70, 0x38, 0xc1, 0x3d, 0x12, 0x1f, 0x50, 0x6b, 0x39, 0x43, 0xbb, 0xf8, - 0x7c, 0x24, 0x3c, 0x35, 0x3c, 0x12, 0x81, 0xe9, 0x1c, 0x89, 0x14, 0xa0, 0x35, 0x4f, 0xb3, 0x4b, - 0xf1, 0xd0, 0xcf, 0x3f, 0xed, 0xc6, 0xbd, 0xd3, 0xe3, 0x70, 0x53, 0x29, 0xc0, 0xa4, 0x2e, 0x72, - 0x63, 0xcf, 0x29, 0x41, 0x3a, 0x29, 0x84, 0xb8, 0x0e, 0x43, 0xce, 0x20, 0xb2, 0xfd, 0xbd, 0x43, - 0x59, 0x98, 0xee, 0x92, 0xba, 0x62, 0xf8, 0xfc, 0xbe, 0xe7, 0xfc, 0xac, 0x33, 0x88, 0xce, 0xeb, - 0xa9, 0x34, 0xf4, 0x4a, 0x04, 0x73, 0xc5, 0x10, 0xe6, 0x7b, 0x1d, 0x6c, 0xf4, 0xf4, 0xfb, 0x93, - 0x65, 0x6e, 0x50, 0xeb, 0xe3, 0xbc, 0x4a, 0xec, 0x5f, 0xc9, 0xac, 0x4a, 0x96, 0x26, 0x69, 0xbe, - 0x24, 0x89, 0xfb, 0x8d, 0xb9, 0x58, 0x6f, 0xb4, 0x17, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x51, - 0x5a, 0xbc, 0xc0, 0x8e, 0x06, 0x00, 0x00, + 0x10, 0xc7, 0xf5, 0xa1, 0xad, 0x4c, 0x4f, 0xad, 0x5b, 0xab, 0x36, 0xad, 0x6d, 0xed, 0x9b, 0x08, + 0x17, 0x51, 0x41, 0x28, 0xf8, 0xd0, 0x0b, 0x16, 0xa5, 0xf8, 0xeb, 0xce, 0x16, 0xf1, 0x41, 0xd8, + 0x4b, 0x86, 0x33, 0x34, 0x97, 0x4d, 0x93, 0xbd, 0x43, 0xff, 0x6a, 0xff, 0x05, 0x31, 0x9b, 0x99, + 0xec, 0xee, 0x25, 0xbe, 0xdd, 0x7e, 0xbf, 0x33, 0x1f, 0x26, 0x3b, 0x37, 0xb3, 0x20, 0xae, 0x17, + 0x58, 0xfe, 0xae, 0xb0, 0x5c, 0xa6, 0x31, 0x0e, 0x8b, 0x52, 0x69, 0x25, 0x06, 0xb6, 0x16, 0x6c, + 0xd6, 0x27, 0x63, 0x05, 0x5b, 0xd3, 0x34, 0xcf, 0xd4, 0x2c, 0x91, 0x5a, 0x1a, 0xe5, 0xc5, 0x9f, + 0x01, 0xac, 0x7d, 0xf9, 0x17, 0x21, 0x4e, 0x60, 0xe3, 0xed, 0x2f, 0x8c, 0x17, 0x1a, 0xc5, 0xce, + 0xd0, 0x24, 0x35, 0xe7, 0x31, 0x5e, 0x2f, 0xb0, 0xd2, 0xc1, 0x03, 0x5f, 0xae, 0x0a, 0x95, 0x57, + 0x78, 0x7c, 0x43, 0xbc, 0x87, 0x41, 0x23, 0x8e, 0xa4, 0x8e, 0x7f, 0x8a, 0xc0, 0x8d, 0xac, 0x45, + 0xa2, 0xec, 0x75, 0x7a, 0x8c, 0xfa, 0x08, 0xb7, 0x27, 0xba, 0x44, 0x39, 0xa7, 0x62, 0x28, 0xde, + 0x51, 0x09, 0xb6, 0xdf, 0x6d, 0x12, 0xed, 0xf9, 0x4d, 0xf1, 0x0a, 0xd6, 0x46, 0x38, 0x4b, 0x73, + 0xb1, 0xdd, 0x84, 0xd6, 0x27, 0xca, 0xbf, 0xef, 0x8a, 0x5c, 0xc5, 0x6b, 0x58, 0x8f, 0xd4, 0x7c, + 0x9e, 0x6a, 0x41, 0x11, 0xe6, 0x48, 0x79, 0x3b, 0x9e, 0xca, 0x89, 0x6f, 0xe0, 0xd6, 0x58, 0x65, + 0xd9, 0x54, 0xc6, 0x57, 0x82, 0xee, 0x8b, 0x04, 0x4a, 0x7e, 0xb8, 0xa2, 0x73, 0xfa, 0x09, 0x6c, + 0x7c, 0x2e, 0xb1, 0x90, 0x65, 0xdb, 0x84, 0xe6, 0xec, 0x37, 0x81, 0x65, 0xce, 0xfd, 0x04, 0x77, + 0x4c, 0x39, 0x8d, 0x95, 0x88, 0x7d, 0xa7, 0x4a, 0x92, 0x89, 0xf4, 0xb8, 0xc7, 0x65, 0xe0, 0x05, + 0x6c, 0x51, 0x89, 0x8c, 0x3c, 0xf0, 0x6a, 0xf7, 0xa1, 0x87, 0xbd, 0x3e, 0x63, 0xbf, 0xc1, 0xbd, + 0xa8, 0x44, 0xa9, 0xf1, 0x6b, 0x29, 0xf3, 0x4a, 0xc6, 0x3a, 0x55, 0xb9, 0xa0, 0xbc, 0x15, 0x87, + 0xc0, 0x47, 0xfd, 0x01, 0x4c, 0x3e, 0x83, 0xcd, 0x89, 0x96, 0xa5, 0x6e, 0x5a, 0xb7, 0xcb, 0x7f, + 0x0e, 0xd6, 0x88, 0x16, 0x74, 0x59, 0x0e, 0x07, 0x35, 0xf7, 0x91, 0x39, 0xad, 0xb6, 0xc2, 0xb1, + 0x2d, 0xe6, 0xfc, 0x80, 0xed, 0x48, 0xe5, 0x71, 0xb6, 0x48, 0x9c, 0x6f, 0x7d, 0xc2, 0x17, 0xbf, + 0xe2, 0x11, 0xf7, 0xf8, 0x7f, 0x21, 0xcc, 0x1f, 0xc3, 0xdd, 0x31, 0xca, 0xc4, 0x66, 0x53, 0x53, + 0x3d, 0x9d, 0xb8, 0x07, 0x7d, 0xb6, 0x3d, 0xca, 0xf5, 0x30, 0xd0, 0xf8, 0x05, 0xf6, 0x84, 0x78, + 0xd3, 0xb7, 0xd7, 0xe9, 0xd9, 0x8d, 0xb6, 0x1d, 0xb3, 0x1a, 0x0e, 0x3b, 0x72, 0x9c, 0xfd, 0x70, + 0xd4, 0x1f, 0x60, 0x2f, 0x89, 0x0f, 0x58, 0x55, 0x72, 0x86, 0x66, 0xf0, 0x79, 0x49, 0x38, 0xaa, + 0xbf, 0x24, 0x3c, 0xd3, 0x5a, 0x12, 0x11, 0x40, 0x63, 0x9e, 0xc6, 0x57, 0xe2, 0x91, 0x1b, 0x7f, + 0xda, 0xb6, 0x7b, 0xb7, 0xc3, 0xe1, 0xa2, 0x22, 0x80, 0x49, 0x91, 0xa5, 0xda, 0xac, 0x53, 0x82, + 0xb4, 0x92, 0x0f, 0xb1, 0x1d, 0x86, 0x9c, 0xc3, 0xc0, 0xd4, 0xf7, 0x0e, 0x65, 0xa6, 0xdb, 0x4d, + 0x6a, 0x8b, 0xfe, 0xf5, 0xbb, 0x9e, 0xf5, 0x59, 0xe7, 0x30, 0xb8, 0x28, 0x12, 0xa9, 0xe9, 0x96, + 0x08, 0x66, 0x8b, 0x3e, 0xcc, 0xf5, 0x2c, 0xd8, 0x19, 0x6c, 0x5c, 0x32, 0xc7, 0x7a, 0x47, 0x2e, + 0x7d, 0x4e, 0x97, 0xd7, 0x72, 0x46, 0xcf, 0xbe, 0x3f, 0x5d, 0xa6, 0x1a, 0xab, 0x6a, 0x98, 0xaa, + 0xd0, 0xfc, 0x0a, 0x67, 0x2a, 0x5c, 0xea, 0xb0, 0x7e, 0x91, 0x42, 0xfb, 0xf5, 0x9a, 0xae, 0xd7, + 0xda, 0xcb, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xbd, 0x47, 0x8e, 0x80, 0xe8, 0x06, 0x00, 0x00, } diff --git a/go/vt/vtcombo/tablet_map.go b/go/vt/vtcombo/tablet_map.go index 0e6c79d254a..849fbb979af 100644 --- a/go/vt/vtcombo/tablet_map.go +++ b/go/vt/vtcombo/tablet_map.go @@ -47,6 +47,7 @@ import ( "vitess.io/vitess/go/vt/vttablet/tmclient" "vitess.io/vitess/go/vt/wrangler" + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" querypb "vitess.io/vitess/go/vt/proto/query" replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" @@ -472,6 +473,12 @@ func (itc *internalTabletConn) UpdateStream(ctx context.Context, target *querypb return tabletconn.ErrorFromGRPC(vterrors.ToGRPC(err)) } +// VStream is part of queryservice.QueryService. +func (itc *internalTabletConn) VStream(ctx context.Context, target *querypb.Target, startPos string, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { + err := itc.tablet.qsc.QueryService().VStream(ctx, target, startPos, filter, send) + return tabletconn.ErrorFromGRPC(vterrors.ToGRPC(err)) +} + // // TabletManagerClient implementation // diff --git a/go/vt/vttablet/endtoend/compatibility_test.go b/go/vt/vttablet/endtoend/compatibility_test.go index 6edc6628e4b..1bf90904633 100644 --- a/go/vt/vttablet/endtoend/compatibility_test.go +++ b/go/vt/vttablet/endtoend/compatibility_test.go @@ -794,7 +794,14 @@ func TestJSONType(t *testing.T) { }, } if !reflect.DeepEqual(*qr, want) { - t.Errorf("Execute: \n%v, want \n%v", prettyPrint(*qr), prettyPrint(want)) + // MariaDB 10.3 has different behavior. + want2 := want.Copy() + want2.Fields[1].Type = sqltypes.Blob + want2.Fields[1].Charset = 33 + want2.Rows[0][1] = sqltypes.TestValue(sqltypes.Blob, "{\"foo\": \"bar\"}") + if !reflect.DeepEqual(*qr, *want2) { + t.Errorf("Execute:\n%v, want\n%v or\n%v", prettyPrint(*qr), prettyPrint(want), prettyPrint(*want2)) + } } } diff --git a/go/vt/vttablet/endtoend/misc_test.go b/go/vt/vttablet/endtoend/misc_test.go index 5e961a9ae32..81fb606ba0a 100644 --- a/go/vt/vttablet/endtoend/misc_test.go +++ b/go/vt/vttablet/endtoend/misc_test.go @@ -364,7 +364,13 @@ func TestBindInSelect(t *testing.T) { }, } if !qr.Equal(want) { - t.Errorf("Execute: \n%#v, want \n%#v", prettyPrint(*qr), prettyPrint(*want)) + // MariaDB 10.3 has different behavior. + want2 := want.Copy() + want2.Fields[0].Type = sqltypes.Int32 + want2.Rows[0][0] = sqltypes.NewInt32(1) + if !qr.Equal(want2) { + t.Errorf("Execute:\n%v, want\n%v or\n%v", prettyPrint(*qr), prettyPrint(*want), prettyPrint(*want2)) + } } // String bind var. @@ -382,7 +388,6 @@ func TestBindInSelect(t *testing.T) { Type: sqltypes.VarChar, ColumnLength: 12, Charset: 33, - Decimals: 31, Flags: 1, }}, RowsAffected: 1, @@ -392,6 +397,8 @@ func TestBindInSelect(t *testing.T) { }, }, } + // MariaDB 10.3 has different behavior. + qr.Fields[0].Decimals = 0 if !qr.Equal(want) { t.Errorf("Execute: \n%#v, want \n%#v", prettyPrint(*qr), prettyPrint(*want)) } @@ -411,7 +418,6 @@ func TestBindInSelect(t *testing.T) { Type: sqltypes.VarChar, ColumnLength: 6, Charset: 33, - Decimals: 31, Flags: 1, }}, RowsAffected: 1, @@ -421,6 +427,8 @@ func TestBindInSelect(t *testing.T) { }, }, } + // MariaDB 10.3 has different behavior. + qr.Fields[0].Decimals = 0 if !qr.Equal(want) { t.Errorf("Execute: \n%#v, want \n%#v", prettyPrint(*qr), prettyPrint(*want)) } diff --git a/go/vt/vttablet/grpcqueryservice/server.go b/go/vt/vttablet/grpcqueryservice/server.go index c3ada0faecf..76f90995f74 100644 --- a/go/vt/vttablet/grpcqueryservice/server.go +++ b/go/vt/vttablet/grpcqueryservice/server.go @@ -26,6 +26,7 @@ import ( "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vttablet/queryservice" + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" querypb "vitess.io/vitess/go/vt/proto/query" queryservicepb "vitess.io/vitess/go/vt/proto/queryservice" ) @@ -375,6 +376,23 @@ func (q *query) UpdateStream(request *querypb.UpdateStreamRequest, stream querys return nil } +// VStream is part of the queryservice.QueryServer interface +func (q *query) VStream(request *binlogdatapb.VStreamRequest, stream queryservicepb.Query_VStreamServer) (err error) { + defer q.server.HandlePanic(&err) + ctx := callerid.NewContext(callinfo.GRPCCallInfo(stream.Context()), + request.EffectiveCallerId, + request.ImmediateCallerId, + ) + if err := q.server.VStream(ctx, request.Target, request.Position, request.Filter, func(events []*binlogdatapb.VEvent) error { + return stream.Send(&binlogdatapb.VStreamResponse{ + Events: events, + }) + }); err != nil { + return vterrors.ToGRPC(err) + } + return nil +} + // Register registers the implementation on the provide gRPC Server. func Register(s *grpc.Server, server queryservice.QueryService) { queryservicepb.RegisterQueryServer(s, &query{server}) diff --git a/go/vt/vttablet/grpctabletconn/conn.go b/go/vt/vttablet/grpctabletconn/conn.go index f4871b5a860..7410c56e5ac 100644 --- a/go/vt/vttablet/grpctabletconn/conn.go +++ b/go/vt/vttablet/grpctabletconn/conn.go @@ -30,6 +30,7 @@ import ( "vitess.io/vitess/go/vt/vttablet/queryservice" "vitess.io/vitess/go/vt/vttablet/tabletconn" + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" querypb "vitess.io/vitess/go/vt/proto/query" queryservicepb "vitess.io/vitess/go/vt/proto/queryservice" topodatapb "vitess.io/vitess/go/vt/proto/topodata" @@ -668,6 +669,50 @@ func (conn *gRPCQueryClient) UpdateStream(ctx context.Context, target *querypb.T } } +// VStream starts a VReplication stream. +func (conn *gRPCQueryClient) VStream(ctx context.Context, target *querypb.Target, position string, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { + stream, err := func() (queryservicepb.Query_VStreamClient, error) { + conn.mu.RLock() + defer conn.mu.RUnlock() + if conn.cc == nil { + return nil, tabletconn.ConnClosed + } + + req := &binlogdatapb.VStreamRequest{ + Target: target, + EffectiveCallerId: callerid.EffectiveCallerIDFromContext(ctx), + ImmediateCallerId: callerid.ImmediateCallerIDFromContext(ctx), + Position: position, + Filter: filter, + } + stream, err := conn.c.VStream(ctx, req) + if err != nil { + return nil, tabletconn.ErrorFromGRPC(err) + } + return stream, nil + }() + if err != nil { + return err + } + for { + r, err := stream.Recv() + if err != nil { + return tabletconn.ErrorFromGRPC(err) + } + select { + case <-ctx.Done(): + return nil + default: + } + if err := send(r.Events); err != nil { + if err == io.EOF { + return nil + } + return err + } + } +} + // HandlePanic is a no-op. func (conn *gRPCQueryClient) HandlePanic(err *error) { } diff --git a/go/vt/vttablet/queryservice/queryservice.go b/go/vt/vttablet/queryservice/queryservice.go index 225912ab26f..4f3204fc800 100644 --- a/go/vt/vttablet/queryservice/queryservice.go +++ b/go/vt/vttablet/queryservice/queryservice.go @@ -25,6 +25,7 @@ import ( "vitess.io/vitess/go/sqltypes" + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" querypb "vitess.io/vitess/go/vt/proto/query" ) @@ -99,6 +100,9 @@ type QueryService interface { // UpdateStream streams updates from the provided position or timestamp. UpdateStream(ctx context.Context, target *querypb.Target, position string, timestamp int64, callback func(*querypb.StreamEvent) error) error + // VStream streams VReplication events based on the specified filter. + VStream(ctx context.Context, target *querypb.Target, startPos string, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error + // StreamHealth streams health status. StreamHealth(ctx context.Context, callback func(*querypb.StreamHealthResponse) error) error diff --git a/go/vt/vttablet/queryservice/wrapped.go b/go/vt/vttablet/queryservice/wrapped.go index 0ba474dff63..611c0630711 100644 --- a/go/vt/vttablet/queryservice/wrapped.go +++ b/go/vt/vttablet/queryservice/wrapped.go @@ -22,6 +22,7 @@ import ( "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/vterrors" + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" querypb "vitess.io/vitess/go/vt/proto/query" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" ) @@ -250,6 +251,13 @@ func (ws *wrappedService) UpdateStream(ctx context.Context, target *querypb.Targ }) } +func (ws *wrappedService) VStream(ctx context.Context, target *querypb.Target, startPos string, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { + return ws.wrapper(ctx, target, ws.impl, "UpdateStream", false, func(ctx context.Context, target *querypb.Target, conn QueryService) (error, bool) { + innerErr := conn.VStream(ctx, target, startPos, filter, send) + return innerErr, canRetry(ctx, innerErr) + }) +} + func (ws *wrappedService) StreamHealth(ctx context.Context, callback func(*querypb.StreamHealthResponse) error) error { return ws.wrapper(ctx, nil, ws.impl, "StreamHealth", false, func(ctx context.Context, target *querypb.Target, conn QueryService) (error, bool) { innerErr := conn.StreamHealth(ctx, callback) diff --git a/go/vt/vttablet/sandboxconn/sandboxconn.go b/go/vt/vttablet/sandboxconn/sandboxconn.go index dc614c7d2be..a856555098f 100644 --- a/go/vt/vttablet/sandboxconn/sandboxconn.go +++ b/go/vt/vttablet/sandboxconn/sandboxconn.go @@ -27,6 +27,7 @@ import ( "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vttablet/queryservice" + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" @@ -354,7 +355,11 @@ func (sbc *SandboxConn) StreamHealth(ctx context.Context, callback func(*querypb // UpdateStream is part of the QueryService interface. func (sbc *SandboxConn) UpdateStream(ctx context.Context, target *querypb.Target, position string, timestamp int64, callback func(*querypb.StreamEvent) error) error { - // FIXME(alainjobart) implement, use in vtgate tests. + return fmt.Errorf("Not implemented in test") +} + +// VStream is part of the QueryService interface. +func (sbc *SandboxConn) VStream(ctx context.Context, target *querypb.Target, startPos string, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { return fmt.Errorf("Not implemented in test") } diff --git a/go/vt/vttablet/tabletconntest/fakequeryservice.go b/go/vt/vttablet/tabletconntest/fakequeryservice.go index 813a48efd3a..f6ebc1e5bdd 100644 --- a/go/vt/vttablet/tabletconntest/fakequeryservice.go +++ b/go/vt/vttablet/tabletconntest/fakequeryservice.go @@ -28,6 +28,7 @@ import ( "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/callerid" + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" @@ -850,6 +851,11 @@ func (f *FakeQueryService) UpdateStream(ctx context.Context, target *querypb.Tar return nil } +// VStream is part of the queryservice.QueryService interface +func (f *FakeQueryService) VStream(ctx context.Context, target *querypb.Target, position string, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { + panic("not implemented") +} + // CreateFakeServer returns the fake server for the tests func CreateFakeServer(t *testing.T) *FakeQueryService { return &FakeQueryService{ diff --git a/go/vt/vttablet/tabletmanager/vreplication/controller.go b/go/vt/vttablet/tabletmanager/vreplication/controller.go index 784d104c1f8..75d1c7c32de 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/controller.go +++ b/go/vt/vttablet/tabletmanager/vreplication/controller.go @@ -131,8 +131,14 @@ func (ct *controller) run(ctx context.Context) { return default: } - log.Warningf("stream %v: %v, retrying after %v", ct.id, err, *retryDelay) - time.Sleep(*retryDelay) + log.Errorf("stream %v: %v, retrying after %v", ct.id, err, *retryDelay) + timer := time.NewTimer(*retryDelay) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-timer.C: + } } } @@ -169,7 +175,8 @@ func (ct *controller) runBlp(ctx context.Context) (err error) { } ct.sourceTablet.Set(tablet.Alias.String()) - if len(ct.source.Tables) > 0 { + switch { + case len(ct.source.Tables) > 0: // Table names can have search patterns. Resolve them against the schema. tables, err := mysqlctl.ResolveTables(ct.mysqld, dbClient.DBName(), ct.source.Tables) if err != nil { @@ -178,9 +185,18 @@ func (ct *controller) runBlp(ctx context.Context) (err error) { player := binlogplayer.NewBinlogPlayerTables(dbClient, tablet, tables, ct.id, ct.blpStats) return player.ApplyBinlogEvents(ctx) + case ct.source.KeyRange != nil: + player := binlogplayer.NewBinlogPlayerKeyRange(dbClient, tablet, ct.source.KeyRange, ct.id, ct.blpStats) + return player.ApplyBinlogEvents(ctx) + case ct.source.Filter != nil: + // VPlayer requires the timezone to be UTC. + if _, err := dbClient.ExecuteFetch("set @@session.time_zone = '+00:00'", 10000); err != nil { + return err + } + vplayer := newVPlayer(ct.id, &ct.source, tablet, ct.blpStats, dbClient, ct.mysqld) + return vplayer.Play(ctx) } - player := binlogplayer.NewBinlogPlayerKeyRange(dbClient, tablet, ct.source.KeyRange, ct.id, ct.blpStats) - return player.ApplyBinlogEvents(ctx) + return fmt.Errorf("missing source") } func (ct *controller) Stop() { diff --git a/go/vt/vttablet/tabletmanager/vreplication/planbuilder.go b/go/vt/vttablet/tabletmanager/vreplication/controller_plan.go similarity index 88% rename from go/vt/vttablet/tabletmanager/vreplication/planbuilder.go rename to go/vt/vttablet/tabletmanager/vreplication/controller_plan.go index 22713df9478..1a964e7b02a 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/planbuilder.go +++ b/go/vt/vttablet/tabletmanager/vreplication/controller_plan.go @@ -23,8 +23,8 @@ import ( "vitess.io/vitess/go/vt/sqlparser" ) -// plan is the plan for vreplication control statements. -type plan struct { +// controllerPlan is the plan for vreplication control statements. +type controllerPlan struct { opcode int query string id int @@ -37,8 +37,8 @@ const ( selectQuery ) -// getPlan parses the input query and returns an appropriate plan. -func getPlan(query string) (*plan, error) { +// buildControllerPlan parses the input query and returns an appropriate plan. +func buildControllerPlan(query string) (*controllerPlan, error) { stmt, err := sqlparser.Parse(query) if err != nil { return nil, err @@ -57,7 +57,7 @@ func getPlan(query string) (*plan, error) { } } -func buildInsertPlan(ins *sqlparser.Insert) (*plan, error) { +func buildInsertPlan(ins *sqlparser.Insert) (*controllerPlan, error) { if ins.Action != sqlparser.InsertStr { return nil, fmt.Errorf("unsupported construct: %v", sqlparser.String(ins)) } @@ -99,13 +99,13 @@ func buildInsertPlan(ins *sqlparser.Insert) (*plan, error) { return nil, fmt.Errorf("id should not have a value: %v", sqlparser.String(ins)) } } - return &plan{ + return &controllerPlan{ opcode: insertQuery, query: sqlparser.String(ins), }, nil } -func buildUpdatePlan(upd *sqlparser.Update) (*plan, error) { +func buildUpdatePlan(upd *sqlparser.Update) (*controllerPlan, error) { if sqlparser.String(upd.TableExprs) != "_vt.vreplication" { return nil, fmt.Errorf("invalid table name: %v", sqlparser.String(upd.TableExprs)) } @@ -123,14 +123,14 @@ func buildUpdatePlan(upd *sqlparser.Update) (*plan, error) { return nil, err } - return &plan{ + return &controllerPlan{ opcode: updateQuery, query: sqlparser.String(upd), id: id, }, nil } -func buildDeletePlan(del *sqlparser.Delete) (*plan, error) { +func buildDeletePlan(del *sqlparser.Delete) (*controllerPlan, error) { if del.Targets != nil { return nil, fmt.Errorf("unsupported construct: %v", sqlparser.String(del)) } @@ -149,18 +149,18 @@ func buildDeletePlan(del *sqlparser.Delete) (*plan, error) { return nil, err } - return &plan{ + return &controllerPlan{ opcode: deleteQuery, query: sqlparser.String(del), id: id, }, nil } -func buildSelectPlan(sel *sqlparser.Select) (*plan, error) { +func buildSelectPlan(sel *sqlparser.Select) (*controllerPlan, error) { if sqlparser.String(sel.From) != "_vt.vreplication" { return nil, fmt.Errorf("invalid table name: %v", sqlparser.String(sel.From)) } - return &plan{ + return &controllerPlan{ opcode: selectQuery, query: sqlparser.String(sel), }, nil diff --git a/go/vt/vttablet/tabletmanager/vreplication/planbuilder_test.go b/go/vt/vttablet/tabletmanager/vreplication/controller_plan_test.go similarity index 96% rename from go/vt/vttablet/tabletmanager/vreplication/planbuilder_test.go rename to go/vt/vttablet/tabletmanager/vreplication/controller_plan_test.go index af89c949c65..533668a2955 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/planbuilder_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/controller_plan_test.go @@ -21,27 +21,27 @@ import ( "testing" ) -func TestPlanBuilder(t *testing.T) { +func TestControllerPlan(t *testing.T) { tcases := []struct { in string - plan *plan + plan *controllerPlan err string }{{ // Insert in: "insert into _vt.vreplication values(null)", - plan: &plan{ + plan: &controllerPlan{ opcode: insertQuery, query: "insert into _vt.vreplication values (null)", }, }, { in: "insert into _vt.vreplication(id) values(null)", - plan: &plan{ + plan: &controllerPlan{ opcode: insertQuery, query: "insert into _vt.vreplication(id) values (null)", }, }, { in: "insert into _vt.vreplication(workflow, id) values('', null)", - plan: &plan{ + plan: &controllerPlan{ opcode: insertQuery, query: "insert into _vt.vreplication(workflow, id) values ('', null)", }, @@ -79,7 +79,7 @@ func TestPlanBuilder(t *testing.T) { // Update }, { in: "update _vt.vreplication set state='Running' where id = 1", - plan: &plan{ + plan: &controllerPlan{ opcode: updateQuery, query: "update _vt.vreplication set state = 'Running' where id = 1", id: 1, @@ -115,7 +115,7 @@ func TestPlanBuilder(t *testing.T) { // Delete }, { in: "delete from _vt.vreplication where id = 1", - plan: &plan{ + plan: &controllerPlan{ opcode: deleteQuery, query: "delete from _vt.vreplication where id = 1", id: 1, @@ -154,7 +154,7 @@ func TestPlanBuilder(t *testing.T) { // Select }, { in: "select * from _vt.vreplication where id = 1", - plan: &plan{ + plan: &controllerPlan{ opcode: selectQuery, query: "select * from _vt.vreplication where id = 1", }, @@ -171,7 +171,7 @@ func TestPlanBuilder(t *testing.T) { err: "unsupported construct: set a = 1", }} for _, tcase := range tcases { - pl, err := getPlan(tcase.in) + pl, err := buildControllerPlan(tcase.in) if err != nil { if err.Error() != tcase.err { t.Errorf("getPlan(%v) error:\n%v, want\n%v", tcase.in, err, tcase.err) diff --git a/go/vt/vttablet/tabletmanager/vreplication/controller_test.go b/go/vt/vttablet/tabletmanager/vreplication/controller_test.go index 52c2a0f0b69..e261444ee6c 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/controller_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/controller_test.go @@ -18,6 +18,7 @@ package vreplication import ( "errors" + "fmt" "testing" "time" @@ -51,14 +52,14 @@ var ( ) func TestControllerKeyRange(t *testing.T) { - ts := createTopo() - fbc := newFakeBinlogClient() - wantTablet := addTablet(ts, 100, "0", topodatapb.TabletType_REPLICA, true, true) + resetBinlogClient() + wantTablet := addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true) + defer deleteTablet(wantTablet) params := map[string]string{ "id": "1", "state": binlogplayer.BlpRunning, - "source": `keyspace:"ks" shard:"0" key_range: `, + "source": fmt.Sprintf(`keyspace:"%s" shard:"0" key_range: `, env.KeyspaceName), } dbClient := binlogplayer.NewMockDBClient(t) @@ -72,7 +73,7 @@ func TestControllerKeyRange(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - ct, err := newController(context.Background(), params, dbClientFactory, mysqld, ts, testCell, "replica", nil) + ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil) if err != nil { t.Fatal(err) } @@ -82,18 +83,18 @@ func TestControllerKeyRange(t *testing.T) { }() dbClient.Wait() - expectFBCRequest(t, fbc, wantTablet, testPos, nil, &topodatapb.KeyRange{End: []byte{0x80}}) + expectFBCRequest(t, wantTablet, testPos, nil, &topodatapb.KeyRange{End: []byte{0x80}}) } func TestControllerTables(t *testing.T) { - ts := createTopo() - wantTablet := addTablet(ts, 100, "0", topodatapb.TabletType_REPLICA, true, true) - fbc := newFakeBinlogClient() + wantTablet := addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true) + defer deleteTablet(wantTablet) + resetBinlogClient() params := map[string]string{ "id": "1", "state": binlogplayer.BlpRunning, - "source": `keyspace:"ks" shard:"0" tables:"table1" tables:"/funtables_/" `, + "source": fmt.Sprintf(`keyspace:"%s" shard:"0" tables:"table1" tables:"/funtables_/" `, env.KeyspaceName), } dbClient := binlogplayer.NewMockDBClient(t) @@ -132,7 +133,7 @@ func TestControllerTables(t *testing.T) { }, } - ct, err := newController(context.Background(), params, dbClientFactory, mysqld, ts, testCell, "replica", nil) + ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil) if err != nil { t.Fatal(err) } @@ -142,7 +143,7 @@ func TestControllerTables(t *testing.T) { }() dbClient.Wait() - expectFBCRequest(t, fbc, wantTablet, testPos, []string{"table1", "funtables_one"}, nil) + expectFBCRequest(t, wantTablet, testPos, []string{"table1", "funtables_one"}, nil) } func TestControllerBadID(t *testing.T) { @@ -176,15 +177,15 @@ func TestControllerStopped(t *testing.T) { } func TestControllerOverrides(t *testing.T) { - ts := createTopo() - fbc := newFakeBinlogClient() - wantTablet := addTablet(ts, 100, "0", topodatapb.TabletType_REPLICA, true, true) + resetBinlogClient() + wantTablet := addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true) + defer deleteTablet(wantTablet) params := map[string]string{ "id": "1", "state": binlogplayer.BlpRunning, - "source": `keyspace:"ks" shard:"0" key_range: `, - "cell": testCell, + "source": fmt.Sprintf(`keyspace:"%s" shard:"0" key_range: `, env.KeyspaceName), + "cell": env.Cells[0], "tablet_types": "replica", } @@ -199,7 +200,7 @@ func TestControllerOverrides(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - ct, err := newController(context.Background(), params, dbClientFactory, mysqld, ts, testCell, "rdonly", nil) + ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil) if err != nil { t.Fatal(err) } @@ -209,22 +210,21 @@ func TestControllerOverrides(t *testing.T) { }() dbClient.Wait() - expectFBCRequest(t, fbc, wantTablet, testPos, nil, &topodatapb.KeyRange{End: []byte{0x80}}) + expectFBCRequest(t, wantTablet, testPos, nil, &topodatapb.KeyRange{End: []byte{0x80}}) } func TestControllerCanceledContext(t *testing.T) { - ts := createTopo() - _ = addTablet(ts, 100, "0", topodatapb.TabletType_REPLICA, true, true) + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) params := map[string]string{ "id": "1", "state": binlogplayer.BlpRunning, - "source": `keyspace:"ks" shard:"0" key_range: `, + "source": fmt.Sprintf(`keyspace:"%s" shard:"0" key_range: `, env.KeyspaceName), } ctx, cancel := context.WithCancel(context.Background()) cancel() - ct, err := newController(ctx, params, nil, nil, ts, testCell, "rdonly", nil) + ct, err := newController(ctx, params, nil, nil, env.TopoServ, env.Cells[0], "rdonly", nil) if err != nil { t.Fatal(err) } @@ -242,15 +242,14 @@ func TestControllerRetry(t *testing.T) { defer func() { *retryDelay = savedDelay }() *retryDelay = 10 * time.Millisecond - ts := createTopo() - _ = newFakeBinlogClient() - _ = addTablet(ts, 100, "0", topodatapb.TabletType_REPLICA, true, true) + resetBinlogClient() + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) params := map[string]string{ "id": "1", "state": binlogplayer.BlpRunning, - "source": `keyspace:"ks" shard:"0" key_range: `, - "cell": testCell, + "source": fmt.Sprintf(`keyspace:"%s" shard:"0" key_range: `, env.KeyspaceName), + "cell": env.Cells[0], "tablet_types": "replica", } @@ -267,7 +266,7 @@ func TestControllerRetry(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - ct, err := newController(context.Background(), params, dbClientFactory, mysqld, ts, testCell, "rdonly", nil) + ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil) if err != nil { t.Fatal(err) } @@ -277,14 +276,14 @@ func TestControllerRetry(t *testing.T) { } func TestControllerStopPosition(t *testing.T) { - ts := createTopo() - fbc := newFakeBinlogClient() - wantTablet := addTablet(ts, 100, "0", topodatapb.TabletType_REPLICA, true, true) + resetBinlogClient() + wantTablet := addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true) + defer deleteTablet(wantTablet) params := map[string]string{ "id": "1", "state": binlogplayer.BlpRunning, - "source": `keyspace:"ks" shard:"0" key_range: `, + "source": fmt.Sprintf(`keyspace:"%s" shard:"0" key_range: `, env.KeyspaceName), } dbClient := binlogplayer.NewMockDBClient(t) @@ -312,7 +311,7 @@ func TestControllerStopPosition(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - ct, err := newController(context.Background(), params, dbClientFactory, mysqld, ts, testCell, "replica", nil) + ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil) if err != nil { t.Fatal(err) } @@ -329,5 +328,5 @@ func TestControllerStopPosition(t *testing.T) { } dbClient.Wait() - expectFBCRequest(t, fbc, wantTablet, testPos, nil, &topodatapb.KeyRange{End: []byte{0x80}}) + expectFBCRequest(t, wantTablet, testPos, nil, &topodatapb.KeyRange{End: []byte{0x80}}) } diff --git a/go/vt/vttablet/tabletmanager/vreplication/engine.go b/go/vt/vttablet/tabletmanager/vreplication/engine.go index c499225cf4c..3577508abdc 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/engine.go +++ b/go/vt/vttablet/tabletmanager/vreplication/engine.go @@ -200,7 +200,7 @@ func (vre *Engine) Exec(query string) (*sqltypes.Result, error) { } defer vre.updateStats() - plan, err := getPlan(query) + plan, err := buildControllerPlan(query) if err != nil { return nil, err } @@ -281,14 +281,19 @@ func (vre *Engine) WaitForPos(ctx context.Context, id int, pos string) error { return err } - vre.mu.Lock() - if !vre.isOpen { - vre.mu.Unlock() - return errors.New("vreplication engine is closed") + if err := func() error { + vre.mu.Lock() + defer vre.mu.Unlock() + if !vre.isOpen { + return errors.New("vreplication engine is closed") + } + + // Ensure that the engine won't be closed while this is running. + vre.wg.Add(1) + return nil + }(); err != nil { + return err } - // Ensure that the engine won't be closed while this is running. - vre.wg.Add(1) - vre.mu.Unlock() defer vre.wg.Done() dbClient := vre.dbClientFactory() @@ -298,13 +303,13 @@ func (vre *Engine) WaitForPos(ctx context.Context, id int, pos string) error { defer dbClient.Close() for { - qr, err := dbClient.ExecuteFetch(binlogplayer.ReadVReplicationPos(uint32(id)), 10) + qr, err := dbClient.ExecuteFetch(binlogplayer.ReadVReplicationStatus(uint32(id)), 10) switch { case err != nil: return err case len(qr.Rows) == 0: return fmt.Errorf("vreplication stream %d not found", id) - case len(qr.Rows) > 1 || len(qr.Rows[0]) != 1: + case len(qr.Rows) > 1 || len(qr.Rows[0]) != 3: return fmt.Errorf("unexpected result: %v", qr) } current, err := mysql.DecodePosition(qr.Rows[0][0].ToString()) @@ -316,6 +321,10 @@ func (vre *Engine) WaitForPos(ctx context.Context, id int, pos string) error { return nil } + if qr.Rows[0][1].ToString() == binlogplayer.BlpStopped { + return fmt.Errorf("replication has stopped at %v before reaching position %v, message: %s", current, mPos, qr.Rows[0][2].ToString()) + } + select { case <-ctx.Done(): return ctx.Err() diff --git a/go/vt/vttablet/tabletmanager/vreplication/engine_test.go b/go/vt/vttablet/tabletmanager/vreplication/engine_test.go index f93ec5559e5..eb106370b56 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/engine_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/engine_test.go @@ -17,6 +17,7 @@ limitations under the License. package vreplication import ( + "fmt" "reflect" "testing" "time" @@ -32,16 +33,15 @@ import ( func TestEngineOpen(t *testing.T) { defer func() { globalStats = &vrStats{} }() - ts := createTopo() - _ = addTablet(ts, 100, "0", topodatapb.TabletType_REPLICA, true, true) - _ = newFakeBinlogClient() + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + resetBinlogClient() dbClient := binlogplayer.NewMockDBClient(t) dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} // Test Insert - vre := NewEngine(ts, testCell, mysqld, dbClientFactory) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory) if vre.IsOpen() { t.Errorf("IsOpen: %v, want false", vre.IsOpen()) } @@ -51,7 +51,7 @@ func TestEngineOpen(t *testing.T) { "id|state|source", "int64|varchar|varchar", ), - `1|Running|keyspace:"ks" shard:"0" key_range: `, + fmt.Sprintf(`1|Running|keyspace:"%s" shard:"0" key_range: `, env.KeyspaceName), ), nil) dbClient.ExpectRequest("update _vt.vreplication set state='Running', message='' where id=1", testDMLResponse, nil) dbClient.ExpectRequest("select pos, stop_pos, max_tps, max_replication_lag from _vt.vreplication where id=1", testSettingsResponse, nil) @@ -81,16 +81,15 @@ func TestEngineOpen(t *testing.T) { func TestEngineExec(t *testing.T) { defer func() { globalStats = &vrStats{} }() - ts := createTopo() - _ = addTablet(ts, 100, "0", topodatapb.TabletType_REPLICA, true, true) - _ = newFakeBinlogClient() + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + resetBinlogClient() dbClient := binlogplayer.NewMockDBClient(t) dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} // Test Insert - vre := NewEngine(ts, testCell, mysqld, dbClientFactory) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory) dbClient.ExpectRequest("select * from _vt.vreplication", &sqltypes.Result{}, nil) if err := vre.Open(context.Background()); err != nil { @@ -105,7 +104,7 @@ func TestEngineExec(t *testing.T) { "id|state|source", "int64|varchar|varchar", ), - `1|Running|keyspace:"ks" shard:"0" key_range: `, + fmt.Sprintf(`1|Running|keyspace:"%s" shard:"0" key_range: `, env.KeyspaceName), ), nil) dbClient.ExpectRequest("update _vt.vreplication set state='Running', message='' where id=1", testDMLResponse, nil) dbClient.ExpectRequest("select pos, stop_pos, max_tps, max_replication_lag from _vt.vreplication where id=1", testSettingsResponse, nil) @@ -145,7 +144,7 @@ func TestEngineExec(t *testing.T) { "id|state|source", "int64|varchar|varchar", ), - `1|Running|keyspace:"ks" shard:"0" key_range: `, + fmt.Sprintf(`1|Running|keyspace:"%s" shard:"0" key_range: `, env.KeyspaceName), ), nil) dbClient.ExpectRequest("update _vt.vreplication set state='Running', message='' where id=1", testDMLResponse, nil) dbClient.ExpectRequest("select pos, stop_pos, max_tps, max_replication_lag from _vt.vreplication where id=1", testSettingsResponse, nil) @@ -206,15 +205,14 @@ func TestEngineExec(t *testing.T) { func TestEngineBadInsert(t *testing.T) { defer func() { globalStats = &vrStats{} }() - ts := createTopo() - _ = addTablet(ts, 100, "0", topodatapb.TabletType_REPLICA, true, true) - _ = newFakeBinlogClient() + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + resetBinlogClient() dbClient := binlogplayer.NewMockDBClient(t) dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - vre := NewEngine(ts, testCell, mysqld, dbClientFactory) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory) dbClient.ExpectRequest("select * from _vt.vreplication", &sqltypes.Result{}, nil) if err := vre.Open(context.Background()); err != nil { @@ -237,15 +235,14 @@ func TestEngineBadInsert(t *testing.T) { } func TestEngineSelect(t *testing.T) { - ts := createTopo() - _ = addTablet(ts, 100, "0", topodatapb.TabletType_REPLICA, true, true) - _ = newFakeBinlogClient() + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + resetBinlogClient() dbClient := binlogplayer.NewMockDBClient(t) dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - vre := NewEngine(ts, testCell, mysqld, dbClientFactory) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory) dbClient.ExpectRequest("select * from _vt.vreplication", &sqltypes.Result{}, nil) if err := vre.Open(context.Background()); err != nil { @@ -260,7 +257,7 @@ func TestEngineSelect(t *testing.T) { "id|state|source|pos", "int64|varchar|varchar|varchar", ), - `1|Running|keyspace:"ks" shard:"0" key_range: |MariaDB/0-1-1083`, + fmt.Sprintf(`1|Running|keyspace:"%s" shard:"0" key_range: |MariaDB/0-1-1083`, env.KeyspaceName), ) dbClient.ExpectRequest(wantQuery, wantResult, nil) qr, err := vre.Exec(wantQuery) @@ -280,18 +277,22 @@ func TestWaitForPos(t *testing.T) { dbClient := binlogplayer.NewMockDBClient(t) mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} dbClientFactory := func() binlogplayer.DBClient { return dbClient } - vre := NewEngine(createTopo(), testCell, mysqld, dbClientFactory) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory) dbClient.ExpectRequest("select * from _vt.vreplication", &sqltypes.Result{}, nil) if err := vre.Open(context.Background()); err != nil { t.Fatal(err) } - dbClient.ExpectRequest("select pos from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ + dbClient.ExpectRequest("select pos, state, message from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ sqltypes.NewVarBinary("MariaDB/0-1-1083"), + sqltypes.NewVarBinary("Running"), + sqltypes.NewVarBinary(""), }}}, nil) - dbClient.ExpectRequest("select pos from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ + dbClient.ExpectRequest("select pos, state, message from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ sqltypes.NewVarBinary("MariaDB/0-1-1084"), + sqltypes.NewVarBinary("Running"), + sqltypes.NewVarBinary(""), }}}, nil) start := time.Now() if err := vre.WaitForPos(context.Background(), 1, "MariaDB/0-1-1084"); err != nil { @@ -306,7 +307,7 @@ func TestWaitForPosError(t *testing.T) { dbClient := binlogplayer.NewMockDBClient(t) mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} dbClientFactory := func() binlogplayer.DBClient { return dbClient } - vre := NewEngine(createTopo(), testCell, mysqld, dbClientFactory) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory) err := vre.WaitForPos(context.Background(), 1, "MariaDB/0-1-1084") want := `vreplication engine is closed` @@ -325,14 +326,14 @@ func TestWaitForPosError(t *testing.T) { t.Errorf("WaitForPos: %v, want %v", err, want) } - dbClient.ExpectRequest("select pos from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{}}}, nil) + dbClient.ExpectRequest("select pos, state, message from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{}}}, nil) err = vre.WaitForPos(context.Background(), 1, "MariaDB/0-1-1084") want = "unexpected result: &{[] 0 0 [[]] }" if err == nil || err.Error() != want { t.Errorf("WaitForPos: %v, want %v", err, want) } - dbClient.ExpectRequest("select pos from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ + dbClient.ExpectRequest("select pos, state, message from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ sqltypes.NewVarBinary("MariaDB/0-1-1083"), }, { sqltypes.NewVarBinary("MariaDB/0-1-1083"), @@ -348,15 +349,17 @@ func TestWaitForPosCancel(t *testing.T) { dbClient := binlogplayer.NewMockDBClient(t) mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} dbClientFactory := func() binlogplayer.DBClient { return dbClient } - vre := NewEngine(createTopo(), testCell, mysqld, dbClientFactory) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory) dbClient.ExpectRequest("select * from _vt.vreplication", &sqltypes.Result{}, nil) if err := vre.Open(context.Background()); err != nil { t.Fatal(err) } - dbClient.ExpectRequest("select pos from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ + dbClient.ExpectRequest("select pos, state, message from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ sqltypes.NewVarBinary("MariaDB/0-1-1083"), + sqltypes.NewVarBinary("Running"), + sqltypes.NewVarBinary(""), }}}, nil) ctx, cancel := context.WithCancel(context.Background()) cancel() @@ -370,8 +373,10 @@ func TestWaitForPosCancel(t *testing.T) { time.Sleep(5 * time.Millisecond) vre.Close() }() - dbClient.ExpectRequest("select pos from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ + dbClient.ExpectRequest("select pos, state, message from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ sqltypes.NewVarBinary("MariaDB/0-1-1083"), + sqltypes.NewVarBinary("Running"), + sqltypes.NewVarBinary(""), }}}, nil) err = vre.WaitForPos(context.Background(), 1, "MariaDB/0-1-1084") want := "vreplication is closing: context canceled" @@ -383,16 +388,15 @@ func TestWaitForPosCancel(t *testing.T) { func TestCreateDBAndTable(t *testing.T) { defer func() { globalStats = &vrStats{} }() - ts := createTopo() - _ = addTablet(ts, 100, "0", topodatapb.TabletType_REPLICA, true, true) - _ = newFakeBinlogClient() + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + resetBinlogClient() dbClient := binlogplayer.NewMockDBClient(t) dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} // Test Insert - vre := NewEngine(ts, testCell, mysqld, dbClientFactory) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory) tableNotFound := mysql.SQLError{Num: 1146, Message: "table not found"} dbClient.ExpectRequest("select * from _vt.vreplication", nil, &tableNotFound) @@ -432,7 +436,7 @@ func TestCreateDBAndTable(t *testing.T) { "id|state|source", "int64|varchar|varchar", ), - `1|Running|keyspace:"ks" shard:"0" key_range: `, + fmt.Sprintf(`1|Running|keyspace:"%s" shard:"0" key_range: `, env.KeyspaceName), ), nil) dbClient.ExpectRequest("update _vt.vreplication set state='Running', message='' where id=1", testDMLResponse, nil) dbClient.ExpectRequest("select pos, stop_pos, max_tps, max_replication_lag from _vt.vreplication where id=1", testSettingsResponse, nil) diff --git a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go index 559f285e1ff..23c019bd638 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go @@ -18,58 +18,119 @@ package vreplication import ( "flag" + "fmt" + "os" "reflect" + "regexp" + "strings" "testing" + "time" "github.com/golang/protobuf/proto" "golang.org/x/net/context" + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" "vitess.io/vitess/go/vt/grpcclient" "vitess.io/vitess/go/vt/topo" - "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/vttablet/queryservice" "vitess.io/vitess/go/vt/vttablet/queryservice/fakes" "vitess.io/vitess/go/vt/vttablet/tabletconn" + "vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer" + "vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/testenv" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) -const ( - testCell = "cell" - testKeyspace = "ks" - testShard = "0" +var ( + playerEngine *Engine + streamerEngine *vstreamer.Engine + env *testenv.Env + globalFBC = &fakeBinlogClient{} + vrepldb = "vrepl" + globalDBQueries = make(chan string, 1000) ) -// This file provides support functions for tests. -// It's capable of creating a single unsharded keyspace -// and allows you to add various tablet types. +func init() { + tabletconn.RegisterDialer("test", func(tablet *topodatapb.Tablet, failFast grpcclient.FailFast) (queryservice.QueryService, error) { + return &fakeTabletConn{ + QueryService: fakes.ErrorQueryService, + tablet: tablet, + }, nil + }) + flag.Set("tablet_protocol", "test") -//-------------------------------------- -// Topos and tablets + binlogplayer.RegisterClientFactory("test", func() binlogplayer.Client { return globalFBC }) + flag.Set("binlog_player_protocol", "test") +} -func createTopo() *topo.Server { - ts := memorytopo.NewServer(testCell) - ctx := context.Background() - if err := ts.CreateKeyspace(ctx, testKeyspace, &topodatapb.Keyspace{}); err != nil { - panic(err) - } - if err := ts.CreateShard(ctx, testKeyspace, testShard); err != nil { - panic(err) +func TestMain(m *testing.M) { + flag.Parse() // Do not remove this comment, import into google3 depends on it + + if testing.Short() { + os.Exit(m.Run()) } - return ts + + exitCode := func() int { + var err error + env, err = testenv.Init() + if err != nil { + fmt.Fprintf(os.Stderr, "%v", err) + return 1 + } + defer env.Close() + + // engines cannot be initialized in testenv because it introduces + // circular dependencies. + streamerEngine = vstreamer.NewEngine(env.SrvTopo, env.SchemaEngine) + streamerEngine.InitDBConfig(env.Dbcfgs) + streamerEngine.Open(env.KeyspaceName, env.Cells[0]) + defer streamerEngine.Close() + + if err := env.Mysqld.ExecuteSuperQuery(context.Background(), fmt.Sprintf("create database %s", vrepldb)); err != nil { + fmt.Fprintf(os.Stderr, "%v", err) + return 1 + } + + if err := env.Mysqld.ExecuteSuperQuery(context.Background(), "set @@global.innodb_lock_wait_timeout=1"); err != nil { + fmt.Fprintf(os.Stderr, "%v", err) + return 1 + } + + playerEngine = NewEngine(env.TopoServ, env.Cells[0], env.Mysqld, realDBClientFactory) + if err := playerEngine.Open(context.Background()); err != nil { + fmt.Fprintf(os.Stderr, "%v", err) + return 1 + } + defer playerEngine.Close() + + return m.Run() + }() + os.Exit(exitCode) +} + +func resetBinlogClient() { + globalFBC = &fakeBinlogClient{} } -func addTablet(ts *topo.Server, id int, shard string, tabletType topodatapb.TabletType, serving, healthy bool) *topodatapb.Tablet { +//-------------------------------------- +// Topos and tablets + +func addTablet(id int, shard string, tabletType topodatapb.TabletType, serving, healthy bool) *topodatapb.Tablet { t := newTablet(id, shard, tabletType, serving, healthy) - if err := ts.CreateTablet(context.Background(), t); err != nil { + if err := env.TopoServ.CreateTablet(context.Background(), t); err != nil { panic(err) } return t } +func deleteTablet(t *topodatapb.Tablet) { + env.TopoServ.DeleteTablet(context.Background(), t.Alias) +} + func newTablet(id int, shard string, tabletType topodatapb.TabletType, serving, healthy bool) *topodatapb.Tablet { stag := "not_serving" if serving { @@ -85,11 +146,11 @@ func newTablet(id int, shard string, tabletType topodatapb.TabletType, serving, } return &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ - Cell: testCell, + Cell: env.Cells[0], Uid: uint32(id), }, - Keyspace: testKeyspace, - Shard: testShard, + Keyspace: env.KeyspaceName, + Shard: env.ShardName, KeyRange: kr, Type: tabletType, Tags: map[string]string{ @@ -132,6 +193,11 @@ func (ftc *fakeTabletConn) StreamHealth(ctx context.Context, callback func(*quer return nil } +// VStream directly calls into the pre-initialized engine. +func (ftc *fakeTabletConn) VStream(ctx context.Context, target *querypb.Target, startPos string, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { + return streamerEngine.Stream(ctx, startPos, filter, send) +} + //-------------------------------------- // Binlog Client to TabletManager @@ -145,11 +211,6 @@ type fakeBinlogClient struct { lastCharset *binlogdatapb.Charset } -func newFakeBinlogClient() *fakeBinlogClient { - globalFBC = &fakeBinlogClient{} - return globalFBC -} - func (fbc *fakeBinlogClient) Dial(tablet *topodatapb.Tablet) error { fbc.lastTablet = tablet return nil @@ -198,37 +259,138 @@ func (t *btStream) Recv() (*binlogdatapb.BinlogTransaction, error) { return nil, t.ctx.Err() } -func expectFBCRequest(t *testing.T, fbc *fakeBinlogClient, tablet *topodatapb.Tablet, pos string, tables []string, kr *topodatapb.KeyRange) { +func expectFBCRequest(t *testing.T, tablet *topodatapb.Tablet, pos string, tables []string, kr *topodatapb.KeyRange) { t.Helper() - if !proto.Equal(tablet, fbc.lastTablet) { - t.Errorf("Request tablet: %v, want %v", fbc.lastTablet, tablet) + if !proto.Equal(tablet, globalFBC.lastTablet) { + t.Errorf("Request tablet: %v, want %v", globalFBC.lastTablet, tablet) } - if pos != fbc.lastPos { - t.Errorf("Request pos: %v, want %v", fbc.lastPos, pos) + if pos != globalFBC.lastPos { + t.Errorf("Request pos: %v, want %v", globalFBC.lastPos, pos) } - if !reflect.DeepEqual(tables, fbc.lastTables) { - t.Errorf("Request tables: %v, want %v", fbc.lastTables, tables) + if !reflect.DeepEqual(tables, globalFBC.lastTables) { + t.Errorf("Request tables: %v, want %v", globalFBC.lastTables, tables) } - if !proto.Equal(kr, fbc.lastKeyRange) { - t.Errorf("Request KeyRange: %v, want %v", fbc.lastKeyRange, kr) + if !proto.Equal(kr, globalFBC.lastKeyRange) { + t.Errorf("Request KeyRange: %v, want %v", globalFBC.lastKeyRange, kr) } } //-------------------------------------- -// init +// DBCLient wrapper -// globalFBC is set by newFakeBinlogClient, which is then returned by the client factory below. -var globalFBC *fakeBinlogClient +func realDBClientFactory() binlogplayer.DBClient { + return &realDBClient{} +} -func init() { - tabletconn.RegisterDialer("test", func(tablet *topodatapb.Tablet, failFast grpcclient.FailFast) (queryservice.QueryService, error) { - return &fakeTabletConn{ - QueryService: fakes.ErrorQueryService, - tablet: tablet, - }, nil - }) - flag.Set("tablet_protocol", "test") +type realDBClient struct { + conn *mysql.Conn + nolog bool +} - binlogplayer.RegisterClientFactory("test", func() binlogplayer.Client { return globalFBC }) - flag.Set("binlog_player_protocol", "test") +func (dbc *realDBClient) DBName() string { + return vrepldb +} + +func (dbc *realDBClient) Connect() error { + app := env.Dbcfgs.AppWithDB() + app.DbName = vrepldb + conn, err := mysql.Connect(context.Background(), app) + if err != nil { + return err + } + dbc.conn = conn + return nil +} + +func (dbc *realDBClient) Begin() error { + _, err := dbc.ExecuteFetch("begin", 10000) + return err +} + +func (dbc *realDBClient) Commit() error { + _, err := dbc.ExecuteFetch("commit", 10000) + return err +} + +func (dbc *realDBClient) Rollback() error { + _, err := dbc.ExecuteFetch("rollback", 10000) + return err +} + +func (dbc *realDBClient) Close() { + dbc.conn.Close() + dbc.conn = nil +} + +func (dbc *realDBClient) ExecuteFetch(query string, maxrows int) (*sqltypes.Result, error) { + if strings.HasPrefix(query, "use") { + return nil, nil + } + qr, err := dbc.conn.ExecuteFetch(query, 10000, true) + if !strings.HasPrefix(query, "select") && !strings.HasPrefix(query, "set") && !dbc.nolog { + globalDBQueries <- query + } + return qr, err +} + +func expectDBClientQueries(t *testing.T, queries []string) { + t.Helper() + failed := false + for i, query := range queries { + if failed { + t.Errorf("no query received, expecting %s", query) + continue + } + var got string + select { + case got = <-globalDBQueries: + var match bool + if query[0] == '/' { + result, err := regexp.MatchString(query[1:], got) + if err != nil { + panic(err) + } + match = result + } else { + match = (got == query) + } + if !match { + t.Errorf("query:\n%q, does not match query %d:\n%q", got, i, query) + } + case <-time.After(5 * time.Second): + t.Errorf("no query received, expecting %s", query) + failed = true + } + } + for { + select { + case got := <-globalDBQueries: + t.Errorf("unexpected query: %s", got) + default: + return + } + } +} + +func expectData(t *testing.T, table string, values [][]string) { + t.Helper() + + qr, err := env.Mysqld.FetchSuperQuery(context.Background(), fmt.Sprintf("select * from %s.%s", vrepldb, table)) + if err != nil { + t.Error(err) + return + } + if len(values) != len(qr.Rows) { + t.Fatalf("row counts don't match: %v, want %v", qr.Rows, values) + } + for i, row := range values { + if len(row) != len(qr.Rows[i]) { + t.Fatalf("Too few columns, result: %v, row: %d, want: %v", qr.Rows[i], i, row) + } + for j, val := range row { + if got := qr.Rows[i][j].ToString(); got != val { + t.Errorf("Mismatch at (%d, %d): %v, want %s", i, j, qr.Rows[i][j], val) + } + } + } } diff --git a/go/vt/vttablet/tabletmanager/vreplication/player_plan.go b/go/vt/vttablet/tabletmanager/vreplication/player_plan.go new file mode 100644 index 00000000000..77ecde44d5d --- /dev/null +++ b/go/vt/vttablet/tabletmanager/vreplication/player_plan.go @@ -0,0 +1,245 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vreplication + +import ( + "fmt" + "strings" + + "vitess.io/vitess/go/vt/sqlparser" + + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + querypb "vitess.io/vitess/go/vt/proto/query" +) + +// PlayerPlan is the execution plan for a player stream. +type PlayerPlan struct { + VStreamFilter *binlogdatapb.Filter + TablePlans map[string]*TablePlan +} + +// TablePlan is the execution plan for a table within a player stream. +// There are two incarantions of this per table. The first one is built +// while analyzing the inital stream request. A tentative plan is built +// without knowing the table info. The second incarnation is built when +// we receive the field info for a table. At that time, we copy the +// original TablePlan into a separtae map and populate the Fields and +// PKCols members. +type TablePlan struct { + Name string + ColExprs []*ColExpr `json:",omitempty"` + OnInsert InsertType `json:",omitempty"` + + Fields []*querypb.Field `json:",omitempty"` + PKCols []*ColExpr `json:",omitempty"` +} + +func (tp *TablePlan) findCol(name sqlparser.ColIdent) *ColExpr { + for _, cExpr := range tp.ColExprs { + if cExpr.ColName.Equal(name) { + return cExpr + } + } + return nil +} + +// ColExpr describes the processing to be performed to +// compute the value of the target table column. +type ColExpr struct { + ColName sqlparser.ColIdent + ColNum int + Operation Operation `json:",omitempty"` + IsGrouped bool `json:",omitempty"` +} + +// Operation is the opcode for the ColExpr. +type Operation int + +// The following values are the various ColExpr opcodes. +const ( + OpNone = Operation(iota) + OpCount + OpSum +) + +// InsertType describes the type of insert statement to generate. +type InsertType int + +// The following values are the various insert types. +const ( + InsertNormal = InsertType(iota) + InsertOndup + InsertIgnore +) + +func buildPlayerPlan(filter *binlogdatapb.Filter) (*PlayerPlan, error) { + plan := &PlayerPlan{ + VStreamFilter: &binlogdatapb.Filter{ + Rules: make([]*binlogdatapb.Rule, len(filter.Rules)), + }, + TablePlans: make(map[string]*TablePlan), + } + for i, rule := range filter.Rules { + if strings.HasPrefix(rule.Match, "/") { + plan.VStreamFilter.Rules[i] = rule + continue + } + sendRule, tplan, err := buildTablePlan(rule) + if err != nil { + return nil, err + } + plan.VStreamFilter.Rules[i] = sendRule + plan.TablePlans[sendRule.Match] = tplan + } + return plan, nil +} + +func buildTablePlan(rule *binlogdatapb.Rule) (*binlogdatapb.Rule, *TablePlan, error) { + statement, err := sqlparser.Parse(rule.Filter) + if err != nil { + return nil, nil, err + } + sel, ok := statement.(*sqlparser.Select) + if !ok { + return nil, nil, fmt.Errorf("unexpected: %v", sqlparser.String(statement)) + } + if sel.Distinct != "" { + return nil, nil, fmt.Errorf("unexpected: %v", sqlparser.String(sel)) + } + if len(sel.From) > 1 { + return nil, nil, fmt.Errorf("unexpected: %v", sqlparser.String(sel)) + } + node, ok := sel.From[0].(*sqlparser.AliasedTableExpr) + if !ok { + return nil, nil, fmt.Errorf("unexpected: %v", sqlparser.String(sel)) + } + fromTable := sqlparser.GetTableName(node.Expr) + if fromTable.IsEmpty() { + return nil, nil, fmt.Errorf("unexpected: %v", sqlparser.String(sel)) + } + + if _, ok := sel.SelectExprs[0].(*sqlparser.StarExpr); ok { + if len(sel.SelectExprs) != 1 { + return nil, nil, fmt.Errorf("unexpected: %v", sqlparser.String(sel)) + } + sendRule := &binlogdatapb.Rule{ + Match: fromTable.String(), + Filter: rule.Filter, + } + return sendRule, &TablePlan{Name: rule.Match}, nil + } + + tplan := &TablePlan{ + Name: rule.Match, + } + sendSelect := &sqlparser.Select{ + From: sel.From, + Where: sel.Where, + } + for _, expr := range sel.SelectExprs { + selExpr, cExpr, err := analyzeExpr(expr) + if err != nil { + return nil, nil, err + } + if selExpr != nil { + sendSelect.SelectExprs = append(sendSelect.SelectExprs, selExpr) + cExpr.ColNum = len(sendSelect.SelectExprs) - 1 + } + tplan.ColExprs = append(tplan.ColExprs, cExpr) + } + + if sel.GroupBy != nil { + if err := analyzeGroupBy(sel.GroupBy, tplan); err != nil { + return nil, nil, err + } + tplan.OnInsert = InsertIgnore + for _, cExpr := range tplan.ColExprs { + if !cExpr.IsGrouped { + tplan.OnInsert = InsertOndup + break + } + } + } + sendRule := &binlogdatapb.Rule{ + Match: fromTable.String(), + Filter: sqlparser.String(sendSelect), + } + return sendRule, tplan, nil +} + +func analyzeExpr(selExpr sqlparser.SelectExpr) (sqlparser.SelectExpr, *ColExpr, error) { + aliased, ok := selExpr.(*sqlparser.AliasedExpr) + if !ok { + return nil, nil, fmt.Errorf("unexpected: %v", sqlparser.String(selExpr)) + } + as := aliased.As + if as.IsEmpty() { + as = sqlparser.NewColIdent(sqlparser.String(aliased.Expr)) + } + switch expr := aliased.Expr.(type) { + case *sqlparser.ColName: + return selExpr, &ColExpr{ColName: as}, nil + case *sqlparser.FuncExpr: + if expr.Distinct || len(expr.Exprs) != 1 { + return nil, nil, fmt.Errorf("unexpected: %v", sqlparser.String(expr)) + } + if aliased.As.IsEmpty() { + return nil, nil, fmt.Errorf("expression needs an alias: %v", sqlparser.String(expr)) + } + switch fname := expr.Name.Lowered(); fname { + case "month", "day", "hour": + return selExpr, &ColExpr{ColName: as}, nil + case "count": + if _, ok := expr.Exprs[0].(*sqlparser.StarExpr); !ok { + return nil, nil, fmt.Errorf("only count(*) is supported: %v", sqlparser.String(expr)) + } + return nil, &ColExpr{ColName: as, Operation: OpCount}, nil + case "sum": + aInner, ok := expr.Exprs[0].(*sqlparser.AliasedExpr) + if !ok { + return nil, nil, fmt.Errorf("unexpected: %v", sqlparser.String(expr)) + } + innerCol, ok := aInner.Expr.(*sqlparser.ColName) + if !ok { + return nil, nil, fmt.Errorf("unexpected: %v", sqlparser.String(expr)) + } + return &sqlparser.AliasedExpr{Expr: innerCol}, &ColExpr{ColName: as, Operation: OpSum}, nil + default: + return nil, nil, fmt.Errorf("unexpected: %v", sqlparser.String(expr)) + } + default: + return nil, nil, fmt.Errorf("unexpected: %v", sqlparser.String(expr)) + } +} + +func analyzeGroupBy(groupBy sqlparser.GroupBy, tplan *TablePlan) error { + for _, expr := range groupBy { + colname, ok := expr.(*sqlparser.ColName) + if !ok { + return fmt.Errorf("unexpected: %v", sqlparser.String(expr)) + } + cExpr := tplan.findCol(colname.Name) + if cExpr == nil { + return fmt.Errorf("group by expression does not reference an alias in the select list: %v", sqlparser.String(expr)) + } + if cExpr.Operation != OpNone { + return fmt.Errorf("group by expression is not allowed to reference an aggregate expression: %v", sqlparser.String(expr)) + } + cExpr.IsGrouped = true + } + return nil +} diff --git a/go/vt/vttablet/tabletmanager/vreplication/player_plan_test.go b/go/vt/vttablet/tabletmanager/vreplication/player_plan_test.go new file mode 100644 index 00000000000..bf5066002f2 --- /dev/null +++ b/go/vt/vttablet/tabletmanager/vreplication/player_plan_test.go @@ -0,0 +1,435 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vreplication + +import ( + "encoding/json" + "testing" + + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + "vitess.io/vitess/go/vt/sqlparser" +) + +func TestPlayerPlan(t *testing.T) { + testcases := []struct { + input *binlogdatapb.Filter + plan *PlayerPlan + err string + }{{ + // Regular expression + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + }, + plan: &PlayerPlan{ + VStreamFilter: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + }, + TablePlans: map[string]*TablePlan{}, + }, + }, { + // '*' expression + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select * from t2", + }}, + }, + plan: &PlayerPlan{ + VStreamFilter: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t2", + Filter: "select * from t2", + }}, + }, + TablePlans: map[string]*TablePlan{ + "t2": { + Name: "t1", + }, + }, + }, + }, { + // Explicit columns + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select c1, c2 from t2", + }}, + }, + plan: &PlayerPlan{ + VStreamFilter: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t2", + Filter: "select c1, c2 from t2", + }}, + }, + TablePlans: map[string]*TablePlan{ + "t2": { + Name: "t1", + ColExprs: []*ColExpr{{ + ColName: sqlparser.NewColIdent("c1"), + ColNum: 0, + }, { + ColName: sqlparser.NewColIdent("c2"), + ColNum: 1, + }}, + }, + }, + }, + }, { + // func expr + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select hour(c1) as hc1, day(c2) as dc2 from t2", + }}, + }, + plan: &PlayerPlan{ + VStreamFilter: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t2", + Filter: "select hour(c1) as hc1, day(c2) as dc2 from t2", + }}, + }, + TablePlans: map[string]*TablePlan{ + "t2": { + Name: "t1", + ColExprs: []*ColExpr{{ + ColName: sqlparser.NewColIdent("hc1"), + ColNum: 0, + }, { + ColName: sqlparser.NewColIdent("dc2"), + ColNum: 1, + }}, + }, + }, + }, + }, { + // count expr + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select hour(c1) as hc1, count(*) as c, day(c2) as dc2 from t2", + }}, + }, + plan: &PlayerPlan{ + VStreamFilter: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t2", + Filter: "select hour(c1) as hc1, day(c2) as dc2 from t2", + }}, + }, + TablePlans: map[string]*TablePlan{ + "t2": { + Name: "t1", + ColExprs: []*ColExpr{{ + ColName: sqlparser.NewColIdent("hc1"), + ColNum: 0, + }, { + ColName: sqlparser.NewColIdent("c"), + Operation: OpCount, + }, { + ColName: sqlparser.NewColIdent("dc2"), + ColNum: 1, + }}, + }, + }, + }, + }, { + // sum expr + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select hour(c1) as hc1, sum(c3) as s, day(c2) as dc2 from t2", + }}, + }, + plan: &PlayerPlan{ + VStreamFilter: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t2", + Filter: "select hour(c1) as hc1, c3, day(c2) as dc2 from t2", + }}, + }, + TablePlans: map[string]*TablePlan{ + "t2": { + Name: "t1", + ColExprs: []*ColExpr{{ + ColName: sqlparser.NewColIdent("hc1"), + ColNum: 0, + }, { + ColName: sqlparser.NewColIdent("s"), + ColNum: 1, + Operation: OpSum, + }, { + ColName: sqlparser.NewColIdent("dc2"), + ColNum: 2, + }}, + }, + }, + }, + }, { + // partial group by + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select c1, c2, c3 from t2 group by c3, c1", + }}, + }, + plan: &PlayerPlan{ + VStreamFilter: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t2", + Filter: "select c1, c2, c3 from t2", + }}, + }, + TablePlans: map[string]*TablePlan{ + "t2": { + Name: "t1", + ColExprs: []*ColExpr{{ + ColName: sqlparser.NewColIdent("c1"), + ColNum: 0, + IsGrouped: true, + }, { + ColName: sqlparser.NewColIdent("c2"), + ColNum: 1, + }, { + ColName: sqlparser.NewColIdent("c3"), + ColNum: 2, + IsGrouped: true, + }}, + OnInsert: InsertOndup, + }, + }, + }, + }, { + // full group by + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select c1, c2, c3 from t2 group by c3, c1, c2", + }}, + }, + plan: &PlayerPlan{ + VStreamFilter: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t2", + Filter: "select c1, c2, c3 from t2", + }}, + }, + TablePlans: map[string]*TablePlan{ + "t2": { + Name: "t1", + ColExprs: []*ColExpr{{ + ColName: sqlparser.NewColIdent("c1"), + ColNum: 0, + IsGrouped: true, + }, { + ColName: sqlparser.NewColIdent("c2"), + ColNum: 1, + IsGrouped: true, + }, { + ColName: sqlparser.NewColIdent("c3"), + ColNum: 2, + IsGrouped: true, + }}, + OnInsert: InsertIgnore, + }, + }, + }, + }, { + // syntax error + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "bad query", + }}, + }, + err: "syntax error at position 4 near 'bad'", + }, { + // not a select + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "update t1 set val=1", + }}, + }, + err: "unexpected: update t1 set val = 1", + }, { + // no distinct + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select distinct c1 from t1", + }}, + }, + err: "unexpected: select distinct c1 from t1", + }, { + // no ',' join + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select * from t1, t2", + }}, + }, + err: "unexpected: select * from t1, t2", + }, { + // no join + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select * from t1 join t2", + }}, + }, + err: "unexpected: select * from t1 join t2", + }, { + // no subqueries + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select * from (select * from t2) as a", + }}, + }, + err: "unexpected: select * from (select * from t2) as a", + }, { + // cannot combine '*' with other + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select *, c1 from t1", + }}, + }, + err: "unexpected: select *, c1 from t1", + }, { + // cannot combine '*' with other (different code path) + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select c1, * from t1", + }}, + }, + err: "unexpected: *", + }, { + // no distinct in func + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select hour(distinct c1) from t1", + }}, + }, + err: "unexpected: hour(distinct c1)", + }, { + // funcs need alias + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select hour(c1) from t1", + }}, + }, + err: "expression needs an alias: hour(c1)", + }, { + // only count(*) + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select count(c1) as c from t1", + }}, + }, + err: "only count(*) is supported: count(c1)", + }, { + // no sum(*) + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select sum(*) as c from t1", + }}, + }, + err: "unexpected: sum(*)", + }, { + // no complex expr in sum + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select sum(a + b) as c from t1", + }}, + }, + err: "unexpected: sum(a + b)", + }, { + // unsupported func + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select foo(a) as c from t1", + }}, + }, + err: "unexpected: foo(a)", + }, { + // no complex expr in select + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select a + b from t1", + }}, + }, + err: "unexpected: a + b", + }, { + // no complex expr in group by + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select a from t1 group by a + 1", + }}, + }, + err: "unexpected: a + 1", + }, { + // group by does not reference alias + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select a as b from t1 group by a", + }}, + }, + err: "group by expression does not reference an alias in the select list: a", + }, { + // cannot group by aggr + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select count(*) as a from t1 group by a", + }}, + }, + err: "group by expression is not allowed to reference an aggregate expression: a", + }} + + for _, tcase := range testcases { + plan, err := buildPlayerPlan(tcase.input) + gotPlan, _ := json.Marshal(plan) + wantPlan, _ := json.Marshal(tcase.plan) + if string(gotPlan) != string(wantPlan) { + t.Errorf("Filter(%v):\n%s, want\n%s", tcase.input, gotPlan, wantPlan) + } + gotErr := "" + if err != nil { + gotErr = err.Error() + } + if gotErr != tcase.err { + t.Errorf("Filter err(%v): %s, want %v", tcase.input, gotErr, tcase.err) + } + } +} diff --git a/go/vt/vttablet/tabletmanager/vreplication/relaylog.go b/go/vt/vttablet/tabletmanager/vreplication/relaylog.go new file mode 100644 index 00000000000..f0714390d2e --- /dev/null +++ b/go/vt/vttablet/tabletmanager/vreplication/relaylog.go @@ -0,0 +1,154 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vreplication + +import ( + "io" + "sync" + "time" + + "golang.org/x/net/context" + + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" +) + +type relayLog struct { + ctx context.Context + maxItems int + maxSize int + + // mu controls all variables below and is shared by canAccept and hasItems. + // Broadcasting must be done while holding mu. This is mainly necessary because both + // conditions depend on ctx.Done(), which can change state asynchronously. + mu sync.Mutex + curSize int + items [][]*binlogdatapb.VEvent + timedout bool + err error + // canAccept is true if: curSize<=maxSize, len(items)0, ctx is not Done, and interuptFetch is false. + hasItems sync.Cond +} + +func newRelayLog(ctx context.Context, maxItems, maxSize int) *relayLog { + rl := &relayLog{ + ctx: ctx, + maxItems: maxItems, + maxSize: maxSize, + } + rl.canAccept.L = &rl.mu + rl.hasItems.L = &rl.mu + + // Any time context is done, wake up all waiters to make them exit. + go func() { + <-ctx.Done() + rl.mu.Lock() + defer rl.mu.Unlock() + rl.canAccept.Broadcast() + rl.hasItems.Broadcast() + }() + return rl +} + +func (rl *relayLog) Send(events []*binlogdatapb.VEvent) error { + rl.mu.Lock() + defer rl.mu.Unlock() + + if err := rl.checkDone(); err != nil { + return err + } + for rl.curSize > rl.maxSize || len(rl.items) >= rl.maxItems { + rl.canAccept.Wait() + if err := rl.checkDone(); err != nil { + return err + } + } + rl.items = append(rl.items, events) + rl.curSize += eventsSize(events) + rl.hasItems.Broadcast() + return nil +} + +func (rl *relayLog) Fetch() ([][]*binlogdatapb.VEvent, error) { + rl.mu.Lock() + defer rl.mu.Unlock() + + if err := rl.checkDone(); err != nil { + return nil, err + } + cancelTimer := rl.startTimer() + defer cancelTimer() + for len(rl.items) == 0 && !rl.timedout { + rl.hasItems.Wait() + if err := rl.checkDone(); err != nil { + return nil, err + } + } + rl.timedout = false + items := rl.items + rl.items = nil + rl.curSize = 0 + rl.canAccept.Broadcast() + return items, nil +} + +func (rl *relayLog) checkDone() error { + select { + case <-rl.ctx.Done(): + return io.EOF + default: + } + return nil +} + +func (rl *relayLog) startTimer() (cancel func()) { + timer := time.NewTimer(idleTimeout) + timerDone := make(chan struct{}) + go func() { + select { + case <-timer.C: + rl.mu.Lock() + defer rl.mu.Unlock() + rl.timedout = true + rl.hasItems.Broadcast() + case <-timerDone: + } + }() + return func() { + timer.Stop() + close(timerDone) + } +} + +func eventsSize(events []*binlogdatapb.VEvent) int { + size := 0 + for _, event := range events { + if event.Type != binlogdatapb.VEventType_ROW { + continue + } + for _, rowChange := range event.RowEvent.RowChanges { + if rowChange.Before != nil { + size += len(rowChange.Before.Values) + } + if rowChange.After != nil { + size += len(rowChange.After.Values) + } + } + } + return size +} diff --git a/go/vt/vttablet/tabletmanager/vreplication/retryable_client.go b/go/vt/vttablet/tabletmanager/vreplication/retryable_client.go new file mode 100644 index 00000000000..f5e8eaa4efc --- /dev/null +++ b/go/vt/vttablet/tabletmanager/vreplication/retryable_client.go @@ -0,0 +1,84 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vreplication + +import ( + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/binlog/binlogplayer" +) + +// retryableClient is a wrapper on binlogplayer.DBClient. +// It allows us to retry a failed transactions on lock errors. +type retryableClient struct { + binlogplayer.DBClient + InTransaction bool + queries []string +} + +func (rt *retryableClient) Begin() error { + if rt.InTransaction { + return nil + } + if err := rt.DBClient.Begin(); err != nil { + return err + } + rt.queries = append(rt.queries, "begin") + rt.InTransaction = true + return nil +} + +func (rt *retryableClient) Commit() error { + if err := rt.DBClient.Commit(); err != nil { + return err + } + rt.InTransaction = false + rt.queries = nil + return nil +} + +func (rt *retryableClient) Rollback() error { + if err := rt.DBClient.Rollback(); err != nil { + return err + } + rt.InTransaction = false + // Don't reset queries to allow for vplayer to retry. + return nil +} + +func (rt *retryableClient) ExecuteFetch(query string, maxrows int) (*sqltypes.Result, error) { + if !rt.InTransaction { + rt.queries = []string{query} + } else { + rt.queries = append(rt.queries, query) + } + return rt.DBClient.ExecuteFetch(query, maxrows) +} + +func (rt *retryableClient) Retry() error { + for _, q := range rt.queries { + if q == "begin" { + if err := rt.Begin(); err != nil { + return err + } + continue + } + if _, err := rt.DBClient.ExecuteFetch(q, 10000); err != nil { + return err + } + } + return nil +} diff --git a/go/vt/vttablet/tabletmanager/vreplication/tablet_picker_test.go b/go/vt/vttablet/tabletmanager/vreplication/tablet_picker_test.go index 3bbec75cb5e..e3e4daf5e7b 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/tablet_picker_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/tablet_picker_test.go @@ -17,6 +17,7 @@ limitations under the License. package vreplication import ( + "fmt" "testing" "github.com/golang/protobuf/proto" @@ -26,11 +27,10 @@ import ( ) func TestPickSimple(t *testing.T) { - ts := createTopo() - defer ts.Close() - want := addTablet(ts, 100, "0", topodatapb.TabletType_REPLICA, true, true) + want := addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true) + defer deleteTablet(want) - tp, err := newTabletPicker(ts, testCell, testKeyspace, testShard, "replica") + tp, err := newTabletPicker(env.TopoServ, env.Cells[0], env.KeyspaceName, env.ShardName, "replica") if err != nil { t.Fatal(err) } @@ -46,12 +46,12 @@ func TestPickSimple(t *testing.T) { } func TestPickFromTwoHealthy(t *testing.T) { - ts := createTopo() - defer ts.Close() - want1 := addTablet(ts, 100, "0", topodatapb.TabletType_REPLICA, true, true) - want2 := addTablet(ts, 101, "0", topodatapb.TabletType_RDONLY, true, true) + want1 := addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true) + defer deleteTablet(want1) + want2 := addTablet(101, "0", topodatapb.TabletType_RDONLY, true, true) + defer deleteTablet(want2) - tp, err := newTabletPicker(ts, testCell, testKeyspace, testShard, "replica,rdonly") + tp, err := newTabletPicker(env.TopoServ, env.Cells[0], env.KeyspaceName, env.ShardName, "replica,rdonly") if err != nil { t.Fatal(err) } @@ -65,7 +65,7 @@ func TestPickFromTwoHealthy(t *testing.T) { t.Errorf("Pick:\n%v, want\n%v", tablet, want1) } - tp, err = newTabletPicker(ts, testCell, testKeyspace, testShard, "rdonly,replica") + tp, err = newTabletPicker(env.TopoServ, env.Cells[0], env.KeyspaceName, env.ShardName, "rdonly,replica") if err != nil { t.Fatal(err) } @@ -81,12 +81,11 @@ func TestPickFromTwoHealthy(t *testing.T) { } func TestPickFromSomeUnhealthy(t *testing.T) { - ts := createTopo() - defer ts.Close() - _ = addTablet(ts, 100, "0", topodatapb.TabletType_REPLICA, false, false) - want := addTablet(ts, 101, "0", topodatapb.TabletType_RDONLY, false, true) + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, false, false)) + want := addTablet(101, "0", topodatapb.TabletType_RDONLY, false, true) + defer deleteTablet(want) - tp, err := newTabletPicker(ts, testCell, testKeyspace, testShard, "replica,rdonly") + tp, err := newTabletPicker(env.TopoServ, env.Cells[0], env.KeyspaceName, env.ShardName, "replica,rdonly") if err != nil { t.Fatal(err) } @@ -102,24 +101,22 @@ func TestPickFromSomeUnhealthy(t *testing.T) { } func TestPickError(t *testing.T) { - ts := createTopo() - defer ts.Close() - _ = addTablet(ts, 100, "0", topodatapb.TabletType_REPLICA, false, false) + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, false, false)) - _, err := newTabletPicker(ts, testCell, testKeyspace, testShard, "badtype") + _, err := newTabletPicker(env.TopoServ, env.Cells[0], env.KeyspaceName, env.ShardName, "badtype") want := "failed to parse list of tablet types: badtype" if err == nil || err.Error() != want { t.Errorf("newTabletPicker err: %v, want %v", err, want) } - tp, err := newTabletPicker(ts, testCell, testKeyspace, testShard, "replica,rdonly") + tp, err := newTabletPicker(env.TopoServ, env.Cells[0], env.KeyspaceName, env.ShardName, "replica,rdonly") if err != nil { t.Fatal(err) } defer tp.Close() _, err = tp.Pick(context.Background()) - want = "can't find any healthy source tablet for ks 0 [REPLICA RDONLY]" + want = fmt.Sprintf("can't find any healthy source tablet for %s 0 [REPLICA RDONLY]", env.KeyspaceName) if err == nil || err.Error() != want { t.Errorf("Pick err: %v, want %v", err, want) } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go new file mode 100644 index 00000000000..fd4bc4746e9 --- /dev/null +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go @@ -0,0 +1,641 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vreplication + +import ( + "errors" + "fmt" + "io" + "time" + + "golang.org/x/net/context" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/binlog/binlogplayer" + "vitess.io/vitess/go/vt/grpcclient" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/mysqlctl" + "vitess.io/vitess/go/vt/sqlparser" + "vitess.io/vitess/go/vt/vttablet/tabletconn" + + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" +) + +var ( + idleTimeout = 1 * time.Second + dbLockRetryDelay = 1 * time.Second + relayLogMaxSize = 10000 + relayLogMaxItems = 1000 +) + +type vplayer struct { + id uint32 + source *binlogdatapb.BinlogSource + sourceTablet *topodatapb.Tablet + stats *binlogplayer.Stats + dbClient *retryableClient + // mysqld is used to fetch the local schema. + mysqld mysqlctl.MysqlDaemon + + pos mysql.Position + // unsavedGTID when we receive a GTID event and reset + // if it gets saved. If Fetch returns on idleTimeout, + // we save the last unsavedGTID. + unsavedGTID *binlogdatapb.VEvent + // timeLastSaved is set every time a GTID is saved. + timeLastSaved time.Time + stopPos mysql.Position + + // pplan is built based on the source Filter at the beginning. + pplan *PlayerPlan + // tplans[table] is built for each table based on pplan and schema info + // about the table. + tplans map[string]*TablePlan +} + +func newVPlayer(id uint32, source *binlogdatapb.BinlogSource, sourceTablet *topodatapb.Tablet, stats *binlogplayer.Stats, dbClient binlogplayer.DBClient, mysqld mysqlctl.MysqlDaemon) *vplayer { + return &vplayer{ + id: id, + source: source, + sourceTablet: sourceTablet, + stats: stats, + dbClient: &retryableClient{DBClient: dbClient}, + mysqld: mysqld, + timeLastSaved: time.Now(), + tplans: make(map[string]*TablePlan), + } +} + +func (vp *vplayer) Play(ctx context.Context) error { + if err := vp.setState(binlogplayer.BlpRunning, ""); err != nil { + return err + } + if err := vp.play(ctx); err != nil { + msg := err.Error() + vp.stats.History.Add(&binlogplayer.StatsHistoryRecord{ + Time: time.Now(), + Message: msg, + }) + if err := vp.setState(binlogplayer.BlpError, msg); err != nil { + return err + } + return err + } + return nil +} + +func (vp *vplayer) play(ctx context.Context) error { + startPos, stopPos, _, _, err := binlogplayer.ReadVRSettings(vp.dbClient, vp.id) + if err != nil { + return vp.setState(binlogplayer.BlpStopped, fmt.Sprintf("error reading VReplication settings: %v", err)) + } + vp.pos, err = mysql.DecodePosition(startPos) + if err != nil { + return vp.setState(binlogplayer.BlpStopped, fmt.Sprintf("error decoding start position %v: %v", startPos, err)) + } + if stopPos != "" { + vp.stopPos, err = mysql.DecodePosition(stopPos) + if err != nil { + return vp.setState(binlogplayer.BlpStopped, fmt.Sprintf("error decoding stop position %v: %v", stopPos, err)) + } + } + if !vp.stopPos.IsZero() { + if vp.pos.AtLeast(vp.stopPos) { + return vp.setState(binlogplayer.BlpStopped, fmt.Sprintf("Stop position %v already reached: %v", vp.pos, vp.stopPos)) + } + } + log.Infof("Starting VReplication player id: %v, startPos: %v, stop: %v, source: %v, filter: %v", vp.id, startPos, vp.stopPos, vp.sourceTablet, vp.source) + + plan, err := buildPlayerPlan(vp.source.Filter) + if err != nil { + return err + } + vp.pplan = plan + + vsClient, err := tabletconn.GetDialer()(vp.sourceTablet, grpcclient.FailFast(false)) + if err != nil { + return fmt.Errorf("error dialing tablet: %v", err) + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + relay := newRelayLog(ctx, relayLogMaxItems, relayLogMaxSize) + + target := &querypb.Target{ + Keyspace: vp.sourceTablet.Keyspace, + Shard: vp.sourceTablet.Shard, + TabletType: vp.sourceTablet.Type, + } + log.Infof("Sending vstream command: %v", plan.VStreamFilter) + streamErr := make(chan error, 1) + go func() { + streamErr <- vsClient.VStream(ctx, target, startPos, plan.VStreamFilter, func(events []*binlogdatapb.VEvent) error { + return relay.Send(events) + }) + }() + + applyErr := make(chan error, 1) + go func() { + applyErr <- vp.applyEvents(ctx, relay) + }() + + select { + case err := <-applyErr: + defer func() { + // cancel and wait for the other thread to finish. + cancel() + <-streamErr + }() + // If the apply thread ends with io.EOF, it means either the Engine + // is shutting down and canceled the context, or stop position was reached. + // If so, we return nil which will cause the controller to not retry. + if err == io.EOF { + return nil + } + return err + case err := <-streamErr: + defer func() { + // cancel and wait for the other thread to finish. + cancel() + <-applyErr + }() + // If context is done, don't return an error. + select { + case <-ctx.Done(): + return nil + default: + } + // If the stream ends normally we have to return an error indicating + // that the controller has to retry a different vttablet. + if err == nil || err == io.EOF { + return errors.New("vstream ended") + } + return err + } +} + +func (vp *vplayer) applyEvents(ctx context.Context, relay *relayLog) error { + for { + items, err := relay.Fetch() + if err != nil { + return err + } + // Filtered replication often ends up receiving a large number of empty transactions. + // This is required because the player needs to know the latest position of the source. + // This allows it to stop at that position if requested. + // This position also needs to be saved, which will allow an external request + // to check if a required position has been reached. + // However, this leads to a large number of empty commits which not only slow + // down the replay, but also generate binlog bloat on the target. + // In order to mitigate this problem, empty transactions are saved at most + // once every idleTimeout. + // This covers two situations: + // 1. Fetch was idle for idleTimeout. + // 2. We've been receiving empty events for longer than idleTimeout. + // In both cases, now > timeLastSaved. If so, any unsaved GTID should be saved. + if time.Now().Sub(vp.timeLastSaved) >= idleTimeout && vp.unsavedGTID != nil { + // Although unlikely, we should not save if a transaction is still open. + // This can happen if a large transaction is split as multiple events. + if !vp.dbClient.InTransaction { + if err := vp.updatePos(vp.unsavedGTID.Timestamp); err != nil { + return err + } + } + } + for i, events := range items { + for j, event := range events { + mustSave := false + switch event.Type { + case binlogdatapb.VEventType_COMMIT: + if vp.pos.Equal(vp.stopPos) { + mustSave = true + break + } + if hasAnotherCommit(items, i, j+1) { + continue + } + } + if err := vp.applyEvent(ctx, event, mustSave); err != nil { + return err + } + } + } + } +} + +func hasAnotherCommit(items [][]*binlogdatapb.VEvent, i, j int) bool { + for i < len(items) { + for j < len(items[i]) { + // We ignore GTID, BEGIN, FIELD and ROW. + switch items[i][j].Type { + case binlogdatapb.VEventType_COMMIT: + return true + case binlogdatapb.VEventType_DDL: + return false + } + j++ + } + j = 0 + i++ + } + return false +} + +func (vp *vplayer) applyEvent(ctx context.Context, event *binlogdatapb.VEvent, mustSave bool) error { + switch event.Type { + case binlogdatapb.VEventType_GTID: + pos, err := mysql.DecodePosition(event.Gtid) + if err != nil { + return err + } + vp.pos = pos + vp.unsavedGTID = event + if vp.stopPos.IsZero() { + return nil + } + if !vp.pos.Equal(vp.stopPos) && vp.pos.AtLeast(vp.stopPos) { + // Code is unreachable, but bad data can cause this to happen. + if err := vp.setState(binlogplayer.BlpStopped, fmt.Sprintf("next event position %v exceeds stop pos %v, exiting without applying", vp.pos, vp.stopPos)); err != nil { + return err + } + return io.EOF + } + case binlogdatapb.VEventType_BEGIN: + // No-op: begin is called as needed. + case binlogdatapb.VEventType_COMMIT: + if mustSave { + if err := vp.dbClient.Begin(); err != nil { + return err + } + } + + if !vp.dbClient.InTransaction { + return nil + } + if err := vp.updatePos(event.Timestamp); err != nil { + return err + } + posReached := !vp.stopPos.IsZero() && vp.pos.Equal(vp.stopPos) + if posReached { + if err := vp.setState(binlogplayer.BlpStopped, fmt.Sprintf("Stopped at position %v", vp.stopPos)); err != nil { + return err + } + } + if err := vp.dbClient.Commit(); err != nil { + return err + } + if posReached { + return io.EOF + } + case binlogdatapb.VEventType_FIELD: + if err := vp.dbClient.Begin(); err != nil { + return err + } + if err := vp.updatePlan(event.FieldEvent); err != nil { + return err + } + case binlogdatapb.VEventType_ROW: + if err := vp.dbClient.Begin(); err != nil { + return err + } + if err := vp.applyRowEvent(ctx, event.RowEvent); err != nil { + return err + } + case binlogdatapb.VEventType_DDL: + if vp.dbClient.InTransaction { + return fmt.Errorf("unexpected state: DDL encountered in the middle of a transaction: %v", event.Ddl) + } + switch vp.source.OnDdl { + case binlogdatapb.OnDDLAction_IGNORE: + // no-op + case binlogdatapb.OnDDLAction_STOP: + if err := vp.dbClient.Begin(); err != nil { + return err + } + if err := vp.updatePos(event.Timestamp); err != nil { + return err + } + if err := vp.setState(binlogplayer.BlpStopped, fmt.Sprintf("Stopped at DDL %s", event.Ddl)); err != nil { + return err + } + if err := vp.dbClient.Commit(); err != nil { + return err + } + return io.EOF + case binlogdatapb.OnDDLAction_EXEC: + if err := vp.exec(ctx, event.Ddl); err != nil { + return err + } + if err := vp.updatePos(event.Timestamp); err != nil { + return err + } + case binlogdatapb.OnDDLAction_EXEC_IGNORE: + if err := vp.exec(ctx, event.Ddl); err != nil { + log.Infof("Ignoring error: %v for DDL: %s", err, event.Ddl) + } + if err := vp.updatePos(event.Timestamp); err != nil { + return err + } + } + } + return nil +} + +func (vp *vplayer) setState(state, message string) error { + return binlogplayer.SetVReplicationState(vp.dbClient, vp.id, state, message) +} + +func (vp *vplayer) updatePlan(fieldEvent *binlogdatapb.FieldEvent) error { + prelim := vp.pplan.TablePlans[fieldEvent.TableName] + tplan := &TablePlan{ + Name: fieldEvent.TableName, + } + if prelim != nil { + *tplan = *prelim + } + tplan.Fields = fieldEvent.Fields + + if tplan.ColExprs == nil { + tplan.ColExprs = make([]*ColExpr, len(tplan.Fields)) + for i, field := range tplan.Fields { + tplan.ColExprs[i] = &ColExpr{ + ColName: sqlparser.NewColIdent(field.Name), + ColNum: i, + } + } + } else { + for _, cExpr := range tplan.ColExprs { + if cExpr.ColNum >= len(tplan.Fields) { + // Unreachable code. + return fmt.Errorf("columns received from vreplication: %v, do not match expected: %v", tplan.Fields, tplan.ColExprs) + } + } + } + + pkcols, err := vp.mysqld.GetPrimaryKeyColumns(vp.dbClient.DBName(), tplan.Name) + if err != nil { + return fmt.Errorf("error fetching pk columns for %s: %v", tplan.Name, err) + } + if len(pkcols) == 0 { + // If the table doesn't have a PK, then we treat all columns as PK. + pkcols, err = vp.mysqld.GetColumns(vp.dbClient.DBName(), tplan.Name) + if err != nil { + return fmt.Errorf("error fetching pk columns for %s: %v", tplan.Name, err) + } + } + for _, pkcol := range pkcols { + found := false + for i, cExpr := range tplan.ColExprs { + if cExpr.ColName.EqualString(pkcol) { + found = true + tplan.PKCols = append(tplan.PKCols, &ColExpr{ + ColName: cExpr.ColName, + ColNum: i, + }) + break + } + } + if !found { + return fmt.Errorf("primary key column %s missing from select list for table %s", pkcol, tplan.Name) + } + } + vp.tplans[fieldEvent.TableName] = tplan + return nil +} + +func (vp *vplayer) applyRowEvent(ctx context.Context, rowEvent *binlogdatapb.RowEvent) error { + tplan := vp.tplans[rowEvent.TableName] + if tplan == nil { + return fmt.Errorf("unexpected event on table %s", rowEvent.TableName) + } + for _, change := range rowEvent.RowChanges { + if err := vp.applyRowChange(ctx, tplan, change); err != nil { + return err + } + } + return nil +} + +func (vp *vplayer) applyRowChange(ctx context.Context, tplan *TablePlan, rowChange *binlogdatapb.RowChange) error { + // MakeRowTrusted is needed here because because Proto3ToResult is not convenient. + var before, after []sqltypes.Value + if rowChange.Before != nil { + before = sqltypes.MakeRowTrusted(tplan.Fields, rowChange.Before) + } + if rowChange.After != nil { + after = sqltypes.MakeRowTrusted(tplan.Fields, rowChange.After) + } + var query string + switch { + case before == nil && after != nil: + query = vp.generateInsert(tplan, after) + case before != nil && after != nil: + query = vp.generateUpdate(tplan, before, after) + case before != nil && after == nil: + query = vp.generateDelete(tplan, before) + case before == nil && after == nil: + // unreachable + } + if query == "" { + return nil + } + return vp.exec(ctx, query) +} + +func (vp *vplayer) generateInsert(tplan *TablePlan, after []sqltypes.Value) string { + sql := sqlparser.NewTrackedBuffer(nil) + if tplan.OnInsert == InsertIgnore { + sql.Myprintf("insert ignore into %v set ", sqlparser.NewTableIdent(tplan.Name)) + } else { + sql.Myprintf("insert into %v set ", sqlparser.NewTableIdent(tplan.Name)) + } + vp.writeInsertValues(sql, tplan, after) + if tplan.OnInsert == InsertOndup { + sql.Myprintf(" on duplicate key update ") + _ = vp.writeUpdateValues(sql, tplan, nil, after) + } + return sql.String() +} + +func (vp *vplayer) generateUpdate(tplan *TablePlan, before, after []sqltypes.Value) string { + if tplan.OnInsert == InsertIgnore { + return vp.generateInsert(tplan, after) + } + sql := sqlparser.NewTrackedBuffer(nil) + sql.Myprintf("update %v set ", sqlparser.NewTableIdent(tplan.Name)) + if ok := vp.writeUpdateValues(sql, tplan, before, after); !ok { + return "" + } + sql.Myprintf(" where ") + vp.writeWhereValues(sql, tplan, before) + return sql.String() +} + +func (vp *vplayer) generateDelete(tplan *TablePlan, before []sqltypes.Value) string { + sql := sqlparser.NewTrackedBuffer(nil) + switch tplan.OnInsert { + case InsertOndup: + return vp.generateUpdate(tplan, before, nil) + case InsertIgnore: + return "" + default: // insertNormal + sql.Myprintf("delete from %v where ", sqlparser.NewTableIdent(tplan.Name)) + vp.writeWhereValues(sql, tplan, before) + } + return sql.String() +} + +func (vp *vplayer) writeInsertValues(sql *sqlparser.TrackedBuffer, tplan *TablePlan, after []sqltypes.Value) { + separator := "" + for _, cExpr := range tplan.ColExprs { + sql.Myprintf("%s%v=", separator, cExpr.ColName) + separator = ", " + if cExpr.Operation == OpCount { + sql.WriteString("1") + } else { + if cExpr.Operation == OpSum && after[cExpr.ColNum].IsNull() { + sql.WriteString("0") + } else { + encodeValue(sql, after[cExpr.ColNum]) + } + } + } +} + +// writeUpdateValues returns true if at least one value was set. Otherwise, it returns false. +func (vp *vplayer) writeUpdateValues(sql *sqlparser.TrackedBuffer, tplan *TablePlan, before, after []sqltypes.Value) bool { + separator := "" + hasSet := false + for _, cExpr := range tplan.ColExprs { + if cExpr.IsGrouped { + continue + } + if len(before) != 0 && len(after) != 0 { + if cExpr.Operation == OpCount { + continue + } + bef := before[cExpr.ColNum] + aft := after[cExpr.ColNum] + // If both are null, there's no change + if bef.IsNull() && aft.IsNull() { + continue + } + // If any one of them is null, something has changed. + if bef.IsNull() || aft.IsNull() { + goto mustSet + } + // Compare content only if none are null. + if bef.ToString() == aft.ToString() { + continue + } + } + mustSet: + sql.Myprintf("%s%v=", separator, cExpr.ColName) + separator = ", " + hasSet = true + if cExpr.Operation == OpCount || cExpr.Operation == OpSum { + sql.Myprintf("%v", cExpr.ColName) + } + if len(before) != 0 { + switch cExpr.Operation { + case OpNone: + if len(after) == 0 { + sql.WriteString("NULL") + } + case OpCount: + sql.WriteString("-1") + case OpSum: + if !before[cExpr.ColNum].IsNull() { + sql.WriteString("-") + encodeValue(sql, before[cExpr.ColNum]) + } + } + } + if len(after) != 0 { + switch cExpr.Operation { + case OpNone: + encodeValue(sql, after[cExpr.ColNum]) + case OpCount: + sql.WriteString("+1") + case OpSum: + if !after[cExpr.ColNum].IsNull() { + sql.WriteString("+") + encodeValue(sql, after[cExpr.ColNum]) + } + } + } + } + return hasSet +} + +func (vp *vplayer) writeWhereValues(sql *sqlparser.TrackedBuffer, tplan *TablePlan, before []sqltypes.Value) { + separator := "" + for _, cExpr := range tplan.PKCols { + sql.Myprintf("%s%v=", separator, cExpr.ColName) + separator = " and " + encodeValue(sql, before[cExpr.ColNum]) + } +} + +func (vp *vplayer) updatePos(ts int64) error { + updatePos := binlogplayer.GenerateUpdatePos(vp.id, vp.pos, time.Now().Unix(), ts) + if _, err := vp.dbClient.ExecuteFetch(updatePos, 0); err != nil { + vp.dbClient.Rollback() + return fmt.Errorf("error %v updating position", err) + } + vp.unsavedGTID = nil + vp.timeLastSaved = time.Now() + return nil +} + +func (vp *vplayer) exec(ctx context.Context, sql string) error { + vp.stats.Timings.Record("query", time.Now()) + _, err := vp.dbClient.ExecuteFetch(sql, 0) + for err != nil { + // 1213: deadlock, 1205: lock wait timeout + if sqlErr, ok := err.(*mysql.SQLError); ok && sqlErr.Number() == 1213 || sqlErr.Number() == 1205 { + log.Infof("retryable error: %v, waiting for %v and retrying", sqlErr, dbLockRetryDelay) + if err := vp.dbClient.Rollback(); err != nil { + return err + } + time.Sleep(dbLockRetryDelay) + // Check context here. Otherwise this can become an infinite loop. + select { + case <-ctx.Done(): + return io.EOF + default: + } + err = vp.dbClient.Retry() + continue + } + return err + } + return nil +} + +func encodeValue(sql *sqlparser.TrackedBuffer, value sqltypes.Value) { + // This is currently a separate function because special handling + // may be needed for certain types. + // Previously, this function used to convert timestamp to the session + // time zone, but we now set the session timezone to UTC. So, the timestamp + // value we receive as UTC can be sent as is. + // TODO(sougou): handle BIT data type here? + value.EncodeSQL(sql) +} diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go new file mode 100644 index 00000000000..1996015d82a --- /dev/null +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go @@ -0,0 +1,1270 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vreplication + +import ( + "flag" + "fmt" + "strings" + "testing" + "time" + + "golang.org/x/net/context" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/vt/binlog/binlogplayer" + + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" +) + +func TestPlayerFilters(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + execStatements(t, []string{ + "create table src1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.dst1(id int, val varbinary(128), primary key(id))", vrepldb), + "create table src2(id int, val1 int, val2 int, primary key(id))", + fmt.Sprintf("create table %s.dst2(id int, val1 int, sval2 int, rcount int, primary key(id))", vrepldb), + "create table src3(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.dst3(id int, val varbinary(128), primary key(id))", vrepldb), + "create table yes(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.yes(id int, val varbinary(128), primary key(id))", vrepldb), + "create table no(id int, val varbinary(128), primary key(id))", + "create table nopk(id int, val varbinary(128))", + fmt.Sprintf("create table %s.nopk(id int, val varbinary(128))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table src1", + fmt.Sprintf("drop table %s.dst1", vrepldb), + "drop table src2", + fmt.Sprintf("drop table %s.dst2", vrepldb), + "drop table src3", + fmt.Sprintf("drop table %s.dst3", vrepldb), + "drop table yes", + fmt.Sprintf("drop table %s.yes", vrepldb), + "drop table no", + "drop table nopk", + fmt.Sprintf("drop table %s.nopk", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "dst1", + Filter: "select * from src1", + }, { + Match: "dst2", + Filter: "select id, val1, sum(val2) as sval2, count(*) as rcount from src2 group by id", + }, { + Match: "dst3", + Filter: "select id, val from src3 group by id, val", + }, { + Match: "/yes", + }, { + Match: "/nopk", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + + testcases := []struct { + input string + output []string + table string + data [][]string + }{{ + // insert with insertNormal + input: "insert into src1 values(1, 'aaa')", + output: []string{ + "begin", + "insert into dst1 set id=1, val='aaa'", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst1", + data: [][]string{ + {"1", "aaa"}, + }, + }, { + // update with insertNormal + input: "update src1 set val='bbb'", + output: []string{ + "begin", + "update dst1 set val='bbb' where id=1", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst1", + data: [][]string{ + {"1", "bbb"}, + }, + }, { + // delete with insertNormal + input: "delete from src1 where id=1", + output: []string{ + "begin", + "delete from dst1 where id=1", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst1", + data: [][]string{}, + }, { + // insert with insertOnDup + input: "insert into src2 values(1, 2, 3)", + output: []string{ + "begin", + "insert into dst2 set id=1, val1=2, sval2=3, rcount=1 on duplicate key update val1=2, sval2=sval2+3, rcount=rcount+1", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst2", + data: [][]string{ + {"1", "2", "3", "1"}, + }, + }, { + // update with insertOnDup + input: "update src2 set val1=5, val2=1 where id=1", + output: []string{ + "begin", + "update dst2 set val1=5, sval2=sval2-3+1 where id=1", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst2", + data: [][]string{ + {"1", "5", "1", "1"}, + }, + }, { + // delete with insertOnDup + input: "delete from src2 where id=1", + output: []string{ + "begin", + "update dst2 set val1=NULL, sval2=sval2-1, rcount=rcount-1 where id=1", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst2", + data: [][]string{ + {"1", "", "0", "0"}, + }, + }, { + // insert with insertIgnore + input: "insert into src3 values(1, 'aaa')", + output: []string{ + "begin", + "insert ignore into dst3 set id=1, val='aaa'", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst3", + data: [][]string{ + {"1", "aaa"}, + }, + }, { + // update with insertIgnore + input: "update src3 set val='bbb'", + output: []string{ + "begin", + "insert ignore into dst3 set id=1, val='bbb'", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst3", + data: [][]string{ + {"1", "aaa"}, + }, + }, { + // delete with insertIgnore + input: "delete from src3 where id=1", + output: []string{ + "begin", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst3", + data: [][]string{ + {"1", "aaa"}, + }, + }, { + // insert: regular expression filter + input: "insert into yes values(1, 'aaa')", + output: []string{ + "begin", + "insert into yes set id=1, val='aaa'", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "yes", + data: [][]string{ + {"1", "aaa"}, + }, + }, { + // update: regular expression filter + input: "update yes set val='bbb'", + output: []string{ + "begin", + "update yes set val='bbb' where id=1", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "yes", + data: [][]string{ + {"1", "bbb"}, + }, + }, { + // table should not match a rule + input: "insert into no values(1, 'aaa')", + output: []string{}, + }, { + // nopk: insert + input: "insert into nopk values(1, 'aaa')", + output: []string{ + "begin", + "insert into nopk set id=1, val='aaa'", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "nopk", + data: [][]string{ + {"1", "aaa"}, + }, + }, { + // nopk: update + input: "update nopk set val='bbb' where id=1", + output: []string{ + "begin", + "update nopk set val='bbb' where id=1 and val='aaa'", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "nopk", + data: [][]string{ + {"1", "bbb"}, + }, + }, { + // nopk: delete + input: "delete from nopk where id=1", + output: []string{ + "begin", + "delete from nopk where id=1 and val='bbb'", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "nopk", + data: [][]string{}, + }} + + for _, tcases := range testcases { + execStatements(t, []string{tcases.input}) + expectDBClientQueries(t, tcases.output) + if tcases.table != "" { + expectData(t, tcases.table, tcases.data) + } + } +} + +func TestPlayerUpdates(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + execStatements(t, []string{ + "create table t1(id int, grouped int, ungrouped int, summed int, primary key(id))", + fmt.Sprintf("create table %s.t1(id int, grouped int, ungrouped int, summed int, rcount int, primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select id, grouped, ungrouped, sum(summed) as summed, count(*) as rcount from t1 group by id, grouped", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + + testcases := []struct { + input string + output string + table string + data [][]string + }{{ + // Start with all nulls + input: "insert into t1 values(1, null, null, null)", + output: "insert into t1 set id=1, grouped=null, ungrouped=null, summed=0, rcount=1 on duplicate key update ungrouped=null, summed=summed, rcount=rcount+1", + table: "t1", + data: [][]string{ + {"1", "", "", "0", "1"}, + }, + }, { + // null to null values + input: "update t1 set grouped=1 where id=1", + output: "", + table: "t1", + data: [][]string{ + {"1", "", "", "0", "1"}, + }, + }, { + // null to non-null values + input: "update t1 set ungrouped=1, summed=1 where id=1", + output: "update t1 set ungrouped=1, summed=summed+1 where id=1", + table: "t1", + data: [][]string{ + {"1", "", "1", "1", "1"}, + }, + }, { + // non-null to non-null values + input: "update t1 set ungrouped=2, summed=2 where id=1", + output: "update t1 set ungrouped=2, summed=summed-1+2 where id=1", + table: "t1", + data: [][]string{ + {"1", "", "2", "2", "1"}, + }, + }, { + // non-null to null values + input: "update t1 set ungrouped=null, summed=null where id=1", + output: "update t1 set ungrouped=null, summed=summed-2 where id=1", + table: "t1", + data: [][]string{ + {"1", "", "", "0", "1"}, + }, + }, { + // insert non-null values + input: "insert into t1 values(2, 2, 3, 4)", + output: "insert into t1 set id=2, grouped=2, ungrouped=3, summed=4, rcount=1 on duplicate key update ungrouped=3, summed=summed+4, rcount=rcount+1", + table: "t1", + data: [][]string{ + {"1", "", "", "0", "1"}, + {"2", "2", "3", "4", "1"}, + }, + }, { + // delete non-null values + input: "delete from t1 where id=2", + output: "update t1 set ungrouped=NULL, summed=summed-4, rcount=rcount-1 where id=2", + table: "t1", + data: [][]string{ + {"1", "", "", "0", "1"}, + {"2", "2", "", "0", "0"}, + }, + }} + + for _, tcases := range testcases { + execStatements(t, []string{tcases.input}) + output := []string{ + "begin", + tcases.output, + "/update _vt.vreplication set pos=", + "commit", + } + if tcases.output == "" { + output = []string{ + "begin", + "/update _vt.vreplication set pos=", + "commit", + } + } + expectDBClientQueries(t, output) + if tcases.table != "" { + expectData(t, tcases.table, tcases.data) + } + } +} + +func TestPlayerTypes(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + execStatements(t, []string{ + "create table vitess_ints(tiny tinyint, tinyu tinyint unsigned, small smallint, smallu smallint unsigned, medium mediumint, mediumu mediumint unsigned, normal int, normalu int unsigned, big bigint, bigu bigint unsigned, y year, primary key(tiny))", + fmt.Sprintf("create table %s.vitess_ints(tiny tinyint, tinyu tinyint unsigned, small smallint, smallu smallint unsigned, medium mediumint, mediumu mediumint unsigned, normal int, normalu int unsigned, big bigint, bigu bigint unsigned, y year, primary key(tiny))", vrepldb), + "create table vitess_fracts(id int, deci decimal(5,2), num numeric(5,2), f float, d double, primary key(id))", + fmt.Sprintf("create table %s.vitess_fracts(id int, deci decimal(5,2), num numeric(5,2), f float, d double, primary key(id))", vrepldb), + "create table vitess_strings(vb varbinary(16), c char(16), vc varchar(16), b binary(4), tb tinyblob, bl blob, ttx tinytext, tx text, en enum('a','b'), s set('a','b'), primary key(vb))", + fmt.Sprintf("create table %s.vitess_strings(vb varbinary(16), c char(16), vc varchar(16), b binary(4), tb tinyblob, bl blob, ttx tinytext, tx text, en enum('a','b'), s set('a','b'), primary key(vb))", vrepldb), + "create table vitess_misc(id int, b bit(8), d date, dt datetime, t time, g geometry, primary key(id))", + fmt.Sprintf("create table %s.vitess_misc(id int, b bit(8), d date, dt datetime, t time, g geometry, primary key(id))", vrepldb), + "create table vitess_null(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.vitess_null(id int, val varbinary(128), primary key(id))", vrepldb), + "create table src1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.src1(id int, val varbinary(128), primary key(id))", vrepldb), + "create table binary_pk(b binary(4), val varbinary(4), primary key(b))", + fmt.Sprintf("create table %s.binary_pk(b binary(4), val varbinary(4), primary key(b))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table vitess_ints", + fmt.Sprintf("drop table %s.vitess_ints", vrepldb), + "drop table vitess_fracts", + fmt.Sprintf("drop table %s.vitess_fracts", vrepldb), + "drop table vitess_strings", + fmt.Sprintf("drop table %s.vitess_strings", vrepldb), + "drop table vitess_misc", + fmt.Sprintf("drop table %s.vitess_misc", vrepldb), + "drop table vitess_null", + fmt.Sprintf("drop table %s.vitess_null", vrepldb), + "drop table binary_pk", + fmt.Sprintf("drop table %s.binary_pk", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + testcases := []struct { + input string + output string + table string + data [][]string + }{{ + input: "insert into vitess_ints values(-128, 255, -32768, 65535, -8388608, 16777215, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, 2012)", + output: "insert into vitess_ints set tiny=-128, tinyu=255, small=-32768, smallu=65535, medium=-8388608, mediumu=16777215, normal=-2147483648, normalu=4294967295, big=-9223372036854775808, bigu=18446744073709551615, y=2012", + table: "vitess_ints", + data: [][]string{ + {"-128", "255", "-32768", "65535", "-8388608", "16777215", "-2147483648", "4294967295", "-9223372036854775808", "18446744073709551615", "2012"}, + }, + }, { + input: "insert into vitess_fracts values(1, 1.99, 2.99, 3.99, 4.99)", + output: "insert into vitess_fracts set id=1, deci=1.99, num=2.99, f=3.99E+00, d=4.99E+00", + table: "vitess_fracts", + data: [][]string{ + {"1", "1.99", "2.99", "3.99", "4.99"}, + }, + }, { + input: "insert into vitess_strings values('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'a', 'a,b')", + output: "insert into vitess_strings set vb='a', c='b', vc='c', b='d\\0\\0\\0', tb='e', bl='f', ttx='g', tx='h', en='1', s='3'", + table: "vitess_strings", + data: [][]string{ + {"a", "b", "c", "d\x00\x00\x00", "e", "f", "g", "h", "a", "a,b"}, + }, + }, { + input: "insert into vitess_misc values(1, '\x01', '2012-01-01', '2012-01-01 15:45:45', '15:45:45', point(1, 2))", + output: "insert into vitess_misc set id=1, b=b'00000001', d='2012-01-01', dt='2012-01-01 15:45:45', t='15:45:45', g='\\0\\0\\0\\0\x01\x01\\0\\0\\0\\0\\0\\0\\0\\0\\0\xf0?\\0\\0\\0\\0\\0\\0\\0@'", + table: "vitess_misc", + data: [][]string{ + {"1", "\x01", "2012-01-01", "2012-01-01 15:45:45", "15:45:45", "\x00\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00\x00@"}, + }, + }, { + input: "insert into vitess_null values(1, null)", + output: "insert into vitess_null set id=1, val=null", + table: "vitess_null", + data: [][]string{ + {"1", ""}, + }, + }, { + input: "insert into binary_pk values('a', 'aaa')", + output: "insert into binary_pk set b='a\\0\\0\\0', val='aaa'", + table: "binary_pk", + data: [][]string{ + {"a\x00\x00\x00", "aaa"}, + }, + }, { + // Binary pk is a special case: https://github.com/vitessio/vitess/issues/3984 + input: "update binary_pk set val='bbb' where b='a\\0\\0\\0'", + output: "update binary_pk set val='bbb' where b='a\\0\\0\\0'", + table: "binary_pk", + data: [][]string{ + {"a\x00\x00\x00", "bbb"}, + }, + }} + + for _, tcases := range testcases { + execStatements(t, []string{tcases.input}) + want := []string{ + "begin", + tcases.output, + "/update _vt.vreplication set pos=", + "commit", + } + expectDBClientQueries(t, want) + if tcases.table != "" { + expectData(t, tcases.table, tcases.data) + } + } +} + +func TestPlayerDDL(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + execStatements(t, []string{ + "create table dummy(id int, primary key(id))", + fmt.Sprintf("create table %s.dummy(id int, primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table dummy", + fmt.Sprintf("drop table %s.dummy", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + // Issue a dummy change to ensure vreplication is initialized. Otherwise there + // is a race between the DDLs and the schema loader of vstreamer. + // Root cause seems to be with MySQL where t1 shows up in information_schema before + // the actual table is created. + execStatements(t, []string{"insert into dummy values(1)"}) + expectDBClientQueries(t, []string{ + "begin", + "insert into dummy set id=1", + "/update _vt.vreplication set pos=", + "commit", + }) + + execStatements(t, []string{"create table t1(id int, primary key(id))"}) + execStatements(t, []string{"drop table t1"}) + expectDBClientQueries(t, []string{}) + cancel() + + cancel, id := startVReplication(t, filter, binlogdatapb.OnDDLAction_STOP, "") + execStatements(t, []string{"create table t1(id int, primary key(id))"}) + pos1 := masterPosition(t) + execStatements(t, []string{"drop table t1"}) + pos2 := masterPosition(t) + // The stop position must be the GTID of the first DDL + expectDBClientQueries(t, []string{ + "begin", + fmt.Sprintf("/update _vt.vreplication set pos='%s'", pos1), + "/update _vt.vreplication set state='Stopped'", + "commit", + }) + // Restart vreplication + if _, err := playerEngine.Exec(fmt.Sprintf(`update _vt.vreplication set state = 'Running', message='' where id=%d`, id)); err != nil { + t.Fatal(err) + } + // It should stop at the next DDL + expectDBClientQueries(t, []string{ + "/update.*'Running'", + "/update.*'Running'", + "begin", + fmt.Sprintf("/update.*'%s'", pos2), + "/update _vt.vreplication set state='Stopped'", + "commit", + }) + cancel() + + execStatements(t, []string{fmt.Sprintf("create table %s.t2(id int, primary key(id))", vrepldb)}) + cancel, _ = startVReplication(t, filter, binlogdatapb.OnDDLAction_EXEC, "") + execStatements(t, []string{"create table t1(id int, primary key(id))"}) + expectDBClientQueries(t, []string{ + "create table t1(id int, primary key(id))", + "/update _vt.vreplication set pos=", + }) + execStatements(t, []string{"create table t2(id int, primary key(id))"}) + expectDBClientQueries(t, []string{ + "create table t2(id int, primary key(id))", + "/update _vt.vreplication set state='Error'", + }) + cancel() + + // Don't test drop. + // MySQL rewrites them by uppercasing, which may be version specific. + execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + "drop table t2", + fmt.Sprintf("drop table %s.t2", vrepldb), + }) + + execStatements(t, []string{fmt.Sprintf("create table %s.t2(id int, primary key(id))", vrepldb)}) + cancel, _ = startVReplication(t, filter, binlogdatapb.OnDDLAction_EXEC_IGNORE, "") + execStatements(t, []string{"create table t1(id int, primary key(id))"}) + expectDBClientQueries(t, []string{ + "create table t1(id int, primary key(id))", + "/update _vt.vreplication set pos=", + }) + execStatements(t, []string{"create table t2(id int, primary key(id))"}) + expectDBClientQueries(t, []string{ + "create table t2(id int, primary key(id))", + "/update _vt.vreplication set pos=", + }) + cancel() + + execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + "drop table t2", + fmt.Sprintf("drop table %s.t2", vrepldb), + }) +} + +func TestPlayerStopPos(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + execStatements(t, []string{ + "create table yes(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.yes(id int, val varbinary(128), primary key(id))", vrepldb), + "create table no(id int, val varbinary(128), primary key(id))", + }) + defer execStatements(t, []string{ + "drop table yes", + fmt.Sprintf("drop table %s.yes", vrepldb), + "drop table no", + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/yes", + }}, + } + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + startPos := masterPosition(t) + query := binlogplayer.CreateVReplicationStopped("test", bls, startPos) + qr, err := playerEngine.Exec(query) + if err != nil { + t.Fatal(err) + } + id := uint32(qr.InsertID) + for q := range globalDBQueries { + if strings.HasPrefix(q, "insert into _vt.vreplication") { + break + } + } + + // Test normal stop. + execStatements(t, []string{ + "insert into yes values(1, 'aaa')", + }) + stopPos := masterPosition(t) + query = binlogplayer.StartVReplicationUntil(id, stopPos) + if _, err := playerEngine.Exec(query); err != nil { + t.Fatal(err) + } + expectDBClientQueries(t, []string{ + "/update.*'Running'", // done by Engine + "/update.*'Running'", // done by vplayer on start + "begin", + "insert into yes set id=1, val='aaa'", + fmt.Sprintf("/update.*'%s'", stopPos), + "/update.*'Stopped'", + "commit", + }) + + // Test stopping at empty transaction. + execStatements(t, []string{ + "insert into no values(2, 'aaa')", + "insert into no values(3, 'aaa')", + }) + stopPos = masterPosition(t) + execStatements(t, []string{ + "insert into no values(4, 'aaa')", + }) + query = binlogplayer.StartVReplicationUntil(id, stopPos) + if _, err := playerEngine.Exec(query); err != nil { + t.Fatal(err) + } + expectDBClientQueries(t, []string{ + "/update.*'Running'", // done by Engine + "/update.*'Running'", // done by vplayer on start + "begin", + // Since 'no' generates empty transactions that are skipped by + // vplayer, a commit is done only for the stop position event. + fmt.Sprintf("/update.*'%s'", stopPos), + "/update.*'Stopped'", + "commit", + }) + + // Test stopping when position is already reached. + query = binlogplayer.StartVReplicationUntil(id, stopPos) + if _, err := playerEngine.Exec(query); err != nil { + t.Fatal(err) + } + expectDBClientQueries(t, []string{ + "/update.*'Running'", // done by Engine + "/update.*'Running'", // done by vplayer on start + "/update.*'Stopped'.*already reached", + }) +} + +func TestPlayerIdleUpdate(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + savedIdleTimeout := idleTimeout + defer func() { idleTimeout = savedIdleTimeout }() + idleTimeout = 100 * time.Millisecond + + execStatements(t, []string{ + "create table t1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + + execStatements(t, []string{ + "insert into t1 values(1, 'aaa')", + }) + start := time.Now() + expectDBClientQueries(t, []string{ + "begin", + "insert into t1 set id=1, val='aaa'", + "/update _vt.vreplication set pos=", + "commit", + }) + // The above write will generate a new binlog event, and + // that event will loopback into player as an empty event. + // But it must not get saved until idleTimeout has passed. + // The exact positions are hard to verify because of this + // loopback mechanism. + expectDBClientQueries(t, []string{ + "/update _vt.vreplication set pos=", + }) + if duration := time.Now().Sub(start); duration < idleTimeout { + t.Errorf("duration: %v, must be at least %v", duration, idleTimeout) + } +} + +func TestPlayerSplitTransaction(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + flag.Set("vstream_packet_size", "10") + defer flag.Set("vstream_packet_size", "10000") + + execStatements(t, []string{ + "create table t1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + + execStatements(t, []string{ + "begin", + "insert into t1 values(1, '123456')", + "insert into t1 values(2, '789012')", + "commit", + }) + // Because the packet size is 10, this is received as two events, + // but still combined as one transaction. + expectDBClientQueries(t, []string{ + "begin", + "insert into t1 set id=1, val='123456'", + "insert into t1 set id=2, val='789012'", + "/update _vt.vreplication set pos=", + "commit", + }) +} + +func TestPlayerLockErrors(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + execStatements(t, []string{ + "create table t1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + + execStatements(t, []string{ + "begin", + "insert into t1 values(1, 'aaa')", + "insert into t1 values(2, 'bbb')", + "commit", + }) + expectDBClientQueries(t, []string{ + "begin", + "insert into t1 set id=1, val='aaa'", + "insert into t1 set id=2, val='bbb'", + "/update _vt.vreplication set pos=", + "commit", + }) + + vconn := &realDBClient{nolog: true} + if err := vconn.Connect(); err != nil { + t.Error(err) + } + defer vconn.Close() + + // Start a transaction and lock the second row. + if _, err := vconn.ExecuteFetch("begin", 1); err != nil { + t.Error(err) + } + if _, err := vconn.ExecuteFetch("update t1 set val='bbb' where id=2", 1); err != nil { + t.Error(err) + } + + execStatements(t, []string{ + "begin", + "update t1 set val='ccc' where id=1", + "update t1 set val='ccc' where id=2", + "commit", + }) + // The innodb lock wait timeout is set to 1s. + expectDBClientQueries(t, []string{ + "begin", + "update t1 set val='ccc' where id=1", + "update t1 set val='ccc' where id=2", + "rollback", + }) + + // Release the lock, and watch the retry go through. + _, _ = vconn.ExecuteFetch("rollback", 1) + expectDBClientQueries(t, []string{ + "begin", + "update t1 set val='ccc' where id=1", + "update t1 set val='ccc' where id=2", + "/update _vt.vreplication set pos=", + "commit", + }) +} + +func TestPlayerCancelOnLock(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + execStatements(t, []string{ + "create table t1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + + execStatements(t, []string{ + "begin", + "insert into t1 values(1, 'aaa')", + "commit", + }) + expectDBClientQueries(t, []string{ + "begin", + "insert into t1 set id=1, val='aaa'", + "/update _vt.vreplication set pos=", + "commit", + }) + + vconn := &realDBClient{nolog: true} + if err := vconn.Connect(); err != nil { + t.Error(err) + } + defer vconn.Close() + + // Start a transaction and lock the row. + if _, err := vconn.ExecuteFetch("begin", 1); err != nil { + t.Error(err) + } + if _, err := vconn.ExecuteFetch("update t1 set val='bbb' where id=1", 1); err != nil { + t.Error(err) + } + + execStatements(t, []string{ + "begin", + "update t1 set val='ccc' where id=1", + "commit", + }) + // The innodb lock wait timeout is set to 1s. + expectDBClientQueries(t, []string{ + "begin", + "update t1 set val='ccc' where id=1", + "rollback", + }) + + // VReplication should not get stuck if you cancel now. + done := make(chan bool) + go func() { + cancel() + close(done) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("cancel is hung") + } +} + +func TestPlayerBatching(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + execStatements(t, []string{ + "create table t1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_EXEC, "") + defer cancel() + + execStatements(t, []string{ + "insert into t1 values(1, 'aaa')", + }) + expectDBClientQueries(t, []string{ + "begin", + "insert into t1 set id=1, val='aaa'", + "/update _vt.vreplication set pos=", + "commit", + }) + + vconn := &realDBClient{nolog: true} + if err := vconn.Connect(); err != nil { + t.Error(err) + } + defer vconn.Close() + + // Start a transaction and lock the row. + if _, err := vconn.ExecuteFetch("begin", 1); err != nil { + t.Error(err) + } + if _, err := vconn.ExecuteFetch("update t1 set val='bbb' where id=1", 1); err != nil { + t.Error(err) + } + + // create one transaction + execStatements(t, []string{ + "update t1 set val='ccc' where id=1", + }) + // Wait for the begin. The update will be blocked. + expectDBClientQueries(t, []string{ + "begin", + }) + + // Create two more transactions. They will go and wait in the relayLog. + execStatements(t, []string{ + "insert into t1 values(2, 'aaa')", + "insert into t1 values(3, 'aaa')", + "create table t2(id int, val varbinary(128), primary key(id))", + "drop table t2", + }) + + // Release the lock. + _, _ = vconn.ExecuteFetch("rollback", 1) + // First transaction will complete. The other two + // transactions must be batched into one. But the + // DDLs should be on their own. + expectDBClientQueries(t, []string{ + "update t1 set val='ccc' where id=1", + "/update _vt.vreplication set pos=", + "commit", + "begin", + "insert into t1 set id=2, val='aaa'", + "insert into t1 set id=3, val='aaa'", + "/update _vt.vreplication set pos=", + "commit", + "create table t2(id int, val varbinary(128), primary key(id))", + "/update _vt.vreplication set pos=", + "/", // drop table is rewritten by mysql. Don't check. + "/update _vt.vreplication set pos=", + }) +} + +func TestPlayerRelayLogMaxSize(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + for i := 0; i < 2; i++ { + // First iteration checks max size, second checks max items + func() { + switch i { + case 0: + savedSize := relayLogMaxSize + defer func() { relayLogMaxSize = savedSize }() + relayLogMaxSize = 10 + case 1: + savedLen := relayLogMaxItems + defer func() { relayLogMaxItems = savedLen }() + relayLogMaxItems = 2 + } + + execStatements(t, []string{ + "create table t1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + + execStatements(t, []string{ + "insert into t1 values(1, '123456')", + }) + expectDBClientQueries(t, []string{ + "begin", + "insert into t1 set id=1, val='123456'", + "/update _vt.vreplication set pos=", + "commit", + }) + + vconn := &realDBClient{nolog: true} + if err := vconn.Connect(); err != nil { + t.Error(err) + } + defer vconn.Close() + + // Start a transaction and lock the row. + if _, err := vconn.ExecuteFetch("begin", 1); err != nil { + t.Error(err) + } + if _, err := vconn.ExecuteFetch("update t1 set val='bbb' where id=1", 1); err != nil { + t.Error(err) + } + + // create one transaction + execStatements(t, []string{ + "update t1 set val='ccc' where id=1", + }) + // Wait for the begin. The update will be blocked. + expectDBClientQueries(t, []string{ + "begin", + }) + + // Create two more transactions. They will go and wait in the relayLog. + execStatements(t, []string{ + "insert into t1 values(2, '789012')", + "insert into t1 values(3, '345678')", + "insert into t1 values(4, '901234')", + }) + + // Release the lock. + _, _ = vconn.ExecuteFetch("rollback", 1) + // First transaction will complete. The other two + // transactions must be batched into one. The last transaction + // will wait to be sent to the relay until the player fetches + // them. + expectDBClientQueries(t, []string{ + "update t1 set val='ccc' where id=1", + "/update _vt.vreplication set pos=", + "commit", + "begin", + "insert into t1 set id=2, val='789012'", + "insert into t1 set id=3, val='345678'", + "/update _vt.vreplication set pos=", + "commit", + "begin", + "insert into t1 set id=4, val='901234'", + "/update _vt.vreplication set pos=", + "commit", + }) + }() + } +} + +func TestRestartOnVStreamEnd(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + savedDelay := *retryDelay + defer func() { *retryDelay = savedDelay }() + *retryDelay = 1 * time.Millisecond + + execStatements(t, []string{ + "create table t1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + + execStatements(t, []string{ + "insert into t1 values(1, 'aaa')", + }) + expectDBClientQueries(t, []string{ + "begin", + "insert into t1 set id=1, val='aaa'", + "/update _vt.vreplication set pos=", + "commit", + }) + + streamerEngine.Close() + expectDBClientQueries(t, []string{ + "/update.*'Error'.*vstream ended", + }) + if err := streamerEngine.Open(env.KeyspaceName, env.ShardName); err != nil { + t.Fatal(err) + } + + execStatements(t, []string{ + "insert into t1 values(2, 'aaa')", + }) + expectDBClientQueries(t, []string{ + "/update.*'Running'", + "begin", + "insert into t1 set id=2, val='aaa'", + "/update _vt.vreplication set pos=", + "commit", + }) +} + +func TestTimestamp(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + execStatements(t, []string{ + "create table t1(id int, ts timestamp, dt datetime)", + fmt.Sprintf("create table %s.t1(id int, ts timestamp, dt datetime)", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + + qr, err := env.Mysqld.FetchSuperQuery(context.Background(), "select now()") + if err != nil { + t.Fatal(err) + } + want := qr.Rows[0][0].ToString() + t.Logf("want: %s", want) + + execStatements(t, []string{ + fmt.Sprintf("insert into t1 values(1, '%s', '%s')", want, want), + }) + expectDBClientQueries(t, []string{ + "begin", + // The insert value for ts will be in UTC. + // We'll check the row instead. + "/insert into t1 set id=", + "/update _vt.vreplication set pos=", + "commit", + }) + + expectData(t, "t1", [][]string{{"1", want, want}}) +} + +func execStatements(t *testing.T, queries []string) { + t.Helper() + if err := env.Mysqld.ExecuteSuperQueryList(context.Background(), queries); err != nil { + t.Error(err) + } +} + +func startVReplication(t *testing.T, filter *binlogdatapb.Filter, onddl binlogdatapb.OnDDLAction, pos string) (cancelFunc func(), id int) { + t.Helper() + + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: onddl, + } + if pos == "" { + pos = masterPosition(t) + } + query := binlogplayer.CreateVReplication("test", bls, pos, 9223372036854775807, 9223372036854775807, 0) + qr, err := playerEngine.Exec(query) + if err != nil { + t.Fatal(err) + } + // Eat all the initialization queries + for q := range globalDBQueries { + if strings.HasPrefix(q, "update") { + break + } + } + return func() { + t.Helper() + query := fmt.Sprintf("delete from _vt.vreplication where id = %d", qr.InsertID) + if _, err := playerEngine.Exec(query); err != nil { + t.Fatal(err) + } + expectDBClientQueries(t, []string{ + "/delete", + }) + }, int(qr.InsertID) +} + +func masterPosition(t *testing.T) string { + t.Helper() + pos, err := env.Mysqld.MasterPosition() + if err != nil { + t.Fatal(err) + } + return mysql.EncodePosition(pos) +} diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index 51b7c0bf907..eb707aa1031 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -1274,7 +1274,7 @@ func (tsv *TabletServer) execDML(ctx context.Context, target *querypb.Target, qu } // VStream streams VReplication events. -func (tsv *TabletServer) VStream(ctx context.Context, target *querypb.Target, startPos mysql.Position, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { +func (tsv *TabletServer) VStream(ctx context.Context, target *querypb.Target, startPos string, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { // This code is partially duplicated from startRequest. This is because // is allowed even if the tablet is in non-serving state. err := func() error { diff --git a/go/vt/vttablet/tabletserver/vstreamer/engine.go b/go/vt/vttablet/tabletserver/vstreamer/engine.go index 61cb603976c..9e0e014bd95 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/engine.go +++ b/go/vt/vttablet/tabletserver/vstreamer/engine.go @@ -136,7 +136,7 @@ func (vse *Engine) vschema() *vindexes.KeyspaceSchema { } // Stream starts a new stream. -func (vse *Engine) Stream(ctx context.Context, startPos mysql.Position, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { +func (vse *Engine) Stream(ctx context.Context, startPos string, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { // Ensure kschema is initialized and the watcher is started. // Starting of the watcher has to be delayed till the first call to Stream // because this overhead should be incurred only if someone uses this feature. diff --git a/go/vt/vttablet/tabletserver/vstreamer/engine_test.go b/go/vt/vttablet/tabletserver/vstreamer/engine_test.go index 1e209663b77..c88a9aeef40 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/engine_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/engine_test.go @@ -50,7 +50,7 @@ func TestUpdateVSchema(t *testing.T) { t.Skip() } - defer setVSchema("{}") + defer env.SetVSchema("{}") // We have to start at least one stream to start the vschema watcher. ctx, cancel := context.WithCancel(context.Background()) @@ -67,7 +67,7 @@ func TestUpdateVSchema(t *testing.T) { startCount := expectUpdateCount(t, 1) - if err := setVSchema(shardedVSchema); err != nil { + if err := env.SetVSchema(shardedVSchema); err != nil { t.Fatal(err) } expectUpdateCount(t, startCount+1) diff --git a/go/vt/vttablet/tabletserver/vstreamer/main_test.go b/go/vt/vttablet/tabletserver/vstreamer/main_test.go index d308e431690..224091078f0 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/main_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/main_test.go @@ -20,45 +20,16 @@ import ( "flag" "fmt" "os" - "path" "testing" - "golang.org/x/net/context" - "vitess.io/vitess/go/json2" - "vitess.io/vitess/go/mysql" - "vitess.io/vitess/go/vt/dbconfigs" - "vitess.io/vitess/go/vt/logutil" - "vitess.io/vitess/go/vt/mysqlctl" - "vitess.io/vitess/go/vt/srvtopo" - "vitess.io/vitess/go/vt/topo" - "vitess.io/vitess/go/vt/topo/memorytopo" - "vitess.io/vitess/go/vt/topotools" - "vitess.io/vitess/go/vt/vttablet/tabletserver/connpool" - "vitess.io/vitess/go/vt/vttablet/tabletserver/schema" - "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" - "vitess.io/vitess/go/vt/vttest" - - topodatapb "vitess.io/vitess/go/vt/proto/topodata" - vschemapb "vitess.io/vitess/go/vt/proto/vschema" - vttestpb "vitess.io/vitess/go/vt/proto/vttest" + "vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/testenv" ) var ( - engine *Engine - mysqld *mysqlctl.Mysqld - connParams mysql.ConnParams - connAppDebugParams mysql.ConnParams - topoServ *topo.Server - keyspaceName = "vttest" - cells = []string{"cell1"} + engine *Engine + env *testenv.Env ) -type checker struct{} - -var _ = connpool.MySQLChecker(checker{}) - -func (checker) CheckMySQL() {} - func TestMain(m *testing.M) { flag.Parse() // Do not remove this comment, import into google3 depends on it @@ -67,90 +38,22 @@ func TestMain(m *testing.M) { } exitCode := func() int { - // Launch MySQL. - // We need a Keyspace in the topology, so the DbName is set. - // We need a Shard too, so the database 'vttest' is created. - cfg := vttest.Config{ - Topology: &vttestpb.VTTestTopology{ - Keyspaces: []*vttestpb.Keyspace{ - { - Name: keyspaceName, - Shards: []*vttestpb.Shard{ - { - Name: "0", - DbNameOverride: "vttest", - }, - }, - }, - }, - }, - ExtraMyCnf: []string{path.Join(os.Getenv("VTTOP"), "config/mycnf/rbr.cnf")}, - OnlyMySQL: true, - } - defer os.RemoveAll(cfg.SchemaDir) - cluster := vttest.LocalCluster{ - Config: cfg, - } - if err := cluster.Setup(); err != nil { - fmt.Fprintf(os.Stderr, "could not launch mysql: %v\n", err) - return 1 - } - defer cluster.TearDown() - - // initTopo initializes topoServ. - if err := initEngine(&cluster); err != nil { + var err error + env, err = testenv.Init() + if err != nil { fmt.Fprintf(os.Stderr, "%v", err) return 1 } + defer env.Close() + + // engine cannot be initialized in testenv because it introduces + // circular dependencies. + engine = NewEngine(env.SrvTopo, env.SchemaEngine) + engine.InitDBConfig(env.Dbcfgs) + engine.Open(env.KeyspaceName, env.Cells[0]) defer engine.Close() return m.Run() }() os.Exit(exitCode) } - -func initEngine(cluster *vttest.LocalCluster) error { - if err := initTopo(); err != nil { - return err - } - - se := schema.NewEngine(checker{}, tabletenv.DefaultQsConfig) - srvTopoServer := srvtopo.NewResilientServer(topoServ, "TestTopo") - engine = NewEngine(srvTopoServer, se) - - dbcfgs := dbconfigs.NewTestDBConfigs(cluster.MySQLConnParams(), cluster.MySQLAppDebugConnParams(), cluster.DbName()) - mysqld = mysqlctl.NewMysqld(dbcfgs) - se.InitDBConfig(dbcfgs) - engine.InitDBConfig(dbcfgs) - - engine.Open(keyspaceName, cells[0]) - return nil -} - -func initTopo() error { - ctx := context.Background() - - topoServ = memorytopo.NewServer(cells...) - if err := topoServ.CreateKeyspace(ctx, keyspaceName, &topodatapb.Keyspace{}); err != nil { - return err - } - // The first vschema should not be empty. Leads to Node not found error. - // TODO(sougou): need to fix the bug. - return setVSchema(`{"sharded": true}`) -} - -func setVSchema(vs string) error { - ctx := context.Background() - logger := logutil.NewConsoleLogger() - var kspb vschemapb.Keyspace - if err := json2.Unmarshal([]byte(vs), &kspb); err != nil { - return fmt.Errorf("Unmarshal failed: %v", err) - } - if err := topoServ.SaveVSchema(ctx, keyspaceName, &kspb); err != nil { - return fmt.Errorf("SaveVSchema failed: %v", err) - } - if err := topotools.RebuildVSchema(ctx, logger, topoServ, cells); err != nil { - return fmt.Errorf("RebuildVSchema failed: %v", err) - } - return nil -} diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index d3709ca7361..f9920e791fb 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -120,6 +120,61 @@ func (plan *Plan) filter(values []sqltypes.Value) (bool, []sqltypes.Value, error return true, result, nil } +func mustSendDDL(query mysql.Query, dbname string, filter *binlogdatapb.Filter) bool { + if query.Database != "" && query.Database != dbname { + return false + } + ast, err := sqlparser.Parse(query.SQL) + // If there was a parsing error, we send it through. Hopefully, + // recipient can handle it. + if err != nil { + return true + } + switch stmt := ast.(type) { + case *sqlparser.DBDDL: + return false + case *sqlparser.DDL: + if !stmt.Table.IsEmpty() { + return tableMatches(stmt.Table, dbname, filter) + } + for _, table := range stmt.FromTables { + if tableMatches(table, dbname, filter) { + return true + } + } + for _, table := range stmt.ToTables { + if tableMatches(table, dbname, filter) { + return true + } + } + return false + } + return true +} + +func tableMatches(table sqlparser.TableName, dbname string, filter *binlogdatapb.Filter) bool { + if !table.Qualifier.IsEmpty() && table.Qualifier.String() != dbname { + return false + } + for _, rule := range filter.Rules { + switch { + case strings.HasPrefix(rule.Match, "/"): + expr := strings.Trim(rule.Match, "/") + result, err := regexp.MatchString(expr, table.Name.String()) + if err != nil { + return true + } + if !result { + continue + } + return true + case table.Name.String() == rule.Match: + return true + } + } + return false +} + func buildPlan(ti *Table, kschema *vindexes.KeyspaceSchema, filter *binlogdatapb.Filter) (*Plan, error) { for _, rule := range filter.Rules { switch { @@ -293,18 +348,22 @@ func buildTablePlan(ti *Table, kschema *vindexes.KeyspaceSchema, query string) ( return plan, nil } -func analyzeExpr(ti *Table, expr sqlparser.SelectExpr) (cExpr ColExpr, err error) { - aexpr, ok := expr.(*sqlparser.AliasedExpr) +func analyzeExpr(ti *Table, selExpr sqlparser.SelectExpr) (cExpr ColExpr, err error) { + aliased, ok := selExpr.(*sqlparser.AliasedExpr) if !ok { - return ColExpr{}, fmt.Errorf("unexpected: %v", sqlparser.String(expr)) + return ColExpr{}, fmt.Errorf("unexpected: %v", sqlparser.String(selExpr)) + } + as := aliased.As + if as.IsEmpty() { + as = sqlparser.NewColIdent(sqlparser.String(aliased.Expr)) } - switch expr := aexpr.Expr.(type) { + switch expr := aliased.Expr.(type) { case *sqlparser.ColName: colnum, err := findColumn(ti, expr.Name) if err != nil { return ColExpr{}, err } - return ColExpr{ColNum: colnum, Alias: expr.Name, Type: ti.Columns[colnum].Type}, nil + return ColExpr{ColNum: colnum, Alias: as, Type: ti.Columns[colnum].Type}, nil case *sqlparser.FuncExpr: if expr.Distinct || len(expr.Exprs) != 1 { return ColExpr{}, fmt.Errorf("unsupported: %v", sqlparser.String(expr)) @@ -319,10 +378,6 @@ func analyzeExpr(ti *Table, expr sqlparser.SelectExpr) (cExpr ColExpr, err error if !ok { return ColExpr{}, fmt.Errorf("unsupported: %v", sqlparser.String(expr)) } - as := aexpr.As - if as.IsEmpty() { - as = sqlparser.NewColIdent(sqlparser.String(expr)) - } colnum, err := findColumn(ti, innerCol.Name) if err != nil { return ColExpr{}, err diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go index 1dda4116101..f9f357e24e2 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go @@ -60,13 +60,94 @@ func init() { if err := json2.Unmarshal([]byte(input), &kspb); err != nil { panic(fmt.Errorf("Unmarshal failed: %v", err)) } - kschema, err := vindexes.BuildKeyspaceSchema(&kspb, keyspaceName) + kschema, err := vindexes.BuildKeyspaceSchema(&kspb, "ks") if err != nil { panic(err) } testKSChema = kschema } +func TestMustSendDDL(t *testing.T) { + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/t1.*/", + }, { + Match: "t2", + }}, + } + testcases := []struct { + sql string + db string + output bool + }{{ + sql: "create database db", + output: false, + }, { + sql: "create table foo(id int)", + output: false, + }, { + sql: "create table db.foo(id int)", + output: false, + }, { + sql: "create table mydb.foo(id int)", + output: false, + }, { + sql: "create table t1a(id int)", + output: true, + }, { + sql: "create table db.t1a(id int)", + output: false, + }, { + sql: "create table mydb.t1a(id int)", + output: true, + }, { + sql: "rename table t1a to foo, foo to bar", + output: true, + }, { + sql: "rename table foo to t1a, foo to bar", + output: true, + }, { + sql: "rename table foo to bar, t1a to bar", + output: true, + }, { + sql: "rename table foo to bar, bar to foo", + output: false, + }, { + sql: "drop table t1a, foo", + output: true, + }, { + sql: "drop table foo, t1a", + output: true, + }, { + sql: "drop table foo, bar", + output: false, + }, { + sql: "bad query", + output: true, + }, { + sql: "select * from t", + output: true, + }, { + sql: "drop table t2", + output: true, + }, { + sql: "create table t1a(id int)", + db: "db", + output: false, + }, { + sql: "create table t1a(id int)", + db: "mydb", + output: true, + }} + for _, tcase := range testcases { + q := mysql.Query{SQL: tcase.sql, Database: tcase.db} + got := mustSendDDL(q, "mydb", filter) + if got != tcase.output { + t.Errorf("%v: %v, want %v", q, got, tcase.output) + } + } +} + func TestPlanbuilder(t *testing.T) { t1 := &Table{ TableMap: &mysql.TableMap{ diff --git a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go new file mode 100644 index 00000000000..557fa40a60a --- /dev/null +++ b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go @@ -0,0 +1,143 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package testenv supplies test functions for testing vstreamer. +package testenv + +import ( + "context" + "fmt" + "os" + "path" + + "vitess.io/vitess/go/json2" + "vitess.io/vitess/go/vt/dbconfigs" + "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/mysqlctl" + "vitess.io/vitess/go/vt/srvtopo" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/topo/memorytopo" + "vitess.io/vitess/go/vt/topotools" + "vitess.io/vitess/go/vt/vttablet/tabletserver/connpool" + "vitess.io/vitess/go/vt/vttablet/tabletserver/schema" + "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" + "vitess.io/vitess/go/vt/vttest" + + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + vschemapb "vitess.io/vitess/go/vt/proto/vschema" + vttestpb "vitess.io/vitess/go/vt/proto/vttest" +) + +// Env contains all the env vars for a test against a mysql instance. +type Env struct { + cluster *vttest.LocalCluster + + KeyspaceName string + ShardName string + Cells []string + + TopoServ *topo.Server + SrvTopo srvtopo.Server + Dbcfgs *dbconfigs.DBConfigs + Mysqld *mysqlctl.Mysqld + SchemaEngine *schema.Engine +} + +type checker struct{} + +var _ = connpool.MySQLChecker(checker{}) + +func (checker) CheckMySQL() {} + +// Init initializes an Env. +func Init() (*Env, error) { + te := &Env{ + KeyspaceName: "vttest", + ShardName: "0", + Cells: []string{"cell1"}, + } + + ctx := context.Background() + te.TopoServ = memorytopo.NewServer(te.Cells...) + if err := te.TopoServ.CreateKeyspace(ctx, te.KeyspaceName, &topodatapb.Keyspace{}); err != nil { + return nil, err + } + if err := te.TopoServ.CreateShard(ctx, te.KeyspaceName, te.ShardName); err != nil { + panic(err) + } + te.SrvTopo = srvtopo.NewResilientServer(te.TopoServ, "TestTopo") + + cfg := vttest.Config{ + Topology: &vttestpb.VTTestTopology{ + Keyspaces: []*vttestpb.Keyspace{ + { + Name: te.KeyspaceName, + Shards: []*vttestpb.Shard{ + { + Name: "0", + DbNameOverride: "vttest", + }, + }, + }, + }, + }, + ExtraMyCnf: []string{path.Join(os.Getenv("VTTOP"), "config/mycnf/rbr.cnf")}, + OnlyMySQL: true, + } + te.cluster = &vttest.LocalCluster{ + Config: cfg, + } + if err := te.cluster.Setup(); err != nil { + os.RemoveAll(te.cluster.Config.SchemaDir) + return nil, fmt.Errorf("could not launch mysql: %v", err) + } + + te.Dbcfgs = dbconfigs.NewTestDBConfigs(te.cluster.MySQLConnParams(), te.cluster.MySQLAppDebugConnParams(), te.cluster.DbName()) + te.Mysqld = mysqlctl.NewMysqld(te.Dbcfgs) + te.SchemaEngine = schema.NewEngine(checker{}, tabletenv.DefaultQsConfig) + te.SchemaEngine.InitDBConfig(te.Dbcfgs) + + // The first vschema should not be empty. Leads to Node not found error. + // TODO(sougou): need to fix the bug. + if err := te.SetVSchema(`{"sharded": true}`); err != nil { + te.Close() + return nil, err + } + + return te, nil +} + +// Close tears down TestEnv. +func (te *Env) Close() { + te.SchemaEngine.Close() + te.Mysqld.Close() + te.cluster.TearDown() + os.RemoveAll(te.cluster.Config.SchemaDir) +} + +// SetVSchema sets the vschema for the test keyspace. +func (te *Env) SetVSchema(vs string) error { + ctx := context.Background() + logger := logutil.NewConsoleLogger() + var kspb vschemapb.Keyspace + if err := json2.Unmarshal([]byte(vs), &kspb); err != nil { + return err + } + if err := te.TopoServ.SaveVSchema(ctx, te.KeyspaceName, &kspb); err != nil { + return err + } + return topotools.RebuildVSchema(ctx, logger, te.TopoServ, te.Cells) +} diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index e2b4f74c213..4bf378d9f81 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -20,6 +20,7 @@ import ( "context" "flag" "fmt" + "io" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" @@ -41,7 +42,7 @@ type vstreamer struct { cp *mysql.ConnParams se *schema.Engine - startPos mysql.Position + startPos string filter *binlogdatapb.Filter send func([]*binlogdatapb.VEvent) error @@ -55,7 +56,7 @@ type vstreamer struct { pos mysql.Position } -func newVStreamer(ctx context.Context, cp *mysql.ConnParams, se *schema.Engine, startPos mysql.Position, filter *binlogdatapb.Filter, kschema *vindexes.KeyspaceSchema, send func([]*binlogdatapb.VEvent) error) *vstreamer { +func newVStreamer(ctx context.Context, cp *mysql.ConnParams, se *schema.Engine, startPos string, filter *binlogdatapb.Filter, kschema *vindexes.KeyspaceSchema, send func([]*binlogdatapb.VEvent) error) *vstreamer { ctx, cancel := context.WithCancel(ctx) return &vstreamer{ ctx: ctx, @@ -88,7 +89,12 @@ func (vs *vstreamer) Cancel() { // Stream runs a single-threaded loop. func (vs *vstreamer) Stream() error { defer vs.cancel() - vs.pos = vs.startPos + + pos, err := mysql.DecodePosition(vs.startPos) + if err != nil { + return err + } + vs.pos = pos // Ensure se is Open. If vttablet came up in a non_serving role, // the schema engine may not have been initialized. @@ -126,8 +132,8 @@ func (vs *vstreamer) parseEvents(ctx context.Context, events <-chan mysql.Binlog case binlogdatapb.VEventType_GTID, binlogdatapb.VEventType_BEGIN, binlogdatapb.VEventType_FIELD: // We never have to send GTID, BEGIN or FIELD events on their own. bufferedEvents = append(bufferedEvents, vevent) - case binlogdatapb.VEventType_COMMIT, binlogdatapb.VEventType_ROLLBACK, binlogdatapb.VEventType_DDL: - // COMMIT, ROLLBACK and DDL are terminal. There may be no more events after + case binlogdatapb.VEventType_COMMIT, binlogdatapb.VEventType_DDL: + // COMMIT and DDL are terminal. There may be no more events after // these for a long time. So, we have to send whatever we have. bufferedEvents = append(bufferedEvents, vevent) vevents := bufferedEvents @@ -178,6 +184,9 @@ func (vs *vstreamer) parseEvents(ctx context.Context, events <-chan mysql.Binlog } for _, vevent := range vevents { if err := bufferAndTransmit(vevent); err != nil { + if err == io.EOF { + return nil + } return fmt.Errorf("error sending event: %v", err) } } @@ -228,7 +237,7 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, e } var vevents []*binlogdatapb.VEvent switch { - case ev.IsPseudo() || ev.IsGTID(): + case ev.IsGTID(): gtid, hasBegin, err := ev.GTID(vs.format) if err != nil { return nil, fmt.Errorf("can't get GTID from binlog event: %v, event data: %#v", err, ev) @@ -261,15 +270,21 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, e vevents = append(vevents, &binlogdatapb.VEvent{ Type: binlogdatapb.VEventType_COMMIT, }) - case sqlparser.StmtRollback: - vevents = append(vevents, &binlogdatapb.VEvent{ - Type: binlogdatapb.VEventType_ROLLBACK, - }) case sqlparser.StmtDDL: - vevents = append(vevents, &binlogdatapb.VEvent{ - Type: binlogdatapb.VEventType_DDL, - Ddl: q.SQL, - }) + if mustSendDDL(q, vs.cp.DbName, vs.filter) { + vevents = append(vevents, &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_DDL, + Ddl: q.SQL, + }) + } else { + vevents = append(vevents, + &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_BEGIN, + }, + &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_COMMIT, + }) + } // Proactively reload schema. // If the DDL adds a column, comparing with an older snapshot of the // schema will make us think that a column was dropped and error out. @@ -375,6 +390,9 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, e }) } } + for _, vevent := range vevents { + vevent.Timestamp = int64(ev.Timestamp()) + } return vevents, nil } diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go index 9a9ff697b07..517e19534ac 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go @@ -199,10 +199,10 @@ func TestREKeyrange(t *testing.T) { } ch := startStream(ctx, t, filter) - if err := setVSchema(shardedVSchema); err != nil { + if err := env.SetVSchema(shardedVSchema); err != nil { t.Fatal(err) } - defer setVSchema("{}") + defer env.SetVSchema("{}") // 1, 2, 3 and 5 are in shard -80. // 4 and 6 are in shard 80-. @@ -249,7 +249,7 @@ func TestREKeyrange(t *testing.T) { } } }` - if err := setVSchema(altVSchema); err != nil { + if err := env.SetVSchema(altVSchema); err != nil { t.Fatal(err) } @@ -374,10 +374,7 @@ func TestDDLAddColumn(t *testing.T) { }) // Record position before the next few statements. - pos, err := mysqld.MasterPosition() - if err != nil { - t.Fatal(err) - } + pos := masterPosition(t) execStatements(t, []string{ "begin", "insert into ddl_test1 values(1, 'aaa')", @@ -450,10 +447,7 @@ func TestDDLDropColumn(t *testing.T) { defer execStatement(t, "drop table ddl_test2") // Record position before the next few statements. - pos, err := mysqld.MasterPosition() - if err != nil { - t.Fatal(err) - } + pos := masterPosition(t) execStatements(t, []string{ "insert into ddl_test2 values(1, 'aaa', 'ccc')", // Adding columns is allowed. @@ -471,13 +465,40 @@ func TestDDLDropColumn(t *testing.T) { } }() defer close(ch) - err = vstream(ctx, t, pos, nil, ch) + err := vstream(ctx, t, pos, nil, ch) want := "cannot determine table columns" if err == nil || !strings.Contains(err.Error(), want) { t.Errorf("err: %v, must contain %s", err, want) } } +func TestUnsentDDL(t *testing.T) { + if testing.Short() { + t.Skip() + } + + execStatement(t, "create table unsent(id int, val varbinary(128), primary key(id))") + + testcases := []testcase{{ + input: []string{ + "drop table unsent", + }, + // An unsent DDL is sent as an empty transaction. + output: [][]string{{ + `gtid|begin`, + `gtid|begin`, + `commit`, + }}, + }} + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/none/", + }}, + } + runCases(t, filter, testcases) +} + func TestBuffering(t *testing.T) { if testing.Short() { t.Skip() @@ -675,8 +696,8 @@ func TestTypes(t *testing.T) { `fields: ` + `fields: ` + `fields: > `, - `type:ROW row_event: > > `, + `type:ROW row_event: > > `, `commit`, }}, }, { @@ -720,12 +741,10 @@ func TestTypes(t *testing.T) { } func TestJSON(t *testing.T) { - if testing.Short() { - t.Skip() - } + t.Skip("This test is disabled because every flavor of mysql has a different behavior.") // JSON is supported only after mysql57. - if err := mysqld.ExecuteSuperQuery(context.Background(), "create table vitess_json(id int default 1, val json, primary key(id))"); err != nil { + if err := env.Mysqld.ExecuteSuperQuery(context.Background(), "create table vitess_json(id int default 1, val json, primary key(id))"); err != nil { // If it's a syntax error, MySQL is an older version. Skip this test. if strings.Contains(err.Error(), "syntax") { return @@ -795,10 +814,7 @@ func TestMinimalMode(t *testing.T) { engine.se.Reload(context.Background()) // Record position before the next few statements. - pos, err := mysqld.MasterPosition() - if err != nil { - t.Fatal(err) - } + pos := masterPosition(t) execStatements(t, []string{ "set @@session.binlog_row_image='minimal'", "update t1 set val1='bbb' where id=1", @@ -815,7 +831,7 @@ func TestMinimalMode(t *testing.T) { } }() defer close(ch) - err = vstream(ctx, t, pos, nil, ch) + err := vstream(ctx, t, pos, nil, ch) want := "partial row image encountered" if err == nil || !strings.Contains(err.Error(), want) { t.Errorf("err: %v, must contain '%s'", err, want) @@ -837,10 +853,7 @@ func TestStatementMode(t *testing.T) { engine.se.Reload(context.Background()) // Record position before the next few statements. - pos, err := mysqld.MasterPosition() - if err != nil { - t.Fatal(err) - } + pos := masterPosition(t) execStatements(t, []string{ "set @@session.binlog_format='statement'", "update t1 set val1='bbb' where id=1", @@ -857,7 +870,7 @@ func TestStatementMode(t *testing.T) { } }() defer close(ch) - err = vstream(ctx, t, pos, nil, ch) + err := vstream(ctx, t, pos, nil, ch) want := "unexpected statement type" if err == nil || !strings.Contains(err.Error(), want) { t.Errorf("err: %v, must contain '%s'", err, want) @@ -920,6 +933,10 @@ func expectLog(ctx context.Context, t *testing.T, input interface{}, ch <-chan [ t.Fatalf("%v (%d): event: %v, want commit", input, i, evs[i]) } default: + if evs[i].Timestamp == 0 { + t.Fatalf("evs[%d].Timestamp: 0, want non-zero", i) + } + evs[i].Timestamp = 0 if got := fmt.Sprintf("%v", evs[i]); got != want { t.Fatalf("%v (%d): event:\n%q, want\n%q", input, i, got, want) } @@ -929,22 +946,19 @@ func expectLog(ctx context.Context, t *testing.T, input interface{}, ch <-chan [ } func startStream(ctx context.Context, t *testing.T, filter *binlogdatapb.Filter) <-chan []*binlogdatapb.VEvent { - pos, err := mysqld.MasterPosition() - if err != nil { - t.Fatal(err) - } + pos := masterPosition(t) ch := make(chan []*binlogdatapb.VEvent) go func() { defer close(ch) if err := vstream(ctx, t, pos, filter, ch); err != nil { - t.Fatal(err) + t.Error(err) } }() return ch } -func vstream(ctx context.Context, t *testing.T, pos mysql.Position, filter *binlogdatapb.Filter, ch chan []*binlogdatapb.VEvent) error { +func vstream(ctx context.Context, t *testing.T, pos string, filter *binlogdatapb.Filter, ch chan []*binlogdatapb.VEvent) error { if filter == nil { filter = &binlogdatapb.Filter{ Rules: []*binlogdatapb.Rule{{ @@ -965,14 +979,23 @@ func vstream(ctx context.Context, t *testing.T, pos mysql.Position, filter *binl func execStatement(t *testing.T, query string) { t.Helper() - if err := mysqld.ExecuteSuperQuery(context.Background(), query); err != nil { + if err := env.Mysqld.ExecuteSuperQuery(context.Background(), query); err != nil { t.Fatal(err) } } func execStatements(t *testing.T, queries []string) { t.Helper() - if err := mysqld.ExecuteSuperQueryList(context.Background(), queries); err != nil { + if err := env.Mysqld.ExecuteSuperQueryList(context.Background(), queries); err != nil { + t.Fatal(err) + } +} + +func masterPosition(t *testing.T) string { + t.Helper() + pos, err := env.Mysqld.MasterPosition() + if err != nil { t.Fatal(err) } + return mysql.EncodePosition(pos) } diff --git a/go/vt/wrangler/testlib/migrate_served_from_test.go b/go/vt/wrangler/testlib/migrate_served_from_test.go index 6194e2150ed..303e820e5b5 100644 --- a/go/vt/wrangler/testlib/migrate_served_from_test.go +++ b/go/vt/wrangler/testlib/migrate_served_from_test.go @@ -111,9 +111,11 @@ func TestMigrateServedFrom(t *testing.T) { if err := destMaster.Agent.VREngine.Open(context.Background()); err != nil { t.Fatal(err) } - // select pos from _vt.vreplication - dbClient.ExpectRequest("select pos from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ + // select pos, state, message from _vt.vreplication + dbClient.ExpectRequest("select pos, state, message from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ sqltypes.NewVarBinary("MariaDB/5-456-892"), + sqltypes.NewVarBinary("Running"), + sqltypes.NewVarBinary(""), }}}, nil) dbClient.ExpectRequest("use _vt", &sqltypes.Result{}, nil) dbClient.ExpectRequest("delete from _vt.vreplication where id = 1", &sqltypes.Result{RowsAffected: 1}, nil) diff --git a/go/vt/wrangler/testlib/migrate_served_types_test.go b/go/vt/wrangler/testlib/migrate_served_types_test.go index 28ae7789ffe..e0e8b849420 100644 --- a/go/vt/wrangler/testlib/migrate_served_types_test.go +++ b/go/vt/wrangler/testlib/migrate_served_types_test.go @@ -147,9 +147,11 @@ func TestMigrateServedTypes(t *testing.T) { if err := dest1Master.Agent.VREngine.Open(context.Background()); err != nil { t.Fatal(err) } - // select pos from _vt.vreplication - dbClient1.ExpectRequest("select pos from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ + // select pos, state, message from _vt.vreplication + dbClient1.ExpectRequest("select pos, state, message from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ sqltypes.NewVarBinary("MariaDB/5-456-892"), + sqltypes.NewVarBinary("Running"), + sqltypes.NewVarBinary(""), }}}, nil) dbClient1.ExpectRequest("use _vt", &sqltypes.Result{}, nil) dbClient1.ExpectRequest("delete from _vt.vreplication where id = 1", &sqltypes.Result{RowsAffected: 1}, nil) @@ -174,9 +176,11 @@ func TestMigrateServedTypes(t *testing.T) { if err := dest2Master.Agent.VREngine.Open(context.Background()); err != nil { t.Fatal(err) } - // select pos from _vt.vreplication - dbClient2.ExpectRequest("select pos from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ + // select pos, state, message from _vt.vreplication + dbClient2.ExpectRequest("select pos, state, message from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ sqltypes.NewVarBinary("MariaDB/5-456-892"), + sqltypes.NewVarBinary("Running"), + sqltypes.NewVarBinary(""), }}}, nil) dbClient2.ExpectRequest("use _vt", &sqltypes.Result{}, nil) dbClient2.ExpectRequest("delete from _vt.vreplication where id = 1", &sqltypes.Result{RowsAffected: 1}, nil) diff --git a/proto/binlogdata.proto b/proto/binlogdata.proto index 8c758d9895f..a1471e44f5c 100644 --- a/proto/binlogdata.proto +++ b/proto/binlogdata.proto @@ -23,6 +23,7 @@ option go_package = "vitess.io/vitess/go/vt/proto/binlogdata"; package binlogdata; +import "vtrpc.proto"; import "query.proto"; import "topodata.proto"; @@ -131,6 +132,14 @@ message Filter { repeated Rule rules = 1; } +// OnDDLAction lists the possible actions for DDLs. +enum OnDDLAction { + IGNORE = 0; + STOP = 1; + EXEC = 2; + EXEC_IGNORE = 3; +} + // BinlogSource specifies the source and filter parameters for // Filtered Replication. It currently supports a keyrange // or a list of tables. @@ -153,6 +162,9 @@ message BinlogSource { // filter is set if we're using the generalized representation // for the filter. Filter filter = 6; + + // on_ddl specifies the action to be taken when a DDL is encountered. + OnDDLAction on_ddl = 7; } // VEventType enumerates the event types. @@ -195,19 +207,24 @@ message FieldEvent { // VEvent represents a vstream event message VEvent { VEventType type = 1; - string gtid = 2; - string ddl = 3; - RowEvent row_event = 4; - FieldEvent field_event = 5; + int64 timestamp = 2; + string gtid = 3; + string ddl = 4; + RowEvent row_event = 5; + FieldEvent field_event = 6; } // VStreamRequest is the payload for VStream message VStreamRequest { - string position = 1; - Filter filter = 2; + vtrpc.CallerID effective_caller_id = 1; + query.VTGateCallerID immediate_caller_id = 2; + query.Target target = 3; + + string position = 4; + Filter filter = 5; } // VStreamResponse is the response from VStream message VStreamResponse { - repeated VEvent event = 1; + repeated VEvent events = 1; } diff --git a/proto/queryservice.proto b/proto/queryservice.proto index a6a7074c9dc..897cbf3f034 100644 --- a/proto/queryservice.proto +++ b/proto/queryservice.proto @@ -22,6 +22,7 @@ package queryservice; option go_package = "vitess.io/vitess/go/vt/proto/queryservice"; import "query.proto"; +import "binlogdata.proto"; // Query defines the tablet query service, implemented by vttablet. service Query { @@ -94,4 +95,7 @@ service Query { // UpdateStream asks the server to return a stream of the updates that have been applied to its database. rpc UpdateStream(query.UpdateStreamRequest) returns (stream query.UpdateStreamResponse) {}; + + // VStream streams vreplication events. + rpc VStream(binlogdata.VStreamRequest) returns (stream binlogdata.VStreamResponse) {}; } diff --git a/py/vtproto/binlogdata_pb2.py b/py/vtproto/binlogdata_pb2.py index d9d8e896d92..e586658b98d 100644 --- a/py/vtproto/binlogdata_pb2.py +++ b/py/vtproto/binlogdata_pb2.py @@ -13,6 +13,7 @@ _sym_db = _symbol_database.Default() +import vtrpc_pb2 as vtrpc__pb2 import query_pb2 as query__pb2 import topodata_pb2 as topodata__pb2 @@ -22,10 +23,41 @@ package='binlogdata', syntax='proto3', serialized_options=_b('Z\'vitess.io/vitess/go/vt/proto/binlogdata'), - serialized_pb=_b('\n\x10\x62inlogdata.proto\x12\nbinlogdata\x1a\x0bquery.proto\x1a\x0etopodata.proto\"7\n\x07\x43harset\x12\x0e\n\x06\x63lient\x18\x01 \x01(\x05\x12\x0c\n\x04\x63onn\x18\x02 \x01(\x05\x12\x0e\n\x06server\x18\x03 \x01(\x05\"\xb5\x03\n\x11\x42inlogTransaction\x12;\n\nstatements\x18\x01 \x03(\x0b\x32\'.binlogdata.BinlogTransaction.Statement\x12&\n\x0b\x65vent_token\x18\x04 \x01(\x0b\x32\x11.query.EventToken\x1a\xae\x02\n\tStatement\x12\x42\n\x08\x63\x61tegory\x18\x01 \x01(\x0e\x32\x30.binlogdata.BinlogTransaction.Statement.Category\x12$\n\x07\x63harset\x18\x02 \x01(\x0b\x32\x13.binlogdata.Charset\x12\x0b\n\x03sql\x18\x03 \x01(\x0c\"\xa9\x01\n\x08\x43\x61tegory\x12\x13\n\x0f\x42L_UNRECOGNIZED\x10\x00\x12\x0c\n\x08\x42L_BEGIN\x10\x01\x12\r\n\tBL_COMMIT\x10\x02\x12\x0f\n\x0b\x42L_ROLLBACK\x10\x03\x12\x15\n\x11\x42L_DML_DEPRECATED\x10\x04\x12\n\n\x06\x42L_DDL\x10\x05\x12\n\n\x06\x42L_SET\x10\x06\x12\r\n\tBL_INSERT\x10\x07\x12\r\n\tBL_UPDATE\x10\x08\x12\r\n\tBL_DELETE\x10\tJ\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04\"v\n\x15StreamKeyRangeRequest\x12\x10\n\x08position\x18\x01 \x01(\t\x12%\n\tkey_range\x18\x02 \x01(\x0b\x32\x12.topodata.KeyRange\x12$\n\x07\x63harset\x18\x03 \x01(\x0b\x32\x13.binlogdata.Charset\"S\n\x16StreamKeyRangeResponse\x12\x39\n\x12\x62inlog_transaction\x18\x01 \x01(\x0b\x32\x1d.binlogdata.BinlogTransaction\"]\n\x13StreamTablesRequest\x12\x10\n\x08position\x18\x01 \x01(\t\x12\x0e\n\x06tables\x18\x02 \x03(\t\x12$\n\x07\x63harset\x18\x03 \x01(\x0b\x32\x13.binlogdata.Charset\"Q\n\x14StreamTablesResponse\x12\x39\n\x12\x62inlog_transaction\x18\x01 \x01(\x0b\x32\x1d.binlogdata.BinlogTransaction\"%\n\x04Rule\x12\r\n\x05match\x18\x01 \x01(\t\x12\x0e\n\x06\x66ilter\x18\x02 \x01(\t\")\n\x06\x46ilter\x12\x1f\n\x05rules\x18\x01 \x03(\x0b\x32\x10.binlogdata.Rule\"\xb5\x01\n\x0c\x42inlogSource\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\x12)\n\x0btablet_type\x18\x03 \x01(\x0e\x32\x14.topodata.TabletType\x12%\n\tkey_range\x18\x04 \x01(\x0b\x32\x12.topodata.KeyRange\x12\x0e\n\x06tables\x18\x05 \x03(\t\x12\"\n\x06\x66ilter\x18\x06 \x01(\x0b\x32\x12.binlogdata.Filter\"B\n\tRowChange\x12\x1a\n\x06\x62\x65\x66ore\x18\x01 \x01(\x0b\x32\n.query.Row\x12\x19\n\x05\x61\x66ter\x18\x02 \x01(\x0b\x32\n.query.Row\"J\n\x08RowEvent\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12*\n\x0brow_changes\x18\x02 \x03(\x0b\x32\x15.binlogdata.RowChange\">\n\nFieldEvent\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x1c\n\x06\x66ields\x18\x02 \x03(\x0b\x32\x0c.query.Field\"\x9f\x01\n\x06VEvent\x12$\n\x04type\x18\x01 \x01(\x0e\x32\x16.binlogdata.VEventType\x12\x0c\n\x04gtid\x18\x02 \x01(\t\x12\x0b\n\x03\x64\x64l\x18\x03 \x01(\t\x12\'\n\trow_event\x18\x04 \x01(\x0b\x32\x14.binlogdata.RowEvent\x12+\n\x0b\x66ield_event\x18\x05 \x01(\x0b\x32\x16.binlogdata.FieldEvent\"F\n\x0eVStreamRequest\x12\x10\n\x08position\x18\x01 \x01(\t\x12\"\n\x06\x66ilter\x18\x02 \x01(\x0b\x32\x12.binlogdata.Filter\"4\n\x0fVStreamResponse\x12!\n\x05\x65vent\x18\x01 \x03(\x0b\x32\x12.binlogdata.VEvent*\xaa\x01\n\nVEventType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x08\n\x04GTID\x10\x01\x12\t\n\x05\x42\x45GIN\x10\x02\x12\n\n\x06\x43OMMIT\x10\x03\x12\x0c\n\x08ROLLBACK\x10\x04\x12\x07\n\x03\x44\x44L\x10\x05\x12\n\n\x06INSERT\x10\x06\x12\x0b\n\x07REPLACE\x10\x07\x12\n\n\x06UPDATE\x10\x08\x12\n\n\x06\x44\x45LETE\x10\t\x12\x07\n\x03SET\x10\n\x12\t\n\x05OTHER\x10\x0b\x12\x07\n\x03ROW\x10\x0c\x12\t\n\x05\x46IELD\x10\rB)Z\'vitess.io/vitess/go/vt/proto/binlogdatab\x06proto3') + serialized_pb=_b('\n\x10\x62inlogdata.proto\x12\nbinlogdata\x1a\x0bvtrpc.proto\x1a\x0bquery.proto\x1a\x0etopodata.proto\"7\n\x07\x43harset\x12\x0e\n\x06\x63lient\x18\x01 \x01(\x05\x12\x0c\n\x04\x63onn\x18\x02 \x01(\x05\x12\x0e\n\x06server\x18\x03 \x01(\x05\"\xb5\x03\n\x11\x42inlogTransaction\x12;\n\nstatements\x18\x01 \x03(\x0b\x32\'.binlogdata.BinlogTransaction.Statement\x12&\n\x0b\x65vent_token\x18\x04 \x01(\x0b\x32\x11.query.EventToken\x1a\xae\x02\n\tStatement\x12\x42\n\x08\x63\x61tegory\x18\x01 \x01(\x0e\x32\x30.binlogdata.BinlogTransaction.Statement.Category\x12$\n\x07\x63harset\x18\x02 \x01(\x0b\x32\x13.binlogdata.Charset\x12\x0b\n\x03sql\x18\x03 \x01(\x0c\"\xa9\x01\n\x08\x43\x61tegory\x12\x13\n\x0f\x42L_UNRECOGNIZED\x10\x00\x12\x0c\n\x08\x42L_BEGIN\x10\x01\x12\r\n\tBL_COMMIT\x10\x02\x12\x0f\n\x0b\x42L_ROLLBACK\x10\x03\x12\x15\n\x11\x42L_DML_DEPRECATED\x10\x04\x12\n\n\x06\x42L_DDL\x10\x05\x12\n\n\x06\x42L_SET\x10\x06\x12\r\n\tBL_INSERT\x10\x07\x12\r\n\tBL_UPDATE\x10\x08\x12\r\n\tBL_DELETE\x10\tJ\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04\"v\n\x15StreamKeyRangeRequest\x12\x10\n\x08position\x18\x01 \x01(\t\x12%\n\tkey_range\x18\x02 \x01(\x0b\x32\x12.topodata.KeyRange\x12$\n\x07\x63harset\x18\x03 \x01(\x0b\x32\x13.binlogdata.Charset\"S\n\x16StreamKeyRangeResponse\x12\x39\n\x12\x62inlog_transaction\x18\x01 \x01(\x0b\x32\x1d.binlogdata.BinlogTransaction\"]\n\x13StreamTablesRequest\x12\x10\n\x08position\x18\x01 \x01(\t\x12\x0e\n\x06tables\x18\x02 \x03(\t\x12$\n\x07\x63harset\x18\x03 \x01(\x0b\x32\x13.binlogdata.Charset\"Q\n\x14StreamTablesResponse\x12\x39\n\x12\x62inlog_transaction\x18\x01 \x01(\x0b\x32\x1d.binlogdata.BinlogTransaction\"%\n\x04Rule\x12\r\n\x05match\x18\x01 \x01(\t\x12\x0e\n\x06\x66ilter\x18\x02 \x01(\t\")\n\x06\x46ilter\x12\x1f\n\x05rules\x18\x01 \x03(\x0b\x32\x10.binlogdata.Rule\"\xde\x01\n\x0c\x42inlogSource\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\x12)\n\x0btablet_type\x18\x03 \x01(\x0e\x32\x14.topodata.TabletType\x12%\n\tkey_range\x18\x04 \x01(\x0b\x32\x12.topodata.KeyRange\x12\x0e\n\x06tables\x18\x05 \x03(\t\x12\"\n\x06\x66ilter\x18\x06 \x01(\x0b\x32\x12.binlogdata.Filter\x12\'\n\x06on_ddl\x18\x07 \x01(\x0e\x32\x17.binlogdata.OnDDLAction\"B\n\tRowChange\x12\x1a\n\x06\x62\x65\x66ore\x18\x01 \x01(\x0b\x32\n.query.Row\x12\x19\n\x05\x61\x66ter\x18\x02 \x01(\x0b\x32\n.query.Row\"J\n\x08RowEvent\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12*\n\x0brow_changes\x18\x02 \x03(\x0b\x32\x15.binlogdata.RowChange\">\n\nFieldEvent\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x1c\n\x06\x66ields\x18\x02 \x03(\x0b\x32\x0c.query.Field\"\xb2\x01\n\x06VEvent\x12$\n\x04type\x18\x01 \x01(\x0e\x32\x16.binlogdata.VEventType\x12\x11\n\ttimestamp\x18\x02 \x01(\x03\x12\x0c\n\x04gtid\x18\x03 \x01(\t\x12\x0b\n\x03\x64\x64l\x18\x04 \x01(\t\x12\'\n\trow_event\x18\x05 \x01(\x0b\x32\x14.binlogdata.RowEvent\x12+\n\x0b\x66ield_event\x18\x06 \x01(\x0b\x32\x16.binlogdata.FieldEvent\"\xc7\x01\n\x0eVStreamRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x10\n\x08position\x18\x04 \x01(\t\x12\"\n\x06\x66ilter\x18\x05 \x01(\x0b\x32\x12.binlogdata.Filter\"5\n\x0fVStreamResponse\x12\"\n\x06\x65vents\x18\x01 \x03(\x0b\x32\x12.binlogdata.VEvent*>\n\x0bOnDDLAction\x12\n\n\x06IGNORE\x10\x00\x12\x08\n\x04STOP\x10\x01\x12\x08\n\x04\x45XEC\x10\x02\x12\x0f\n\x0b\x45XEC_IGNORE\x10\x03*\xaa\x01\n\nVEventType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x08\n\x04GTID\x10\x01\x12\t\n\x05\x42\x45GIN\x10\x02\x12\n\n\x06\x43OMMIT\x10\x03\x12\x0c\n\x08ROLLBACK\x10\x04\x12\x07\n\x03\x44\x44L\x10\x05\x12\n\n\x06INSERT\x10\x06\x12\x0b\n\x07REPLACE\x10\x07\x12\n\n\x06UPDATE\x10\x08\x12\n\n\x06\x44\x45LETE\x10\t\x12\x07\n\x03SET\x10\n\x12\t\n\x05OTHER\x10\x0b\x12\x07\n\x03ROW\x10\x0c\x12\t\n\x05\x46IELD\x10\rB)Z\'vitess.io/vitess/go/vt/proto/binlogdatab\x06proto3') , - dependencies=[query__pb2.DESCRIPTOR,topodata__pb2.DESCRIPTOR,]) + dependencies=[vtrpc__pb2.DESCRIPTOR,query__pb2.DESCRIPTOR,topodata__pb2.DESCRIPTOR,]) +_ONDDLACTION = _descriptor.EnumDescriptor( + name='OnDDLAction', + full_name='binlogdata.OnDDLAction', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='IGNORE', index=0, number=0, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='STOP', index=1, number=1, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='EXEC', index=2, number=2, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='EXEC_IGNORE', index=3, number=3, + serialized_options=None, + type=None), + ], + containing_type=None, + serialized_options=None, + serialized_start=1907, + serialized_end=1969, +) +_sym_db.RegisterEnumDescriptor(_ONDDLACTION) + +OnDDLAction = enum_type_wrapper.EnumTypeWrapper(_ONDDLACTION) _VEVENTTYPE = _descriptor.EnumDescriptor( name='VEventType', full_name='binlogdata.VEventType', @@ -91,12 +123,16 @@ ], containing_type=None, serialized_options=None, - serialized_start=1704, - serialized_end=1874, + serialized_start=1972, + serialized_end=2142, ) _sym_db.RegisterEnumDescriptor(_VEVENTTYPE) VEventType = enum_type_wrapper.EnumTypeWrapper(_VEVENTTYPE) +IGNORE = 0 +STOP = 1 +EXEC = 2 +EXEC_IGNORE = 3 UNKNOWN = 0 GTID = 1 BEGIN = 2 @@ -162,8 +198,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=375, - serialized_end=544, + serialized_start=388, + serialized_end=557, ) _sym_db.RegisterEnumDescriptor(_BINLOGTRANSACTION_STATEMENT_CATEGORY) @@ -208,8 +244,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=61, - serialized_end=116, + serialized_start=74, + serialized_end=129, ) @@ -254,8 +290,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=242, - serialized_end=544, + serialized_start=255, + serialized_end=557, ) _BINLOGTRANSACTION = _descriptor.Descriptor( @@ -291,8 +327,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=119, - serialized_end=556, + serialized_start=132, + serialized_end=569, ) @@ -336,8 +372,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=558, - serialized_end=676, + serialized_start=571, + serialized_end=689, ) @@ -367,8 +403,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=678, - serialized_end=761, + serialized_start=691, + serialized_end=774, ) @@ -412,8 +448,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=763, - serialized_end=856, + serialized_start=776, + serialized_end=869, ) @@ -443,8 +479,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=858, - serialized_end=939, + serialized_start=871, + serialized_end=952, ) @@ -481,8 +517,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=941, - serialized_end=978, + serialized_start=954, + serialized_end=991, ) @@ -512,8 +548,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=980, - serialized_end=1021, + serialized_start=993, + serialized_end=1034, ) @@ -566,6 +602,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='on_ddl', full_name='binlogdata.BinlogSource.on_ddl', index=6, + number=7, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -578,8 +621,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1024, - serialized_end=1205, + serialized_start=1037, + serialized_end=1259, ) @@ -616,8 +659,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1207, - serialized_end=1273, + serialized_start=1261, + serialized_end=1327, ) @@ -654,8 +697,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1275, - serialized_end=1349, + serialized_start=1329, + serialized_end=1403, ) @@ -692,8 +735,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1351, - serialized_end=1413, + serialized_start=1405, + serialized_end=1467, ) @@ -712,33 +755,40 @@ is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='gtid', full_name='binlogdata.VEvent.gtid', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + name='timestamp', full_name='binlogdata.VEvent.timestamp', index=1, + number=2, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='ddl', full_name='binlogdata.VEvent.ddl', index=2, + name='gtid', full_name='binlogdata.VEvent.gtid', index=2, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='row_event', full_name='binlogdata.VEvent.row_event', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, + name='ddl', full_name='binlogdata.VEvent.ddl', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='field_event', full_name='binlogdata.VEvent.field_event', index=4, + name='row_event', full_name='binlogdata.VEvent.row_event', index=4, number=5, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='field_event', full_name='binlogdata.VEvent.field_event', index=5, + number=6, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -751,8 +801,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1416, - serialized_end=1575, + serialized_start=1470, + serialized_end=1648, ) @@ -764,19 +814,40 @@ containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='position', full_name='binlogdata.VStreamRequest.position', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + name='effective_caller_id', full_name='binlogdata.VStreamRequest.effective_caller_id', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='filter', full_name='binlogdata.VStreamRequest.filter', index=1, + name='immediate_caller_id', full_name='binlogdata.VStreamRequest.immediate_caller_id', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='target', full_name='binlogdata.VStreamRequest.target', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='position', full_name='binlogdata.VStreamRequest.position', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='filter', full_name='binlogdata.VStreamRequest.filter', index=4, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -789,8 +860,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1577, - serialized_end=1647, + serialized_start=1651, + serialized_end=1850, ) @@ -802,7 +873,7 @@ containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='event', full_name='binlogdata.VStreamResponse.event', index=0, + name='events', full_name='binlogdata.VStreamResponse.events', index=0, number=1, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, @@ -820,8 +891,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1649, - serialized_end=1701, + serialized_start=1852, + serialized_end=1905, ) _BINLOGTRANSACTION_STATEMENT.fields_by_name['category'].enum_type = _BINLOGTRANSACTION_STATEMENT_CATEGORY @@ -839,6 +910,7 @@ _BINLOGSOURCE.fields_by_name['tablet_type'].enum_type = topodata__pb2._TABLETTYPE _BINLOGSOURCE.fields_by_name['key_range'].message_type = topodata__pb2._KEYRANGE _BINLOGSOURCE.fields_by_name['filter'].message_type = _FILTER +_BINLOGSOURCE.fields_by_name['on_ddl'].enum_type = _ONDDLACTION _ROWCHANGE.fields_by_name['before'].message_type = query__pb2._ROW _ROWCHANGE.fields_by_name['after'].message_type = query__pb2._ROW _ROWEVENT.fields_by_name['row_changes'].message_type = _ROWCHANGE @@ -846,8 +918,11 @@ _VEVENT.fields_by_name['type'].enum_type = _VEVENTTYPE _VEVENT.fields_by_name['row_event'].message_type = _ROWEVENT _VEVENT.fields_by_name['field_event'].message_type = _FIELDEVENT +_VSTREAMREQUEST.fields_by_name['effective_caller_id'].message_type = vtrpc__pb2._CALLERID +_VSTREAMREQUEST.fields_by_name['immediate_caller_id'].message_type = query__pb2._VTGATECALLERID +_VSTREAMREQUEST.fields_by_name['target'].message_type = query__pb2._TARGET _VSTREAMREQUEST.fields_by_name['filter'].message_type = _FILTER -_VSTREAMRESPONSE.fields_by_name['event'].message_type = _VEVENT +_VSTREAMRESPONSE.fields_by_name['events'].message_type = _VEVENT DESCRIPTOR.message_types_by_name['Charset'] = _CHARSET DESCRIPTOR.message_types_by_name['BinlogTransaction'] = _BINLOGTRANSACTION DESCRIPTOR.message_types_by_name['StreamKeyRangeRequest'] = _STREAMKEYRANGEREQUEST @@ -863,6 +938,7 @@ DESCRIPTOR.message_types_by_name['VEvent'] = _VEVENT DESCRIPTOR.message_types_by_name['VStreamRequest'] = _VSTREAMREQUEST DESCRIPTOR.message_types_by_name['VStreamResponse'] = _VSTREAMRESPONSE +DESCRIPTOR.enum_types_by_name['OnDDLAction'] = _ONDDLACTION DESCRIPTOR.enum_types_by_name['VEventType'] = _VEVENTTYPE _sym_db.RegisterFileDescriptor(DESCRIPTOR) diff --git a/py/vtproto/queryservice_pb2.py b/py/vtproto/queryservice_pb2.py index 467e8b4dd44..3c443df129b 100644 --- a/py/vtproto/queryservice_pb2.py +++ b/py/vtproto/queryservice_pb2.py @@ -7,39 +7,39 @@ from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() import query_pb2 as query__pb2 +import binlogdata_pb2 as binlogdata__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='queryservice.proto', package='queryservice', syntax='proto3', - serialized_pb=_b('\n\x12queryservice.proto\x12\x0cqueryservice\x1a\x0bquery.proto2\xa7\x0c\n\x05Query\x12:\n\x07\x45xecute\x12\x15.query.ExecuteRequest\x1a\x16.query.ExecuteResponse\"\x00\x12I\n\x0c\x45xecuteBatch\x12\x1a.query.ExecuteBatchRequest\x1a\x1b.query.ExecuteBatchResponse\"\x00\x12N\n\rStreamExecute\x12\x1b.query.StreamExecuteRequest\x1a\x1c.query.StreamExecuteResponse\"\x00\x30\x01\x12\x34\n\x05\x42\x65gin\x12\x13.query.BeginRequest\x1a\x14.query.BeginResponse\"\x00\x12\x37\n\x06\x43ommit\x12\x14.query.CommitRequest\x1a\x15.query.CommitResponse\"\x00\x12=\n\x08Rollback\x12\x16.query.RollbackRequest\x1a\x17.query.RollbackResponse\"\x00\x12:\n\x07Prepare\x12\x15.query.PrepareRequest\x1a\x16.query.PrepareResponse\"\x00\x12O\n\x0e\x43ommitPrepared\x12\x1c.query.CommitPreparedRequest\x1a\x1d.query.CommitPreparedResponse\"\x00\x12U\n\x10RollbackPrepared\x12\x1e.query.RollbackPreparedRequest\x1a\x1f.query.RollbackPreparedResponse\"\x00\x12X\n\x11\x43reateTransaction\x12\x1f.query.CreateTransactionRequest\x1a .query.CreateTransactionResponse\"\x00\x12\x46\n\x0bStartCommit\x12\x19.query.StartCommitRequest\x1a\x1a.query.StartCommitResponse\"\x00\x12\x46\n\x0bSetRollback\x12\x19.query.SetRollbackRequest\x1a\x1a.query.SetRollbackResponse\"\x00\x12^\n\x13\x43oncludeTransaction\x12!.query.ConcludeTransactionRequest\x1a\".query.ConcludeTransactionResponse\"\x00\x12R\n\x0fReadTransaction\x12\x1d.query.ReadTransactionRequest\x1a\x1e.query.ReadTransactionResponse\"\x00\x12I\n\x0c\x42\x65ginExecute\x12\x1a.query.BeginExecuteRequest\x1a\x1b.query.BeginExecuteResponse\"\x00\x12X\n\x11\x42\x65ginExecuteBatch\x12\x1f.query.BeginExecuteBatchRequest\x1a .query.BeginExecuteBatchResponse\"\x00\x12N\n\rMessageStream\x12\x1b.query.MessageStreamRequest\x1a\x1c.query.MessageStreamResponse\"\x00\x30\x01\x12\x43\n\nMessageAck\x12\x18.query.MessageAckRequest\x1a\x19.query.MessageAckResponse\"\x00\x12\x43\n\nSplitQuery\x12\x18.query.SplitQueryRequest\x1a\x19.query.SplitQueryResponse\"\x00\x12K\n\x0cStreamHealth\x12\x1a.query.StreamHealthRequest\x1a\x1b.query.StreamHealthResponse\"\x00\x30\x01\x12K\n\x0cUpdateStream\x12\x1a.query.UpdateStreamRequest\x1a\x1b.query.UpdateStreamResponse\"\x00\x30\x01\x42+Z)vitess.io/vitess/go/vt/proto/queryserviceb\x06proto3') + serialized_options=_b('Z)vitess.io/vitess/go/vt/proto/queryservice'), + serialized_pb=_b('\n\x12queryservice.proto\x12\x0cqueryservice\x1a\x0bquery.proto\x1a\x10\x62inlogdata.proto2\xef\x0c\n\x05Query\x12:\n\x07\x45xecute\x12\x15.query.ExecuteRequest\x1a\x16.query.ExecuteResponse\"\x00\x12I\n\x0c\x45xecuteBatch\x12\x1a.query.ExecuteBatchRequest\x1a\x1b.query.ExecuteBatchResponse\"\x00\x12N\n\rStreamExecute\x12\x1b.query.StreamExecuteRequest\x1a\x1c.query.StreamExecuteResponse\"\x00\x30\x01\x12\x34\n\x05\x42\x65gin\x12\x13.query.BeginRequest\x1a\x14.query.BeginResponse\"\x00\x12\x37\n\x06\x43ommit\x12\x14.query.CommitRequest\x1a\x15.query.CommitResponse\"\x00\x12=\n\x08Rollback\x12\x16.query.RollbackRequest\x1a\x17.query.RollbackResponse\"\x00\x12:\n\x07Prepare\x12\x15.query.PrepareRequest\x1a\x16.query.PrepareResponse\"\x00\x12O\n\x0e\x43ommitPrepared\x12\x1c.query.CommitPreparedRequest\x1a\x1d.query.CommitPreparedResponse\"\x00\x12U\n\x10RollbackPrepared\x12\x1e.query.RollbackPreparedRequest\x1a\x1f.query.RollbackPreparedResponse\"\x00\x12X\n\x11\x43reateTransaction\x12\x1f.query.CreateTransactionRequest\x1a .query.CreateTransactionResponse\"\x00\x12\x46\n\x0bStartCommit\x12\x19.query.StartCommitRequest\x1a\x1a.query.StartCommitResponse\"\x00\x12\x46\n\x0bSetRollback\x12\x19.query.SetRollbackRequest\x1a\x1a.query.SetRollbackResponse\"\x00\x12^\n\x13\x43oncludeTransaction\x12!.query.ConcludeTransactionRequest\x1a\".query.ConcludeTransactionResponse\"\x00\x12R\n\x0fReadTransaction\x12\x1d.query.ReadTransactionRequest\x1a\x1e.query.ReadTransactionResponse\"\x00\x12I\n\x0c\x42\x65ginExecute\x12\x1a.query.BeginExecuteRequest\x1a\x1b.query.BeginExecuteResponse\"\x00\x12X\n\x11\x42\x65ginExecuteBatch\x12\x1f.query.BeginExecuteBatchRequest\x1a .query.BeginExecuteBatchResponse\"\x00\x12N\n\rMessageStream\x12\x1b.query.MessageStreamRequest\x1a\x1c.query.MessageStreamResponse\"\x00\x30\x01\x12\x43\n\nMessageAck\x12\x18.query.MessageAckRequest\x1a\x19.query.MessageAckResponse\"\x00\x12\x43\n\nSplitQuery\x12\x18.query.SplitQueryRequest\x1a\x19.query.SplitQueryResponse\"\x00\x12K\n\x0cStreamHealth\x12\x1a.query.StreamHealthRequest\x1a\x1b.query.StreamHealthResponse\"\x00\x30\x01\x12K\n\x0cUpdateStream\x12\x1a.query.UpdateStreamRequest\x1a\x1b.query.UpdateStreamResponse\"\x00\x30\x01\x12\x46\n\x07VStream\x12\x1a.binlogdata.VStreamRequest\x1a\x1b.binlogdata.VStreamResponse\"\x00\x30\x01\x42+Z)vitess.io/vitess/go/vt/proto/queryserviceb\x06proto3') , - dependencies=[query__pb2.DESCRIPTOR,]) + dependencies=[query__pb2.DESCRIPTOR,binlogdata__pb2.DESCRIPTOR,]) _sym_db.RegisterFileDescriptor(DESCRIPTOR) -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('Z)vitess.io/vitess/go/vt/proto/queryservice')) +DESCRIPTOR._options = None _QUERY = _descriptor.ServiceDescriptor( name='Query', full_name='queryservice.Query', file=DESCRIPTOR, index=0, - options=None, - serialized_start=50, - serialized_end=1625, + serialized_options=None, + serialized_start=68, + serialized_end=1715, methods=[ _descriptor.MethodDescriptor( name='Execute', @@ -48,7 +48,7 @@ containing_service=None, input_type=query__pb2._EXECUTEREQUEST, output_type=query__pb2._EXECUTERESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='ExecuteBatch', @@ -57,7 +57,7 @@ containing_service=None, input_type=query__pb2._EXECUTEBATCHREQUEST, output_type=query__pb2._EXECUTEBATCHRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='StreamExecute', @@ -66,7 +66,7 @@ containing_service=None, input_type=query__pb2._STREAMEXECUTEREQUEST, output_type=query__pb2._STREAMEXECUTERESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='Begin', @@ -75,7 +75,7 @@ containing_service=None, input_type=query__pb2._BEGINREQUEST, output_type=query__pb2._BEGINRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='Commit', @@ -84,7 +84,7 @@ containing_service=None, input_type=query__pb2._COMMITREQUEST, output_type=query__pb2._COMMITRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='Rollback', @@ -93,7 +93,7 @@ containing_service=None, input_type=query__pb2._ROLLBACKREQUEST, output_type=query__pb2._ROLLBACKRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='Prepare', @@ -102,7 +102,7 @@ containing_service=None, input_type=query__pb2._PREPAREREQUEST, output_type=query__pb2._PREPARERESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='CommitPrepared', @@ -111,7 +111,7 @@ containing_service=None, input_type=query__pb2._COMMITPREPAREDREQUEST, output_type=query__pb2._COMMITPREPAREDRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='RollbackPrepared', @@ -120,7 +120,7 @@ containing_service=None, input_type=query__pb2._ROLLBACKPREPAREDREQUEST, output_type=query__pb2._ROLLBACKPREPAREDRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='CreateTransaction', @@ -129,7 +129,7 @@ containing_service=None, input_type=query__pb2._CREATETRANSACTIONREQUEST, output_type=query__pb2._CREATETRANSACTIONRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='StartCommit', @@ -138,7 +138,7 @@ containing_service=None, input_type=query__pb2._STARTCOMMITREQUEST, output_type=query__pb2._STARTCOMMITRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='SetRollback', @@ -147,7 +147,7 @@ containing_service=None, input_type=query__pb2._SETROLLBACKREQUEST, output_type=query__pb2._SETROLLBACKRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='ConcludeTransaction', @@ -156,7 +156,7 @@ containing_service=None, input_type=query__pb2._CONCLUDETRANSACTIONREQUEST, output_type=query__pb2._CONCLUDETRANSACTIONRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='ReadTransaction', @@ -165,7 +165,7 @@ containing_service=None, input_type=query__pb2._READTRANSACTIONREQUEST, output_type=query__pb2._READTRANSACTIONRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='BeginExecute', @@ -174,7 +174,7 @@ containing_service=None, input_type=query__pb2._BEGINEXECUTEREQUEST, output_type=query__pb2._BEGINEXECUTERESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='BeginExecuteBatch', @@ -183,7 +183,7 @@ containing_service=None, input_type=query__pb2._BEGINEXECUTEBATCHREQUEST, output_type=query__pb2._BEGINEXECUTEBATCHRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='MessageStream', @@ -192,7 +192,7 @@ containing_service=None, input_type=query__pb2._MESSAGESTREAMREQUEST, output_type=query__pb2._MESSAGESTREAMRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='MessageAck', @@ -201,7 +201,7 @@ containing_service=None, input_type=query__pb2._MESSAGEACKREQUEST, output_type=query__pb2._MESSAGEACKRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='SplitQuery', @@ -210,7 +210,7 @@ containing_service=None, input_type=query__pb2._SPLITQUERYREQUEST, output_type=query__pb2._SPLITQUERYRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='StreamHealth', @@ -219,7 +219,7 @@ containing_service=None, input_type=query__pb2._STREAMHEALTHREQUEST, output_type=query__pb2._STREAMHEALTHRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='UpdateStream', @@ -228,7 +228,16 @@ containing_service=None, input_type=query__pb2._UPDATESTREAMREQUEST, output_type=query__pb2._UPDATESTREAMRESPONSE, - options=None, + serialized_options=None, + ), + _descriptor.MethodDescriptor( + name='VStream', + full_name='queryservice.Query.VStream', + index=21, + containing_service=None, + input_type=binlogdata__pb2._VSTREAMREQUEST, + output_type=binlogdata__pb2._VSTREAMRESPONSE, + serialized_options=None, ), ]) _sym_db.RegisterServiceDescriptor(_QUERY) diff --git a/py/vtproto/queryservice_pb2_grpc.py b/py/vtproto/queryservice_pb2_grpc.py index 3fc203d463e..61adf9b42bd 100644 --- a/py/vtproto/queryservice_pb2_grpc.py +++ b/py/vtproto/queryservice_pb2_grpc.py @@ -1,6 +1,7 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! import grpc +import binlogdata_pb2 as binlogdata__pb2 import query_pb2 as query__pb2 @@ -119,6 +120,11 @@ def __init__(self, channel): request_serializer=query__pb2.UpdateStreamRequest.SerializeToString, response_deserializer=query__pb2.UpdateStreamResponse.FromString, ) + self.VStream = channel.unary_stream( + '/queryservice.Query/VStream', + request_serializer=binlogdata__pb2.VStreamRequest.SerializeToString, + response_deserializer=binlogdata__pb2.VStreamResponse.FromString, + ) class QueryServicer(object): @@ -279,6 +285,13 @@ def UpdateStream(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def VStream(self, request, context): + """VStream streams vreplication events. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def add_QueryServicer_to_server(servicer, server): rpc_method_handlers = { @@ -387,6 +400,11 @@ def add_QueryServicer_to_server(servicer, server): request_deserializer=query__pb2.UpdateStreamRequest.FromString, response_serializer=query__pb2.UpdateStreamResponse.SerializeToString, ), + 'VStream': grpc.unary_stream_rpc_method_handler( + servicer.VStream, + request_deserializer=binlogdata__pb2.VStreamRequest.FromString, + response_serializer=binlogdata__pb2.VStreamResponse.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'queryservice.Query', rpc_method_handlers)