Skip to content

Pass "Flags" from MySQL field from the result set e.g. to preserve if integers are unsigned. #618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
michael-berlin merged 1 commit into from
Apr 28, 2015
Merged

Pass "Flags" from MySQL field from the result set e.g. to preserve if integers are unsigned. #618

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions go/mysql/mysql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,7 @@ func (conn *Connection) Fields() (fields []proto.Field) {
fname := (*[maxSize]byte)(unsafe.Pointer(cfields[i].name))[:length]
fields[i].Name = string(fname)
fields[i].Type = int64(cfields[i]._type)
fields[i].Flags = int64(cfields[i].flags)
}
return fields
}
Expand Down
4 changes: 2 additions & 2 deletions go/mysql/proto/bson_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ type extraQueryResult struct {
}

func TestQueryResult(t *testing.T) {
want := "\x85\x00\x00\x00\x04Fields\x00*\x00\x00\x00\x030\x00\"\x00\x00\x00\x05Name\x00\x04\x00\x00\x00\x00name\x12Type\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00?RowsAffected\x00\x02\x00\x00\x00\x00\x00\x00\x00?InsertId\x00\x03\x00\x00\x00\x00\x00\x00\x00\x04Rows\x00 \x00\x00\x00\x040\x00\x18\x00\x00\x00\x050\x00\x01\x00\x00\x00\x001\x051\x00\x02\x00\x00\x00\x00aa\x00\x00\x00"
want := "\x94\x00\x00\x00\x04Fields\x009\x00\x00\x00\x030\x001\x00\x00\x00\x05Name\x00\x04\x00\x00\x00\x00name\x12Type\x00\x01\x00\x00\x00\x00\x00\x00\x00\x12Flags\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00?RowsAffected\x00\x02\x00\x00\x00\x00\x00\x00\x00?InsertId\x00\x03\x00\x00\x00\x00\x00\x00\x00\x04Rows\x00 \x00\x00\x00\x040\x00\x18\x00\x00\x00\x050\x00\x01\x00\x00\x00\x001\x051\x00\x02\x00\x00\x00\x00aa\x00\x00\x00"
custom := QueryResult{
Fields: []Field{{"name", 1}},
Fields: []Field{{"name", 1, VT_ZEROVALUE_FLAG}},
RowsAffected: 2,
InsertId: 3,
Rows: [][]sqltypes.Value{
Expand Down
3 changes: 3 additions & 0 deletions go/mysql/proto/field_bson.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ func (field *Field) MarshalBson(buf *bytes2.ChunkedWriter, key string) {

bson.EncodeString(buf, "Name", field.Name)
bson.EncodeInt64(buf, "Type", field.Type)
bson.EncodeInt64(buf, "Flags", field.Flags)

lenWriter.Close()
}
Expand All @@ -43,6 +44,8 @@ func (field *Field) UnmarshalBson(buf *bytes.Buffer, kind byte) {
field.Name = bson.DecodeString(buf, kind)
case "Type":
field.Type = bson.DecodeInt64(buf, kind)
case "Flags":
field.Flags = bson.DecodeInt64(buf, kind)
default:
bson.Skip(buf, kind)
}
Expand Down
2 changes: 1 addition & 1 deletion go/mysql/proto/qr_bson_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ var testcases = []TestCase{
{Name: "foo", Type: 1},
},
},
encoded: "i\x00\x00\x00\x04Fields\x00)\x00\x00\x00\x030\x00!\x00\x00\x00\x05Name\x00\x03\x00\x00\x00\x00foo\x12Type\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00?RowsAffected\x00\x00\x00\x00\x00\x00\x00\x00\x00?InsertId\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04Rows\x00\x05\x00\x00\x00\x00\x00",
encoded: "x\x00\x00\x00\x04Fields\x008\x00\x00\x00\x030\x000\x00\x00\x00\x05Name\x00\x03\x00\x00\x00\x00foo\x12Type\x00\x01\x00\x00\x00\x00\x00\x00\x00\x12Flags\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00?RowsAffected\x00\x00\x00\x00\x00\x00\x00\x00\x00?InsertId\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04Rows\x00\x05\x00\x00\x00\x00\x00",
},
// Only rows, no fields
{
Expand Down
30 changes: 28 additions & 2 deletions go/mysql/proto/structs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,35 @@ const (
VT_GEOMETRY = 255
)

// MySQL field flags bitset values e.g. to distinguish between signed and unsigned integer.
// Comments are taken from the original source code.
// These numbers should exactly match values defined in dist/mysql-5.1.52/include/mysql_com.h
const (
// VT_ZEROVALUE_FLAG is not part of the MySQL specification and only used in unit tests.
VT_ZEROVALUE_FLAG = 0
VT_NOT_NULL_FLAG = 1 /* Field can't be NULL */
Copy link
Copy Markdown
Contributor

@sougou sougou Apr 24, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You could consider using iota like here: http://play.golang.org/p/FG23LdS_xK

Copy link
Copy Markdown
Contributor

@sougou sougou Apr 24, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

On second thoughts. It's better if these are explicit values if you copied them from the .h file.
Ideally, you should do what I did here: https://github.com/youtube/vitess/blob/master/go/mysql/mysql.go#L41.
But that will introduce a C dependency in this package. So, I think things are fine as is for now.

Copy link
Copy Markdown
Contributor Author

@michael-berlin michael-berlin Apr 24, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, let's just copy&paste the flags.

VT_PRI_KEY_FLAG = 2 /* Field is part of a primary key */
VT_UNIQUE_KEY_FLAG = 4 /* Field is part of a unique key */
VT_MULTIPLE_KEY_FLAG = 8 /* Field is part of a key */
VT_BLOB_FLAG = 16 /* Field is a blob */
VT_UNSIGNED_FLAG = 32 /* Field is unsigned */
VT_ZEROFILL_FLAG = 64 /* Field is zerofill */
VT_BINARY_FLAG = 128 /* Field is binary */
/* The following are only sent to new clients */
Copy link
Copy Markdown
Contributor

@henryanand henryanand Apr 24, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you clarify what 'new' here means? With time, 'new' becomes 'old'.
Edit: Looks like these are just copied as-is from mysql code?

Copy link
Copy Markdown
Contributor Author

@michael-berlin michael-berlin Apr 24, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, copy&paste from the header file.

VT_ENUM_FLAG = 256 /* field is an enum */
VT_AUTO_INCREMENT_FLAG = 512 /* field is a autoincrement field */
VT_TIMESTAMP_FLAG = 1024 /* Field is a timestamp */
VT_SET_FLAG = 2048 /* field is a set */
VT_NO_DEFAULT_VALUE_FLAG = 4096 /* Field doesn't have default value */
VT_ON_UPDATE_NOW_FLAG = 8192 /* Field is set to NOW on UPDATE */
VT_NUM_FLAG = 32768 /* Field is num (for clients) */
)

// Field describes a column returned by MySQL.
type Field struct {
Name string
Type int64
Name string
Type int64
Flags int64
}

//go:generate bsongen -file $GOFILE -type Field -o field_bson.go
Expand Down Expand Up @@ -77,6 +102,7 @@ type Charset struct {
// - int64 if possible, otherwise, uint64
// - float64 for floating point values that fit in a float
// - []byte for everything else
// TODO(mberlin): Make this a method of "Field" and consider VT_UNSIGNED_FLAG in "Flags" as well.
func Convert(mysqlType int64, val sqltypes.Value) (interface{}, error) {
if val.IsNull() {
return nil, nil
Expand Down
1 change: 1 addition & 0 deletions go/vt/client2/tablet/tclient.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,7 @@ type Result struct {
err error
}

// TODO(mberlin): Populate flags here as well (e.g. to correctly identify unsigned integer type)?
func NewResult(rowCount, rowsAffected, insertId int64, fields []mproto.Field) *Result {
return &Result{
qr: &mproto.QueryResult{
Expand Down
39 changes: 5 additions & 34 deletions go/vt/vtgate/proto/vtgate_proto_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,40 +165,11 @@ func TestQueryShard(t *testing.T) {
func TestQueryResult(t *testing.T) {
// We can't do the reflection test because bson
// doesn't do it correctly for embedded fields.
want := "|\x01\x00\x00" +
"\x03Result\x00\x85\x00\x00\x00" +
"\x04Fields\x00*\x00\x00\x00" +
"\x030\x00\"\x00\x00\x00" +
"\x05Name\x00\x04\x00\x00\x00\x00name" +
"\x12Type\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00" +
"?RowsAffected\x00\x02\x00\x00\x00\x00\x00\x00\x00" +
"?InsertId\x00\x03\x00\x00\x00\x00\x00\x00\x00" +
"\x04Rows\x00 \x00\x00\x00" +
"\x040\x00\x18\x00\x00\x00" +
"\x050\x00\x01\x00\x00\x00" +
"\x001\x051\x00\x02\x00\x00\x00\x00aa" +
"\x00\x00\x00" +
"\x03Session\x00\xd0\x00\x00\x00" +
"\bInTransaction\x00\x01" +
"\x04ShardSessions\x00\xac\x00\x00\x00" +
"\x030\x00Q\x00\x00\x00" +
"\x05Keyspace\x00\x01\x00\x00\x00\x00a" +
"\x05Shard\x00\x01\x00\x00\x00\x000" +
"\x05TabletType\x00\a\x00\x00\x00\x00replica" +
"\x12TransactionId\x00\x01\x00\x00\x00\x00\x00\x00\x00" +
"\x00" +
"\x031\x00P\x00\x00\x00" +
"\x05Keyspace\x00\x01\x00\x00\x00\x00b" +
"\x05Shard\x00\x01\x00\x00\x00\x001" +
"\x05TabletType\x00\x06\x00\x00\x00\x00master" +
"\x12TransactionId\x00\x02\x00\x00\x00\x00\x00\x00\x00" +
"\x00\x00\x00" +
"\x05Error\x00\x05\x00\x00\x00\x00error" +
"\x00"
want := "\x8b\x01\x00\x00\x03Result\x00\x94\x00\x00\x00\x04Fields\x009\x00\x00\x00\x030\x001\x00\x00\x00\x05Name\x00\x04\x00\x00\x00\x00name\x12Type\x00\x01\x00\x00\x00\x00\x00\x00\x00\x12Flags\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00?RowsAffected\x00\x02\x00\x00\x00\x00\x00\x00\x00?InsertId\x00\x03\x00\x00\x00\x00\x00\x00\x00\x04Rows\x00 \x00\x00\x00\x040\x00\x18\x00\x00\x00\x050\x00\x01\x00\x00\x00\x001\x051\x00\x02\x00\x00\x00\x00aa\x00\x00\x00\x03Session\x00\xd0\x00\x00\x00\bInTransaction\x00\x01\x04ShardSessions\x00\xac\x00\x00\x00\x030\x00Q\x00\x00\x00\x05Keyspace\x00\x01\x00\x00\x00\x00a\x05Shard\x00\x01\x00\x00\x00\x000\x05TabletType\x00\a\x00\x00\x00\x00replica\x12TransactionId\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x031\x00P\x00\x00\x00\x05Keyspace\x00\x01\x00\x00\x00\x00b\x05Shard\x00\x01\x00\x00\x00\x001\x05TabletType\x00\x06\x00\x00\x00\x00master\x12TransactionId\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05Error\x00\x05\x00\x00\x00\x00error\x00"

custom := QueryResult{
Result: &mproto.QueryResult{
Fields: []mproto.Field{{"name", 1}},
Fields: []mproto.Field{{"name", 1, mproto.VT_ZEROVALUE_FLAG}},
RowsAffected: 2,
InsertId: 3,
Rows: [][]sqltypes.Value{
Expand All @@ -214,7 +185,7 @@ func TestQueryResult(t *testing.T) {
}
got := string(encoded)
if want != got {
t.Errorf("want\n%+v, got\n%+v", want, got)
t.Errorf("want\n%#v, got\n%#v", want, got)
}

var unmarshalled QueryResult
Expand Down Expand Up @@ -353,7 +324,7 @@ type extraQueryResultList struct {
func TestQueryResultList(t *testing.T) {
reflected, err := bson.Marshal(&reflectQueryResultList{
List: []mproto.QueryResult{{
Fields: []mproto.Field{{"name", 1}},
Fields: []mproto.Field{{"name", 1, mproto.VT_ZEROVALUE_FLAG}},
RowsAffected: 2,
InsertId: 3,
Rows: [][]sqltypes.Value{
Expand All @@ -370,7 +341,7 @@ func TestQueryResultList(t *testing.T) {

custom := QueryResultList{
List: []mproto.QueryResult{{
Fields: []mproto.Field{{"name", 1}},
Fields: []mproto.Field{{"name", 1, mproto.VT_ZEROVALUE_FLAG}},
RowsAffected: 2,
InsertId: 3,
Rows: [][]sqltypes.Value{
Expand Down
8 changes: 4 additions & 4 deletions go/vt/vtgate/router_dml_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,8 @@ func TestDeleteEqual(t *testing.T) {

sbc.setResults([]*mproto.QueryResult{&mproto.QueryResult{
Fields: []mproto.Field{
{"id", 3},
{"name", 253},
{"id", 3, mproto.VT_ZEROVALUE_FLAG},
{"name", 253, mproto.VT_ZEROVALUE_FLAG},
},
RowsAffected: 1,
InsertId: 0,
Expand Down Expand Up @@ -257,8 +257,8 @@ func TestDeleteVindexFail(t *testing.T) {

sbc.setResults([]*mproto.QueryResult{&mproto.QueryResult{
Fields: []mproto.Field{
{"id", 3},
{"name", 253},
{"id", 3, mproto.VT_ZEROVALUE_FLAG},
{"name", 253, mproto.VT_ZEROVALUE_FLAG},
},
RowsAffected: 1,
InsertId: 0,
Expand Down
4 changes: 2 additions & 2 deletions go/vt/vtgate/sandbox_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -506,8 +506,8 @@ func (sbc *sandboxConn) getNextResult() *mproto.QueryResult {

var singleRowResult = &mproto.QueryResult{
Fields: []mproto.Field{
{"id", 3},
{"value", 253}},
{"id", 3, mproto.VT_ZEROVALUE_FLAG},
{"value", 253, mproto.VT_ZEROVALUE_FLAG}},
RowsAffected: 1,
InsertId: 0,
Rows: [][]sqltypes.Value{{
Expand Down
2 changes: 1 addition & 1 deletion go/vt/vtgate/vtgateconn/rows_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ func TestRowsFail(t *testing.T) {
ri = NewRows(&badResult2)
dest = make([]driver.Value, 1)
err = ri.Next(dest)
want = `conversion error: field: {field1 3}, val: value: strconv.ParseUint: parsing "value": invalid syntax`
want = `conversion error: field: {field1 3 0}, val: value: strconv.ParseUint: parsing "value": invalid syntax`
if err == nil || err.Error() != want {
t.Errorf("Next: %v, want %s", err, want)
}
Expand Down