Skip to content

binlog_json: fix opaque value parsing to read variable-length #19102

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nickvanw merged 2 commits into from
Jan 7, 2026
Merged

binlog_json: fix opaque value parsing to read variable-length #19102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5b777f8
binlog_json: fix opaque value parsing to read variable-length size in…
nickvanw Jan 6, 2026
68383cd
code review feedback
nickvanw Jan 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 29 additions & 6 deletions go/mysql/binlog/binlog_json.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -449,11 +449,25 @@ func binparserLiteral(_ jsonDataType, data []byte, pos int) (node *json.Value, e
// other types are stored as catch-all opaque types: documentation on these is scarce.
// we currently know about (and support) date/time/datetime/decimal.
func binparserOpaque(_ jsonDataType, data []byte, pos int) (node *json.Value, err error) {
dataType := data[pos]
start := 3 // account for length of stored value
end := start + 8 // all currently supported opaque data types are 8 bytes in size
if pos >= len(data) {
return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "opaque JSON field value missing type at position %d", pos)
}
typePos := pos
dataType := data[typePos]
pos = typePos + 1
if pos >= len(data) {
return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "opaque JSON field value missing length at position %d", typePos)
}
length, start := readVariableLength(data, pos)
end := start + length
if start > len(data) || end > len(data) {
return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "opaque JSON field value length %d exceeds available bytes", length)
}
switch dataType {
case TypeDate:
if length < 8 {
return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "opaque date length %d is too short", length)
}
raw := binary.LittleEndian.Uint64(data[start:end])
value := raw >> 24
yearMonth := (value >> 22) & 0x01ffff // 17 bits starting at 22nd
Expand All @@ -463,6 +477,9 @@ func binparserOpaque(_ jsonDataType, data []byte, pos int) (node *json.Value, er
dateString := fmt.Sprintf("%04d-%02d-%02d", year, month, day)
node = json.NewDate(dateString)
case TypeTime2, TypeTime:
if length < 8 {
return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "opaque time length %d is too short", length)
}
raw := binary.LittleEndian.Uint64(data[start:end])
value := raw >> 24
hour := (value >> 12) & 0x03ff // 10 bits starting at 12th
Expand All @@ -472,6 +489,9 @@ func binparserOpaque(_ jsonDataType, data []byte, pos int) (node *json.Value, er
timeString := fmt.Sprintf("%02d:%02d:%02d.%06d", hour, minute, second, microSeconds)
node = json.NewTime(timeString)
case TypeDateTime2, TypeDateTime, TypeTimestamp2, TypeTimestamp:
if length < 8 {
return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "opaque datetime length %d is too short", length)
}
raw := binary.LittleEndian.Uint64(data[start:end])
value := raw >> 24
yearMonth := (value >> 22) & 0x01ffff // 17 bits starting at 22nd
Expand All @@ -485,6 +505,9 @@ func binparserOpaque(_ jsonDataType, data []byte, pos int) (node *json.Value, er
timeString := fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d.%06d", year, month, day, hour, minute, second, microSeconds)
node = json.NewDateTime(timeString)
case TypeDecimal, TypeNewDecimal:
if length < 2 {
return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "opaque decimal length %d is too short", length)
}
decimalData := data[start:end]
precision := decimalData[0]
scale := decimalData[1]
Expand All @@ -495,11 +518,11 @@ func binparserOpaque(_ jsonDataType, data []byte, pos int) (node *json.Value, er
}
node = json.NewNumber(val.ToString(), json.NumberTypeDecimal)
case TypeVarchar, TypeVarString, TypeString, TypeBlob, TypeTinyBlob, TypeMediumBlob, TypeLongBlob:
node = json.NewBlob(string(data[pos+1:]))
node = json.NewBlob(string(data[start:end]))
case TypeBit:
node = json.NewBit(string(data[pos+1:]))
node = json.NewBit(string(data[start:end]))
default:
node = json.NewOpaqueValue(string(data[pos+1:]))
node = json.NewOpaqueValue(string(data[start:end]))
}
return node, nil
}
Expand Down
60 changes: 48 additions & 12 deletions go/mysql/binlog/binlog_json_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -232,19 +232,24 @@ func TestBinaryJSON(t *testing.T) {
expected: json.NewNumber("123456789.1234", json.NumberTypeDecimal),
},
{
name: `bit literal [2 202 254]`,
name: `small decimal "1.99"`,
data: []byte{15, 246, 4, 3, 2, 0x81, 0x63},
expected: json.NewNumber("1.99", json.NumberTypeDecimal),
},
{
name: `bit literal 0xCAFE`,
data: []byte{15, 16, 2, 202, 254},
expected: json.NewBit(string([]byte{2, 202, 254})),
expected: json.NewBit(string([]byte{202, 254})),
},
{
name: `opaque string [2 202 254]`,
name: `opaque string 0xCAFE`,
data: []byte{15, 15, 2, 202, 254},
expected: json.NewBlob(string([]byte{2, 202, 254})),
expected: json.NewBlob(string([]byte{202, 254})),
},
{
name: `opaque blob [2 202 254]`,
name: `opaque blob 0xCAFE`,
data: []byte{15, 252, 2, 202, 254},
expected: json.NewBlob(string([]byte{2, 202, 254})),
expected: json.NewBlob(string([]byte{202, 254})),
},
}
for _, tc := range testcases {
Expand All @@ -256,6 +261,36 @@ func TestBinaryJSON(t *testing.T) {
}
}

func TestBinaryJSONOpaqueErrors(t *testing.T) {
testcases := []struct {
name string
data []byte
expectedErr string
}{
{
name: "opaque length exceeds payload",
data: []byte{15, 252, 2, 202},
expectedErr: "opaque JSON field value length 2 exceeds available bytes",
},
{
name: "opaque date too short",
data: []byte{15, 10, 4, 0, 0, 0, 0},
expectedErr: "opaque date length 4 is too short",
},
{
name: "opaque decimal too short",
data: []byte{15, 246, 1, 0x01},
expectedErr: "opaque decimal length 1 is too short",
},
}
for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
_, err := ParseBinaryJSON(tc.data)
require.ErrorContains(t, err, tc.expectedErr)
})
}
}

func TestMarshalJSONToSQL(t *testing.T) {
testcases := []struct {
name string
Expand Down Expand Up @@ -443,19 +478,20 @@ func TestMarshalJSONToSQL(t *testing.T) {
expected: `CAST(123456789.1234 as JSON)`,
},
{
name: `bit literal [2 202 254]`,
// 0xCAFE = 51966 = binary 1100101011111110 (16 bits)
name: `bit literal 0xCAFE`,
data: []byte{15, 16, 2, 202, 254},
expected: `CAST(b'101100101011111110' as JSON)`,
expected: `CAST(b'1100101011111110' as JSON)`,
},
{
name: `opaque string [2 202 254]`,
name: `opaque string 0xCAFE`,
data: []byte{15, 15, 2, 202, 254},
expected: `CAST(x'02CAFE' as JSON)`,
expected: `CAST(x'CAFE' as JSON)`,
},
{
name: `opaque blob [2 202 254]`,
name: `opaque blob 0xCAFE`,
data: []byte{15, 252, 2, 202, 254},
expected: `CAST(x'02CAFE' as JSON)`,
expected: `CAST(x'CAFE' as JSON)`,
},
}
for _, tc := range testcases {
Expand Down
Loading