diff --git a/go/sqltypes/bind_variables.go b/go/sqltypes/bind_variables.go index 9473f0efa73..124a9c9d488 100644 --- a/go/sqltypes/bind_variables.go +++ b/go/sqltypes/bind_variables.go @@ -65,6 +65,16 @@ func BuildBindVariables(in map[string]interface{}) (map[string]*querypb.BindVari return out, nil } +// HexNumBindVariable converts bytes representing a hex number to a bind var. +func HexNumBindVariable(v []byte) *querypb.BindVariable { + return ValueBindVariable(NewHexNum(v)) +} + +// HexValBindVariable converts bytes representing a hex encoded string to a bind var. +func HexValBindVariable(v []byte) *querypb.BindVariable { + return ValueBindVariable(NewHexVal(v)) +} + // Int8BindVariable converts an int8 to a bind var. func Int8BindVariable(v int8) *querypb.BindVariable { return ValueBindVariable(NewInt8(v)) diff --git a/go/sqltypes/type.go b/go/sqltypes/type.go index 45edc7b62b5..39afa48a5c1 100644 --- a/go/sqltypes/type.go +++ b/go/sqltypes/type.go @@ -140,6 +140,8 @@ const ( Geometry = querypb.Type_GEOMETRY TypeJSON = querypb.Type_JSON Expression = querypb.Type_EXPRESSION + HexNum = querypb.Type_HEXNUM + HexVal = querypb.Type_HEXVAL ) // bit-shift the mysql flags by two byte so we diff --git a/go/sqltypes/type_test.go b/go/sqltypes/type_test.go index efaeb726121..c21e330a6f3 100644 --- a/go/sqltypes/type_test.go +++ b/go/sqltypes/type_test.go @@ -119,6 +119,12 @@ func TestTypeValues(t *testing.T) { }, { defined: Expression, expected: 31, + }, { + defined: HexNum, + expected: 32 | flagIsText, + }, { + defined: HexVal, + expected: 33 | flagIsText, }} for _, tcase := range testcases { if int(tcase.defined) != tcase.expected { @@ -162,6 +168,8 @@ func TestCategory(t *testing.T) { Geometry, TypeJSON, Expression, + HexNum, + HexVal, } for _, typ := range alltypes { matched := false @@ -192,7 +200,7 @@ func TestCategory(t *testing.T) { } matched = true } - if typ == Null || typ == Decimal || typ == Expression || typ == Bit { + if typ == Null || typ == Decimal || typ == Expression || typ == Bit || typ == HexNum || typ == HexVal { if matched { t.Errorf("%v matched more than one category", typ) } diff --git a/go/sqltypes/value.go b/go/sqltypes/value.go index 85002c8bc4d..c6fbe0fa764 100644 --- a/go/sqltypes/value.go +++ b/go/sqltypes/value.go @@ -79,7 +79,7 @@ func NewValue(typ querypb.Type, val []byte) (v Value, err error) { return NULL, err } return MakeTrusted(typ, val), nil - case IsQuoted(typ) || typ == Bit || typ == Null: + case IsQuoted(typ) || typ == Bit || typ == HexNum || typ == HexVal || typ == Null: return MakeTrusted(typ, val), nil } // All other types are unsafe or invalid. @@ -102,6 +102,16 @@ func MakeTrusted(typ querypb.Type, val []byte) Value { return Value{typ: typ, val: val} } +// NewHexNum builds an Hex Value. +func NewHexNum(v []byte) Value { + return MakeTrusted(HexNum, v) +} + +// NewHexVal builds a HexVal Value. +func NewHexVal(v []byte) Value { + return MakeTrusted(HexVal, v) +} + // NewInt64 builds an Int64 Value. func NewInt64(v int64) Value { return MakeTrusted(Int64, strconv.AppendInt(nil, v, 10)) diff --git a/go/test/endtoend/vtgate/queries/normalize/main_test.go b/go/test/endtoend/vtgate/queries/normalize/main_test.go new file mode 100644 index 00000000000..7d7e6cd8011 --- /dev/null +++ b/go/test/endtoend/vtgate/queries/normalize/main_test.go @@ -0,0 +1,92 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package normalize + +import ( + "flag" + "os" + "testing" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/test/endtoend/cluster" +) + +var ( + clusterInstance *cluster.LocalProcessCluster + vtParams mysql.ConnParams + KeyspaceName = "ks_normalize" + Cell = "test_normalize" + SchemaSQL = ` +create table t1( + id bigint unsigned not null, + charcol char(10), + vcharcol varchar(50), + bincol binary(50), + varbincol varbinary(50), + floatcol float, + deccol decimal(5,2), + bitcol bit, + datecol date, + enumcol enum('small', 'medium', 'large'), + setcol set('a', 'b', 'c'), + jsoncol json, + geocol geometry, + primary key(id) +) Engine=InnoDB; +` +) + +func TestMain(m *testing.M) { + defer cluster.PanicHandler(nil) + flag.Parse() + + exitCode := func() int { + clusterInstance = cluster.NewCluster(Cell, "localhost") + defer clusterInstance.Teardown() + + // Start topo server + err := clusterInstance.StartTopo() + if err != nil { + return 1 + } + + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: KeyspaceName, + SchemaSQL: SchemaSQL, + } + clusterInstance.VtGateExtraArgs = []string{} + clusterInstance.VtTabletExtraArgs = []string{} + err = clusterInstance.StartKeyspace(*keyspace, []string{"-"}, 1, false) + if err != nil { + return 1 + } + + // Start vtgate + err = clusterInstance.StartVtgate() + if err != nil { + return 1 + } + + vtParams = mysql.ConnParams{ + Host: clusterInstance.Hostname, + Port: clusterInstance.VtgateMySQLPort, + } + return m.Run() + }() + os.Exit(exitCode) +} diff --git a/go/test/endtoend/vtgate/queries/normalize/normalize_test.go b/go/test/endtoend/vtgate/queries/normalize/normalize_test.go new file mode 100644 index 00000000000..bf56beb1e63 --- /dev/null +++ b/go/test/endtoend/vtgate/queries/normalize/normalize_test.go @@ -0,0 +1,82 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package normalize + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/test/endtoend/vtgate/utils" +) + +func TestNormalizeAllFields(t *testing.T) { + conn, err := mysql.Connect(context.Background(), &vtParams) + require.NoError(t, err) + defer conn.Close() + + insertQuery := string(`insert into t1 values (1, "chars", "variable chars", x'73757265', 0x676F, 0.33, 9.99, 1, "1976-06-08", "small", "b", "{\"key\":\"value\"}", point(1,5))`) + normalizedInsertQuery := string(`insert into t1 values (:vtg1, :vtg2, :vtg3, :vtg4, :vtg5, :vtg6, :vtg7, :vtg8, :vtg9, :vtg10, :vtg11, :vtg12, point(:vtg13, :vtg14))`) + selectQuery := "select * from t1" + utils.Exec(t, conn, insertQuery) + qr := utils.Exec(t, conn, selectQuery) + assert.Equal(t, 1, len(qr.Rows), "wrong number of table rows, expected 1 but had %d. Results: %v", len(qr.Rows), qr.Rows) + + // Now need to figure out the best way to check the normalized query in the planner cache... + results, err := getPlanCache(fmt.Sprintf("%s:%d", clusterInstance.Hostname, clusterInstance.VtgateProcess.Port)) + require.Nil(t, err) + found := false + for _, record := range results { + key := record["Key"].(string) + if key == normalizedInsertQuery { + found = true + break + } + } + assert.True(t, found, "correctly normalized record not found in planner cache") +} + +func getPlanCache(vtgateHostPort string) ([]map[string]interface{}, error) { + var results []map[string]interface{} + client := http.Client{ + Timeout: 10 * time.Second, + } + resp, err := client.Get(fmt.Sprintf("http://%s/debug/query_plans", vtgateHostPort)) + if err != nil { + return results, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return results, err + } + + err = json.Unmarshal(body, &results) + if err != nil { + return results, err + } + + return results, nil +} diff --git a/go/vt/proto/query/query.pb.go b/go/vt/proto/query/query.pb.go index be2dc2ba141..078247f8ce6 100644 --- a/go/vt/proto/query/query.pb.go +++ b/go/vt/proto/query/query.pb.go @@ -306,6 +306,12 @@ const ( // This type is for internal use only. // Properties: 31, None. Type_EXPRESSION Type = 31 + // HEXNUM specifies a HEXNUM type (unquoted varbinary). + // Properties: 32, IsText. + Type_HEXNUM Type = 4128 + // HEXVAL specifies a HEXVAL type (unquoted varbinary). + // Properties: 33, IsText. + Type_HEXVAL Type = 4129 ) // Enum value maps for Type. @@ -343,6 +349,8 @@ var ( 2077: "GEOMETRY", 2078: "JSON", 31: "EXPRESSION", + 4128: "HEXNUM", + 4129: "HEXVAL", } Type_value = map[string]int32{ "NULL_TYPE": 0, @@ -377,6 +385,8 @@ var ( "GEOMETRY": 2077, "JSON": 2078, "EXPRESSION": 31, + "HEXNUM": 4128, + "HEXVAL": 4129, } ) @@ -5764,7 +5774,7 @@ var file_query_proto_rawDesc = []byte{ 0x0c, 0x0a, 0x07, 0x49, 0x53, 0x46, 0x4c, 0x4f, 0x41, 0x54, 0x10, 0x80, 0x08, 0x12, 0x0d, 0x0a, 0x08, 0x49, 0x53, 0x51, 0x55, 0x4f, 0x54, 0x45, 0x44, 0x10, 0x80, 0x10, 0x12, 0x0b, 0x0a, 0x06, 0x49, 0x53, 0x54, 0x45, 0x58, 0x54, 0x10, 0x80, 0x20, 0x12, 0x0d, 0x0a, 0x08, 0x49, 0x53, 0x42, - 0x49, 0x4e, 0x41, 0x52, 0x59, 0x10, 0x80, 0x40, 0x2a, 0x99, 0x03, 0x0a, 0x04, 0x54, 0x79, 0x70, + 0x49, 0x4e, 0x41, 0x52, 0x59, 0x10, 0x80, 0x40, 0x2a, 0xb3, 0x03, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x55, 0x4c, 0x4c, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x04, 0x49, 0x4e, 0x54, 0x38, 0x10, 0x81, 0x02, 0x12, 0x0a, 0x0a, 0x05, 0x55, 0x49, 0x4e, 0x54, 0x38, 0x10, 0x82, 0x06, 0x12, 0x0a, 0x0a, 0x05, 0x49, 0x4e, 0x54, 0x31, 0x36, @@ -5790,15 +5800,17 @@ var file_query_proto_rawDesc = []byte{ 0x09, 0x0a, 0x05, 0x54, 0x55, 0x50, 0x4c, 0x45, 0x10, 0x1c, 0x12, 0x0d, 0x0a, 0x08, 0x47, 0x45, 0x4f, 0x4d, 0x45, 0x54, 0x52, 0x59, 0x10, 0x9d, 0x10, 0x12, 0x09, 0x0a, 0x04, 0x4a, 0x53, 0x4f, 0x4e, 0x10, 0x9e, 0x10, 0x12, 0x0e, 0x0a, 0x0a, 0x45, 0x58, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, - 0x4f, 0x4e, 0x10, 0x1f, 0x2a, 0x46, 0x0a, 0x10, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, - 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x52, 0x45, 0x50, 0x41, 0x52, 0x45, - 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x10, 0x02, 0x12, 0x0c, - 0x0a, 0x08, 0x52, 0x4f, 0x4c, 0x4c, 0x42, 0x41, 0x43, 0x4b, 0x10, 0x03, 0x42, 0x35, 0x0a, 0x0f, - 0x69, 0x6f, 0x2e, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x5a, - 0x22, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x76, 0x69, 0x74, 0x65, 0x73, - 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x71, 0x75, - 0x65, 0x72, 0x79, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x4f, 0x4e, 0x10, 0x1f, 0x12, 0x0b, 0x0a, 0x06, 0x48, 0x45, 0x58, 0x4e, 0x55, 0x4d, 0x10, 0xa0, + 0x20, 0x12, 0x0b, 0x0a, 0x06, 0x48, 0x45, 0x58, 0x56, 0x41, 0x4c, 0x10, 0xa1, 0x20, 0x2a, 0x46, + 0x0a, 0x10, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, + 0x0b, 0x0a, 0x07, 0x50, 0x52, 0x45, 0x50, 0x41, 0x52, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, + 0x43, 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x52, 0x4f, 0x4c, 0x4c, + 0x42, 0x41, 0x43, 0x4b, 0x10, 0x03, 0x42, 0x35, 0x0a, 0x0f, 0x69, 0x6f, 0x2e, 0x76, 0x69, 0x74, + 0x65, 0x73, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x22, 0x76, 0x69, 0x74, 0x65, 0x73, + 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x76, + 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/go/vt/sqlparser/ast_funcs.go b/go/vt/sqlparser/ast_funcs.go index 596012d9243..b6616030532 100644 --- a/go/vt/sqlparser/ast_funcs.go +++ b/go/vt/sqlparser/ast_funcs.go @@ -17,6 +17,7 @@ limitations under the License. package sqlparser import ( + "bytes" "encoding/hex" "encoding/json" "strings" @@ -480,6 +481,28 @@ func (node *Literal) HexDecode() ([]byte, error) { return hex.DecodeString(node.Val) } +// EncodeHexValToMySQLQueryFormat encodes the hexval back into the query format +// for passing on to MySQL as a bind var +func (node *Literal) encodeHexValToMySQLQueryFormat() ([]byte, error) { + nb := node.Bytes() + if node.Type != HexVal { + return nb, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "Literal value is not a HexVal") + } + + // Let's make this idempotent in case it's called more than once + if nb[0] == 'x' && nb[1] == '0' && nb[len(nb)-1] == '\'' { + return nb, nil + } + + var bb bytes.Buffer + bb.WriteByte('x') + bb.WriteByte('\'') + bb.WriteString(string(nb)) + bb.WriteByte('\'') + nb = bb.Bytes() + return nb, nil +} + // Equal returns true if the column names match. func (node *ColName) Equal(c *ColName) bool { // Failsafe: ColName should not be empty. diff --git a/go/vt/sqlparser/normalizer.go b/go/vt/sqlparser/normalizer.go index 177d9a5ba27..f76d7363602 100644 --- a/go/vt/sqlparser/normalizer.go +++ b/go/vt/sqlparser/normalizer.go @@ -196,6 +196,17 @@ func (nz *normalizer) sqlToBindvar(node SQLNode) *querypb.BindVariable { v, err = sqltypes.NewValue(sqltypes.Int64, node.Bytes()) case FloatVal: v, err = sqltypes.NewValue(sqltypes.Float64, node.Bytes()) + case HexNum: + v, err = sqltypes.NewValue(sqltypes.HexNum, node.Bytes()) + case HexVal: + // We parse the `x'7b7d'` string literal into a hex encoded string of `7b7d` in the parser + // We need to re-encode it back to the original MySQL query format before passing it on as a bindvar value to MySQL + var vbytes []byte + vbytes, err = node.encodeHexValToMySQLQueryFormat() + if err != nil { + return nil + } + v, err = sqltypes.NewValue(sqltypes.HexVal, vbytes) default: return nil } diff --git a/go/vt/sqlparser/normalizer_test.go b/go/vt/sqlparser/normalizer_test.go index 24d7074ce91..f9ebac4e241 100644 --- a/go/vt/sqlparser/normalizer_test.go +++ b/go/vt/sqlparser/normalizer_test.go @@ -131,15 +131,33 @@ func TestNormalize(t *testing.T) { "bv2": sqltypes.TestBindVariable([]interface{}{1, 4, 5}), }, }, { - // Hex value does not convert + // Hex number values should work for selects in: "select * from t where v1 = 0x1234", - outstmt: "select * from t where v1 = 0x1234", - outbv: map[string]*querypb.BindVariable{}, + outstmt: "select * from t where v1 = :bv1", + outbv: map[string]*querypb.BindVariable{ + "bv1": sqltypes.HexNumBindVariable([]byte("0x1234")), + }, }, { - // Hex value does not convert for DMLs - in: "update a set v1 = 0x1234", - outstmt: "update a set v1 = 0x1234", - outbv: map[string]*querypb.BindVariable{}, + // Hex encoded string values should work for selects + in: "select * from t where v1 = x'7b7d'", + outstmt: "select * from t where v1 = :bv1", + outbv: map[string]*querypb.BindVariable{ + "bv1": sqltypes.HexValBindVariable([]byte("x'7b7d'")), + }, + }, { + // Ensure that hex notation bind vars work with collation based conversions + in: "select convert(x'7b7d' using utf8mb4) from dual", + outstmt: "select convert(:bv1 using utf8mb4) from dual", + outbv: map[string]*querypb.BindVariable{ + "bv1": sqltypes.HexValBindVariable([]byte("x'7b7d'")), + }, + }, { + // Hex number values should work for DMLs + in: "update a set v1 = 0x12", + outstmt: "update a set v1 = :bv1", + outbv: map[string]*querypb.BindVariable{ + "bv1": sqltypes.HexNumBindVariable([]byte("0x12")), + }, }, { // Bin value does not convert in: "select * from t where v1 = b'11'", diff --git a/java/jdbc/src/test/java/io/vitess/jdbc/FieldWithMetadataTest.java b/java/jdbc/src/test/java/io/vitess/jdbc/FieldWithMetadataTest.java index d9d5294e89e..85127a2bf1d 100644 --- a/java/jdbc/src/test/java/io/vitess/jdbc/FieldWithMetadataTest.java +++ b/java/jdbc/src/test/java/io/vitess/jdbc/FieldWithMetadataTest.java @@ -294,7 +294,7 @@ public void testPrecisionAdjustFactor() throws SQLException { conn.setIncludedFields(Query.ExecuteOptions.IncludedFields.TYPE_AND_NAME); for (Query.Type type : Query.Type.values()) { - if (type == Query.Type.UNRECOGNIZED || type == Query.Type.EXPRESSION) { + if (type == Query.Type.UNRECOGNIZED || type == Query.Type.EXPRESSION || type == Query.Type.HEXVAL || type == Query.Type.HEXNUM) { continue; } diff --git a/proto/query.proto b/proto/query.proto index 77b4f1610ce..6f3ddc1a304 100644 --- a/proto/query.proto +++ b/proto/query.proto @@ -206,6 +206,12 @@ enum Type { // This type is for internal use only. // Properties: 31, None. EXPRESSION = 31; + // HEXNUM specifies a HEXNUM type (unquoted varbinary). + // Properties: 32, IsText. + HEXNUM = 4128; + // HEXVAL specifies a HEXVAL type (unquoted varbinary). + // Properties: 33, IsText. + HEXVAL = 4129; } // Value represents a typed value. diff --git a/test/config.json b/test/config.json index 76f5110fab4..4f4f1d5dabf 100644 --- a/test/config.json +++ b/test/config.json @@ -633,6 +633,15 @@ "RetryMax": 2, "Tags": [] }, + "vtgate_queries_normalize": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/queries/normalize"], + "Command": [], + "Manual": false, + "Shard": "vtgate_queries", + "RetryMax": 2, + "Tags": [] + }, "vtgate_queries_orderby": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/queries/orderby"],