Use the emulated MySQL version for special comments

go/vt/servenv/buildinfo.go

@@ -23,6 +23,8 @@ import (
 	"strconv"
 	"time"
 
+	"vitess.io/vitess/go/vt/sqlparser"
+
 	"vitess.io/vitess/go/stats"
 )
 
@@ -104,7 +106,7 @@ func init() {
 		goArch:             runtime.GOARCH,
 		version:            versionName,
 	}
-
+	sqlparser.MySQLVersion = AppVersion.MySQLVersion()
 	stats.NewString("BuildHost").Set(AppVersion.buildHost)
 	stats.NewString("BuildUser").Set(AppVersion.buildUser)
 	stats.NewGauge("BuildTimestamp", "build timestamp").Set(AppVersion.buildTime)

go/vt/sqlparser/comments.go

@@ -181,6 +181,9 @@ func ExtractMysqlComment(sql string) (string, string) {
 	if endOfVersionIndex < 0 {
 		return "", ""
 	}
+	if endOfVersionIndex < 5 {
+		endOfVersionIndex = 0
+	}
 	version := sql[0:endOfVersionIndex]
 	innerSQL := strings.TrimFunc(sql[endOfVersionIndex:], unicode.IsSpace)
 

go/vt/sqlparser/parse_test.go

@@ -1762,6 +1762,9 @@ var (
 		input:      "create database test_db character set * unparsable",
 		output:     "create database test_db",
 		partialDDL: true,
+	}, {
+		input:  "CREATE DATABASE /*!32312 IF NOT EXISTS*/ `mysql` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci */ /*!80016 DEFAULT ENCRYPTION='N' */;",
+		output: "create database if not exists mysql default character set utf8mb4 collate utf8mb4_0900_ai_ci",
 	}, {
 		input: "drop database test_db",
 	}, {

go/vt/sqlparser/parser.go

@@ -35,6 +35,9 @@ var parserPool = sync.Pool{}
 // zeroParser is a zero-initialized parser to help reinitialize the parser for pooling.
 var zeroParser = *(yyNewParser().(*yyParserImpl))
 
+// MySQLVersion is the version of MySQL that the parser would emulate
+var MySQLVersion string = "5.7.9"
+
 // yyParsePooled is a wrapper around yyParse that pools the parser objects. There isn't a
 // particularly good reason to use yyParse directly, since it immediately discards its parser.  What
 // would be ideal down the line is to actually pool the stacks themselves rather than the parser

go/vt/sqlparser/token.go

@@ -20,8 +20,12 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"strconv"
 	"strings"
 
+	"vitess.io/vitess/go/vt/proto/vtrpc"
+	"vitess.io/vitess/go/vt/vterrors"
+
 	"vitess.io/vitess/go/bytes2"
 	"vitess.io/vitess/go/sqltypes"
 )
@@ -1027,11 +1031,59 @@ func (tkn *Tokenizer) scanMySQLSpecificComment() (int, []byte) {
 		}
 		tkn.consumeNext(buffer)
 	}
-	_, sql := ExtractMysqlComment(buffer.String())
-	tkn.specialComment = NewStringTokenizer(sql)
+
+	commentVersion, sql := ExtractMysqlComment(buffer.String())
+
+	mysqlVersion, err := convertMySQLVersionToCommentVersion(MySQLVersion)
+	if err != nil {
+		tkn.Error(err.Error())
+		return tkn.Scan()
+	}
+	if mysqlVersion >= commentVersion {
+		// Only add the special comment to the tokenizer if the version of MySQL is higher or equal to the comment version
+		tkn.specialComment = NewStringTokenizer(sql)
+	}
+
 	return tkn.Scan()
 }
 
+func convertMySQLVersionToCommentVersion(version string) (string, error) {
+	var res = make([]int, 3)
+	idx := 0
+	val := ""
+	for _, c := range version {
+		if c <= '9' && c >= '0' {
+			val += string(c)
+		} else if c == '.' {
+			v, err := strconv.Atoi(val)
+			if err != nil {
+				return "", err
+			}
+			val = ""
+			res[idx] = v
+			idx++
+			if idx == 3 {
+				break
+			}
+		} else {
+			break
+		}
+	}
+	if val != "" {
+		v, err := strconv.Atoi(val)
+		if err != nil {
+			return "", err
+		}
+		res[idx] = v
+		idx++
+	}
+	if idx == 0 {
+		return "", vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "MySQL version not correctly setup - %s.", version)
+	}
+
+	return fmt.Sprintf("%01d%02d%02d", res[0], res[1], res[2]), nil
+}
+
 func (tkn *Tokenizer) consumeNext(buffer *bytes2.Buffer) {
 	if tkn.lastChar == eofChar {
 		// This should never happen.

go/vt/sqlparser/token_test.go

@@ -19,6 +19,8 @@ package sqlparser
 import (
 	"fmt"
 	"testing"
+
+	"github.com/stretchr/testify/require"
 )
 
 func TestLiteralID(t *testing.T) {
@@ -146,10 +148,12 @@ func TestString(t *testing.T) {
 	}}
 
 	for _, tcase := range testcases {
-		id, got := NewStringTokenizer(tcase.in).Scan()
-		if tcase.id != id || string(got) != tcase.want {
-			t.Errorf("Scan(%q) = (%s, %q), want (%s, %q)", tcase.in, tokenName(id), got, tokenName(tcase.id), tcase.want)
-		}
+		t.Run(tcase.in, func(t *testing.T) {
+			id, got := NewStringTokenizer(tcase.in).Scan()
+			if tcase.id != id || string(got) != tcase.want {
+				t.Errorf("Scan(%q) = (%s, %q), want (%s, %q)", tcase.in, tokenName(id), got, tokenName(tcase.id), tcase.want)
+			}
+		})
 	}
 }
 
@@ -205,3 +209,99 @@ func TestSplitStatement(t *testing.T) {
 		}
 	}
 }
+
+func TestVersion(t *testing.T) {
+	testcases := []struct {
+		version string
+		in      string
+		id      []int
+	}{{
+		version: "5.7.9",
+		in:      "/*!80102 SELECT*/ FROM IN EXISTS",
+		id:      []int{FROM, IN, EXISTS, 0},
+	}, {
+		version: "8.1.1",
+		in:      "/*!80102 SELECT*/ FROM IN EXISTS",
+		id:      []int{FROM, IN, EXISTS, 0},
+	}, {
+		version: "8.2.1",
+		in:      "/*!80102 SELECT*/ FROM IN EXISTS",
+		id:      []int{SELECT, FROM, IN, EXISTS, 0},
+	}, {
+		version: "8.1.2",
+		in:      "/*!80102 SELECT*/ FROM IN EXISTS",
+		id:      []int{SELECT, FROM, IN, EXISTS, 0},
+	}}
+
+	for _, tcase := range testcases {
+		t.Run(tcase.version+"_"+tcase.in, func(t *testing.T) {
+			MySQLVersion = tcase.version
+			tok := NewStringTokenizer(tcase.in)
+			for _, expectedID := range tcase.id {
+				id, _ := tok.Scan()
+				require.Equal(t, expectedID, id)
+			}
+		})
+	}
+}
+
+func TestConvertMySQLVersion(t *testing.T) {
+	testcases := []struct {
+		version        string
+		commentVersion string
+		error          string
+	}{{
+		version:        "5.7.9",
+		commentVersion: "50709",
+	}, {
+		version:        "0008.08.9",
+		commentVersion: "80809",
+	}, {
+		version:        "5.7.9, Vitess - 10.0.1",
+		commentVersion: "50709",
+	}, {
+		version:        "8.1 Vitess - 10.0.1",
+		commentVersion: "80100",
+	}, {
+		version: "Vitess - 10.0.1",
+		error:   "MySQL version not correctly setup - Vitess - 10.0.1.",
+	}, {
+		version:        "5.7.9.22",
+		commentVersion: "50709",
+	}}
+
+	for _, tcase := range testcases {
+		t.Run(tcase.version, func(t *testing.T) {
+			output, err := convertMySQLVersionToCommentVersion(tcase.version)
+			if tcase.error != "" {
+				require.EqualError(t, err, tcase.error)
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tcase.commentVersion, output)
+			}
+		})
+	}
+}
+
+func TestExtractMySQLComment(t *testing.T) {
+	testcases := []struct {
+		comment string
+		version string
+	}{{
+		comment: "/*!50108 SELECT * FROM */",
+		version: "50108",
+	}, {
+		comment: "/*!5018 SELECT * FROM */",
+		version: "",
+	}, {
+		comment: "/*!SELECT * FROM */",
+		version: "",
+	}}
+
+	for _, tcase := range testcases {
+		t.Run(tcase.version, func(t *testing.T) {
+			output, _ := ExtractMysqlComment(tcase.comment)
+			require.Equal(t, tcase.version, output)
+		})
+	}
+}