perf: zero-copy tokenizer

go/vt/sqlparser/parse_test.go

@@ -19,7 +19,9 @@ package sqlparser
 import (
 	"bufio"
 	"bytes"
+	"compress/gzip"
 	"fmt"
+	"io"
 	"math/rand"
 	"os"
 	"path"
@@ -3030,7 +3032,7 @@ var (
 		output: "syntax error at position 66 near 'using'",
 	}, {
 		input:        "select 'aa",
-		output:       "syntax error at position 11 near 'aa'",
+		output:       "syntax error at position 12 near 'aa'",
 		excludeMulti: true,
 	}, {
 		input:        "select 'aa\\",
@@ -3097,7 +3099,19 @@ func loadQueries(t testing.TB, filename string) (queries []string) {
 	require.NoError(t, err)
 	defer file.Close()
 
-	scanner := bufio.NewScanner(file)
+	var read io.Reader
+	if strings.HasSuffix(filename, ".gz") {
+		gzread, err := gzip.NewReader(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer gzread.Close()
+		read = gzread
+	} else {
+		read = file
+	}
+
+	scanner := bufio.NewScanner(read)
 	for scanner.Scan() {
 		queries = append(queries, scanner.Text())
 	}
@@ -3113,108 +3127,104 @@ func TestParseDjangoQueries(t *testing.T) {
 	}
 }
 
-// Benchmark run on 6/23/17, prior to improvements:
-// BenchmarkParse1-4         100000             16334 ns/op
-// BenchmarkParse2-4          30000             44121 ns/op
-
-// Benchmark run on 9/3/18, comparing pooled parser performance.
-//
-// benchmark                     old ns/op     new ns/op     delta
-// BenchmarkNormalize-4          2540          2533          -0.28%
-// BenchmarkParse1-4             18269         13330         -27.03%
-// BenchmarkParse2-4             46703         41255         -11.67%
-// BenchmarkParse2Parallel-4     22246         20707         -6.92%
-// BenchmarkParse3-4             4064743       4083135       +0.45%
-//
-// benchmark                     old allocs     new allocs     delta
-// BenchmarkNormalize-4          27             27             +0.00%
-// BenchmarkParse1-4             75             74             -1.33%
-// BenchmarkParse2-4             264            263            -0.38%
-// BenchmarkParse2Parallel-4     176            175            -0.57%
-// BenchmarkParse3-4             360            361            +0.28%
-//
-// benchmark                     old bytes     new bytes     delta
-// BenchmarkNormalize-4          821           821           +0.00%
-// BenchmarkParse1-4             22776         2307          -89.87%
-// BenchmarkParse2-4             28352         7881          -72.20%
-// BenchmarkParse2Parallel-4     25712         5235          -79.64%
-// BenchmarkParse3-4             6352082       6336307       -0.25%
-
-const (
-	sql1 = "select 'abcd', 20, 30.0, eid from a where 1=eid and name='3'"
-	sql2 = "select aaaa, bbb, ccc, ddd, eeee, ffff, gggg, hhhh, iiii from tttt, ttt1, ttt3 where aaaa = bbbb and bbbb = cccc and dddd+1 = eeee group by fff, gggg having hhhh = iiii and iiii = jjjj order by kkkk, llll limit 3, 4"
-)
-
-func BenchmarkParseDjango(b *testing.B) {
-	queries := loadQueries(b, "django_queries.txt")
-	b.ResetTimer()
-
-	for i := 0; i < b.N; i++ {
-		_, err := Parse(queries[rand.Intn(len(queries))])
+func TestParseLobstersQueries(t *testing.T) {
+	for _, query := range loadQueries(t, "lobsters.sql.gz") {
+		_, err := Parse(query)
 		if err != nil {
-			b.Fatal(err)
+			t.Errorf("failed to parse %q: %v", query, err)
 		}
 	}
 }
 
-func BenchmarkParse1(b *testing.B) {
-	sql := sql1
-	for i := 0; i < b.N; i++ {
-		_, err := Parse(sql)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
+func BenchmarkParseTraces(b *testing.B) {
+	for _, trace := range []string{"django_queries.txt", "lobsters.sql.gz"} {
+		b.Run(trace, func(b *testing.B) {
+			queries := loadQueries(b, trace)
+			if len(queries) > 10000 {
+				queries = queries[:10000]
+			}
+			b.ResetTimer()
+			b.ReportAllocs()
 
-func BenchmarkParse2(b *testing.B) {
-	sql := sql2
-	for i := 0; i < b.N; i++ {
-		_, err := Parse(sql)
-		if err != nil {
-			b.Fatal(err)
-		}
+			for i := 0; i < b.N; i++ {
+				for _, query := range queries {
+					_, err := Parse(query)
+					if err != nil {
+						b.Fatal(err)
+					}
+				}
+			}
+		})
 	}
+
 }
 
-func BenchmarkParse2Parallel(b *testing.B) {
-	sql := sql2
-	b.RunParallel(func(pb *testing.PB) {
-		for pb.Next() {
-			ast, err := Parse(sql)
-			if err != nil {
-				b.Fatal(err)
+func BenchmarkParseStress(b *testing.B) {
+	const (
+		sql1 = "select 'abcd', 20, 30.0, eid from a where 1=eid and name='3'"
+		sql2 = "select aaaa, bbb, ccc, ddd, eeee, ffff, gggg, hhhh, iiii from tttt, ttt1, ttt3 where aaaa = bbbb and bbbb = cccc and dddd+1 = eeee group by fff, gggg having hhhh = iiii and iiii = jjjj order by kkkk, llll limit 3, 4"
+	)
+
+	for i, sql := range []string{sql1, sql2} {
+		b.Run(fmt.Sprintf("sql%d", i), func(b *testing.B) {
+			var buf bytes.Buffer
+			buf.WriteString(sql)
+			querySQL := buf.String()
+			b.ReportAllocs()
+			b.ResetTimer()
+
+			for i := 0; i < b.N; i++ {
+				_, err := Parse(querySQL)
+				if err != nil {
+					b.Fatal(err)
+				}
 			}
-			_ = ast
-		}
-	})
+		})
+	}
 }
 
 func BenchmarkParse3(b *testing.B) {
-	// benchQuerySize is the approximate size of the query.
-	benchQuerySize := 1000000
-
-	// Size of value is 1/10 size of query. Then we add
-	// 10 such values to the where clause.
-	var baseval bytes.Buffer
-	for i := 0; i < benchQuerySize/100; i++ {
-		// Add an escape character: This will force the upcoming
-		// tokenizer improvement to still create a copy of the string.
-		// Then we can see if avoiding the copy will be worth it.
-		baseval.WriteString("\\'123456789")
-	}
+	largeQueryBenchmark := func(b *testing.B, escape bool) {
+		b.Helper()
 
-	var buf bytes.Buffer
-	buf.WriteString("select a from t1 where v = 1")
-	for i := 0; i < 10; i++ {
-		fmt.Fprintf(&buf, " and v%d = \"%d%s\"", i, i, baseval.String())
-	}
-	benchQuery := buf.String()
-	b.ResetTimer()
+		// benchQuerySize is the approximate size of the query.
+		benchQuerySize := 1000000
 
-	for i := 0; i < b.N; i++ {
-		if _, err := Parse(benchQuery); err != nil {
-			b.Fatal(err)
+		// Size of value is 1/10 size of query. Then we add
+		// 10 such values to the where clause.
+		var baseval bytes.Buffer
+		for i := 0; i < benchQuerySize/100; i++ {
+			// Add an escape character: This will force the upcoming
+			// tokenizer improvement to still create a copy of the string.
+			// Then we can see if avoiding the copy will be worth it.
+			if escape {
+				baseval.WriteString("\\'123456789")
+			} else {
+				baseval.WriteString("123456789")
+			}
+		}
+
+		var buf bytes.Buffer
+		buf.WriteString("select a from t1 where v = 1")
+		for i := 0; i < 10; i++ {
+			fmt.Fprintf(&buf, " and v%d = \"%d%s\"", i, i, baseval.String())
+		}
+		benchQuery := buf.String()
+		b.ResetTimer()
+		b.ReportAllocs()
+
+		for i := 0; i < b.N; i++ {
+			if _, err := Parse(benchQuery); err != nil {
+				b.Fatal(err)
+			}
 		}
 	}
+
+	b.Run("normal", func(b *testing.B) {
+		largeQueryBenchmark(b, false)
+	})
+
+	b.Run("escaped", func(b *testing.B) {
+		largeQueryBenchmark(b, true)
+	})
 }

go/vt/sqlparser/parser.go

@@ -135,11 +135,11 @@ func ParseNextStrictDDL(tokenizer *Tokenizer) (Statement, error) {
 }
 
 func parseNext(tokenizer *Tokenizer, strict bool) (Statement, error) {
-	if tokenizer.lastChar == ';' {
-		tokenizer.next()
+	if tokenizer.cur() == ';' {
+		tokenizer.skip(1)
 		tokenizer.skipBlank()
 	}
-	if tokenizer.lastChar == eofChar {
+	if tokenizer.cur() == eofChar {
 		return nil, io.EOF
 	}
 
@@ -176,7 +176,7 @@ func SplitStatement(blob string) (string, string, error) {
 		return "", "", tokenizer.LastError
 	}
 	if tkn == ';' {
-		return blob[:tokenizer.Position-2], blob[tokenizer.Position-1:], nil
+		return blob[:tokenizer.Pos-1], blob[tokenizer.Pos:], nil
 	}
 	return blob, "", nil
 }
@@ -196,14 +196,14 @@ loop:
 		tkn, _ = tokenizer.Scan()
 		switch tkn {
 		case ';':
-			stmt = blob[stmtBegin : tokenizer.Position-2]
+			stmt = blob[stmtBegin : tokenizer.Pos-1]
 			if !emptyStatement {
 				pieces = append(pieces, stmt)
 				emptyStatement = true
 			}
-			stmtBegin = tokenizer.Position - 1
+			stmtBegin = tokenizer.Pos
 		case 0, eofChar:
-			blobTail := tokenizer.Position - 2
+			blobTail := tokenizer.Pos - 1
 			if stmtBegin < blobTail {
 				stmt = blob[stmtBegin : blobTail+1]
 				if !emptyStatement {