Refactor lexer

pkg/lexer/lexer_test.go

@@ -62,6 +62,73 @@ func TestLexerRegressions(t *testing.T) {
 	}
 }
 
+var _ = Describe("Lexer.Read", func() {
+	It("should not panic if reader is nil", func() {
+		lexer := NewLexer()
+		f := func() {
+			_, err := lexer.Read()
+			Expect(err).To(HaveOccurred())
+		}
+
+		Expect(f).ShouldNot(Panic())
+	})
+	It("should read correctly from reader when re-setting input", func() {
+		lexer := NewLexer()
+		lexer.SetInput(bytes.NewReader([]byte("x")))
+		_, err := lexer.Read()
+		Expect(err).NotTo(HaveOccurred())
+
+		lexer.SetInput(bytes.NewReader([]byte("x")))
+		x, err := lexer.Read()
+		Expect(err).NotTo(HaveOccurred())
+		Expect(x).To(Equal(token.Token{
+			Keyword: keyword.IDENT,
+			Literal: []byte("x"),
+			Position: position.Position{
+				Line: 1,
+				Char: 1,
+			},
+		}))
+	})
+	It("should read eof multiple times correctly", func() {
+		lexer := NewLexer()
+		lexer.SetInput(bytes.NewReader([]byte("x")))
+
+		x, err := lexer.Read()
+		Expect(err).NotTo(HaveOccurred())
+		Expect(x).To(Equal(token.Token{
+			Keyword: keyword.IDENT,
+			Literal: []byte("x"),
+			Position: position.Position{
+				Line: 1,
+				Char: 1,
+			},
+		}))
+
+		eof1, err := lexer.Read()
+		Expect(err).NotTo(HaveOccurred())
+		Expect(eof1).To(Equal(token.Token{
+			Keyword: keyword.EOF,
+			Literal: []byte("eof"),
+			Position: position.Position{
+				Line: 1,
+				Char: 2,
+			},
+		}))
+
+		eof2, err := lexer.Read()
+		Expect(err).NotTo(HaveOccurred())
+		Expect(eof2).To(Equal(token.Token{
+			Keyword: keyword.EOF,
+			Literal: []byte("eof"),
+			Position: position.Position{
+				Line: 1,
+				Char: 2,
+			},
+		}))
+	})
+})
+
 var _ = Describe("Lexer.Read", func() {
 
 	type Case struct {
@@ -99,6 +166,17 @@ var _ = Describe("Lexer.Read", func() {
 				},
 			},
 		}),
+		Entry("should read integer with comma at the end", Case{
+			in: []byte("1337,"),
+			out: token.Token{
+				Keyword: keyword.INTEGER,
+				Literal: []byte("1337"),
+				Position: position.Position{
+					Line: 1,
+					Char: 1,
+				},
+			},
+		}),
 		Entry("should read float", Case{
 			in: []byte("13.37"),
 			out: token.Token{
@@ -717,6 +795,20 @@ var _ = Describe("Lexer.Peek()", func() {
 				},
 			}),
 		}),
+		Entry("should peek ON with whitespace behind", Case{
+			input:              []byte("on "),
+			expectKey:          Equal(keyword.ON),
+			expectErr:          BeNil(),
+			expectNextTokenErr: BeNil(),
+			expectNextToken: Equal(token.Token{
+				Keyword: keyword.ON,
+				Literal: []byte("on"),
+				Position: position.Position{
+					Line: 1,
+					Char: 1,
+				},
+			}),
+		}),
 		Entry("should peek ignore comma", Case{
 			input:     []byte(","),
 			expectKey: Equal(keyword.EOF),
@@ -1115,6 +1207,20 @@ var _ = Describe("Lexer.peekIsFloat", func() {
 	)
 })
 
+func BenchmarkPeekIsFloat(b *testing.B) {
+	input := bytes.NewReader([]byte("13373737.37"))
+	lexer := NewLexer()
+
+	b.ReportAllocs()
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		input.Seek(0, io.SeekStart)
+		lexer.SetInput(input)
+		lexer.peekIsFloat()
+	}
+}
+
 var _ = Describe("Lexer.Read", func() {
 
 	type Case struct {
@@ -1378,6 +1484,35 @@ baz`),
 				},
 			},
 		}),
+		Entry("should read '1,2,3' as three integers", Case{
+			in: []byte("1,2,3"),
+			out: []token.Token{
+				{
+					Keyword: keyword.INTEGER,
+					Literal: []byte("1"),
+					Position: position.Position{
+						Line: 1,
+						Char: 1,
+					},
+				},
+				{
+					Keyword: keyword.INTEGER,
+					Literal: []byte("2"),
+					Position: position.Position{
+						Line: 1,
+						Char: 3,
+					},
+				},
+				{
+					Keyword: keyword.INTEGER,
+					Literal: []byte("3"),
+					Position: position.Position{
+						Line: 1,
+						Char: 5,
+					},
+				},
+			},
+		}),
 	)
 })
 

pkg/parser/parser.go

@@ -32,7 +32,8 @@ func (e errInvalidType) Error() string {
 
 // Parser holds the lexer and a buffer for writing literals
 type Parser struct {
-	l Lexer
+	l                   Lexer
+	selectionSetBuffers []document.SelectionSet
 }
 
 // Lexer is the interface used by the Parser to lex tokens
@@ -45,7 +46,8 @@ type Lexer interface {
 // NewParser returns a new parser using a buffered runestringer
 func NewParser() *Parser {
 	return &Parser{
-		l: lexer.NewLexer(),
+		l:                   lexer.NewLexer(),
+		selectionSetBuffers: make([]document.SelectionSet, 10),
 	}
 }
 
@@ -88,3 +90,21 @@ func (p *Parser) peekExpect(expected keyword.Keyword, swallow bool) (matched boo
 
 	return
 }
+
+func (p *Parser) getSelectionSetBuffer() *document.SelectionSet {
+
+	var s document.SelectionSet
+
+	if len(p.selectionSetBuffers) == 0 {
+		s = make(document.SelectionSet, 10)
+	} else {
+		s, p.selectionSetBuffers = p.selectionSetBuffers[0], p.selectionSetBuffers[1:]
+		s = s[:0]
+	}
+
+	return &s
+}
+
+func (p *Parser) putSelectionSet(set *document.SelectionSet) {
+	p.selectionSetBuffers = append(p.selectionSetBuffers, *set)
+}

pkg/parser/selectionset_parser.go

@@ -16,24 +16,34 @@ func (p *Parser) parseSelectionSet() (selectionSet document.SelectionSet, err er
 		return
 	}
 
+	buffer := p.getSelectionSetBuffer()
+
 	for {
 
 		next, err := p.l.Peek(true)
 		if err != nil {
+			p.putSelectionSet(buffer)
 			return selectionSet, err
 		}
 
 		if next == keyword.CURLYBRACKETCLOSE {
 			_, err = p.l.Read()
+
+			selectionSet = make(document.SelectionSet, len(*buffer))
+			copy(selectionSet, *buffer)
+
+			p.putSelectionSet(buffer)
+
 			return selectionSet, err
 		}
 
 		selection, err := p.parseSelection()
 		if err != nil {
+			p.putSelectionSet(buffer)
 			return selectionSet, err
 		}
 
-		selectionSet = append(selectionSet, selection)
+		*buffer = append(*buffer, selection)
 	}
 
 }