parser has undo next buffer

field.go

@@ -96,23 +96,18 @@ func (f *NormalField) columns() (cols []aligned) {
 	return
 }
 
-var typeUnknown = false
-
 // parse expects:
 // [ "repeated" | "optional" ] type fieldName "=" fieldNumber [ "[" fieldOptions "]" ] ";"
-func (f *NormalField) parse(p *Parser, typeKnown bool) error {
-	if typeKnown {
-		return parseFieldAfterType(f.Field, p)
-	}
+func (f *NormalField) parse(p *Parser) error {
 	for {
 		_, tok, lit := p.next()
 		switch tok {
 		case tREPEATED:
 			f.Repeated = true
-			return f.parse(p, typeUnknown)
+			return f.parse(p)
 		case tOPTIONAL: // proto2
 			f.Optional = true
-			return f.parse(p, typeUnknown)
+			return f.parse(p)
 		case tIDENT:
 			f.Type = lit
 			return parseFieldAfterType(f.Field, p)
@@ -232,14 +227,14 @@ func (f *MapField) parse(p *Parser) error {
 	if tCOMMA != tok {
 		return p.unexpected(lit, "map type separator ,", f)
 	}
-	_, tok, lit = p.next()
+	_, tok, lit = p.nextIdentifier()
 	if tIDENT != tok {
 		return p.unexpected(lit, "map valueType identifier", f)
 	}
 	f.Type = lit
 	_, tok, lit = p.next()
 	if tGREATER != tok {
-		return p.unexpected(lit, "mak valueType >", f)
+		return p.unexpected(lit, "map valueType >", f)
 	}
 	return parseFieldAfterType(f.Field, p)
 }

field_test.go

@@ -29,7 +29,7 @@ func TestField(t *testing.T) {
 	proto := `repeated foo.bar lots =1 [option1=a, option2=b, option3="happy"];`
 	p := newParserOn(proto)
 	f := newNormalField()
-	err := f.parse(p, typeUnknown)
+	err := f.parse(p)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -69,7 +69,7 @@ func TestFieldSimple(t *testing.T) {
 	proto := `string optional_string_piece = 24 [ctype=STRING_PIECE];`
 	p := newParserOn(proto)
 	f := newNormalField()
-	err := f.parse(p, false)
+	err := f.parse(p)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -99,7 +99,7 @@ func TestFieldSyntaxErrors(t *testing.T) {
 		`string lots === 1;`,
 	} {
 		f := newNormalField()
-		if f.parse(newParserOn(each), typeUnknown) == nil {
+		if f.parse(newParserOn(each)) == nil {
 			t.Errorf("uncaught syntax error in test case %d, %#v", i, f)
 		}
 	}
@@ -131,7 +131,7 @@ func TestOptionalWithOption(t *testing.T) {
 	proto := `optional int32 default_int32    = 61 [default =  41    ];`
 	p := newParserOn(proto)
 	f := newNormalField()
-	err := f.parse(p, typeUnknown)
+	err := f.parse(p)
 	if err != nil {
 		t.Fatal(err)
 	}

message.go

@@ -140,14 +140,15 @@ func parseMessageBody(p *Parser, c elementContainer) error {
 				c.addElement(g)
 			} else {
 				// not a group, will be tFIELD
+				p.nextPut(pos, tok, lit)
 				f := newNormalField()
 				f.Type = lit
 				f.Position = pos
 				f.Comment = c.takeLastComment()
 				f.Optional = prevTok == tOPTIONAL
 				f.Repeated = prevTok == tREPEATED
 				f.Required = prevTok == tREQUIRED
-				if err := f.parse(p, !typeUnknown); err != nil {
+				if err := f.parse(p); err != nil {
 					return err
 				}
 				c.addElement(f)
@@ -185,10 +186,11 @@ func parseMessageBody(p *Parser, c elementContainer) error {
 			// continue
 		default:
 			// tFIELD
+			p.nextPut(pos, tok, lit)
 			f := newNormalField()
 			f.Position = pos
 			f.Comment = c.takeLastComment()
-			if err := f.parse(p, typeUnknown); err != nil {
+			if err := f.parse(p); err != nil {
 				return err
 			}
 			c.addElement(f)

parser.go

@@ -38,6 +38,14 @@ var startPosition = scanner.Position{Line: 1, Column: 1}
 type Parser struct {
 	debug   bool
 	scanner *scanner.Scanner
+	buf     *nextValues
+}
+
+// nextValues is to capture the result of next()
+type nextValues struct {
+	pos scanner.Position
+	tok token
+	lit string
 }
 
 // NewParser returns a new instance of Parser.
@@ -48,14 +56,25 @@ func NewParser(r io.Reader) *Parser {
 	return &Parser{scanner: s}
 }
 
+// func isIdentRune(ch rune, i int) bool {
+// 	// adds the dot to regular Go identifiers
+// 	return ch == '.' || ch == '_' || unicode.IsLetter(ch) || unicode.IsDigit(ch) && i > 0
+// }
+
 // Parse parses a proto definition.
 func (p *Parser) Parse() (*Proto, error) {
 	proto := new(Proto)
 	return proto, proto.parse(p)
 }
 
-// Next returns the next token using the scanner.
+// next returns the next token using the scanner or drain the buffer.
 func (p *Parser) next() (pos scanner.Position, tok token, lit string) {
+	if p.buf != nil {
+		// consume buf
+		vals := *p.buf
+		p.buf = nil
+		return vals.pos, vals.tok, vals.lit
+	}
 	ch := p.scanner.Scan()
 	if ch == scanner.EOF {
 		return p.scanner.Position, tEOF, ""
@@ -64,6 +83,11 @@ func (p *Parser) next() (pos scanner.Position, tok token, lit string) {
 	return p.scanner.Position, asToken(lit), lit
 }
 
+// nextPut sets the buffer
+func (p *Parser) nextPut(pos scanner.Position, tok token, lit string) {
+	p.buf = &nextValues{pos, tok, lit}
+}
+
 func (p *Parser) unexpected(found, expected string, obj interface{}) error {
 	debug := ""
 	if p.debug {
@@ -82,6 +106,31 @@ func (p *Parser) nextInteger() (i int, err error) {
 	return
 }
 
+// TODO
+func (p *Parser) nextIdentifier() (pos scanner.Position, tok token, lit string) {
+	pos, tok, lit = p.next()
+	if tIDENT != tok {
+		return
+	}
+	startPos := pos
+	fullLit := lit
+	// see if identifier is namespaced
+	for {
+		r := p.scanner.Peek()
+		if '.' != r {
+			break
+		}
+		p.next() // consume dot
+		pos, tok, lit := p.next()
+		if tIDENT != tok {
+			p.nextPut(pos, tok, lit)
+			break
+		}
+		fullLit = fmt.Sprintf("%s.%s", fullLit, lit)
+	}
+	return startPos, tIDENT, fullLit
+}
+
 func (p *Parser) peekNonWhitespace() rune {
 	r := p.scanner.Peek()
 	if r == scanner.EOF {

parser_test.go

@@ -83,3 +83,15 @@ func TestScanIgnoreWhitespace_Minus(t *testing.T) {
 		t.Errorf("got [%v] want [%v]", got, want)
 	}
 }
+
+func TestNextIdentifier(t *testing.T) {
+	ident := " aap.noot.mies "
+	p := newParserOn(ident)
+	_, tok, lit := p.nextIdentifier()
+	if got, want := tok, tIDENT; got != want {
+		t.Errorf("got [%v] want [%v]", got, want)
+	}
+	if got, want := lit, strings.TrimSpace(ident); got != want {
+		t.Errorf("got [%v] want [%v]", got, want)
+	}
+}

token.go

@@ -142,6 +142,10 @@ func asToken(literal string) token {
 		return tLESS
 	case ">":
 		return tGREATER
+	case ",":
+		return tCOMMA
+	case ".":
+		return tDOT
 	// words
 	case "syntax":
 		return tSYNTAX