make tokens unexposed

uber-go · Jan 28, 2017 · 1ab5546 · 1ab5546
1 parent 768352c
commit 1ab5546
Show file tree

Hide file tree

Showing 14 changed files with 135 additions and 128 deletions.
diff --git a/README.md b/README.md
@@ -1,11 +1,12 @@
 # proto3
 
 [![Build Status](https://travis-ci.org/emicklei/proto3.png)](https://travis-ci.org/emicklei/proto3)
+[![GoDoc](https://godoc.org/github.com/emicklei/proto3?status.svg)](https://godoc.org/github.com/emicklei/proto3)
 
 Package in Go for parsing Google Protocol Buffers [.proto files version 3] (https://developers.google.com/protocol-buffers/docs/reference/proto3-spec)
 
 ### install
 
-    go get -u -v github.com/emicklei/proto
+    go get -u -v github.com/emicklei/proto3
 
-(c) 2017, ernestmicklei.com.  MIT License. Contributions welcome.
+© 2017, ernestmicklei.com.  MIT License. Contributions welcome.
diff --git a/enum.go b/enum.go
@@ -5,13 +5,15 @@ import (
 	"strconv"
 )
 
+// Enum definition consists of a name and an enum body.
 type Enum struct {
 	Line       int
 	Name       string
 	Options    []*Option
 	EnumFields []*EnumField
 }
 
+// EnumField is part of the body of an Enum.
 type EnumField struct {
 	Name        string
 	Integer     int
@@ -20,11 +22,11 @@ type EnumField struct {
 
 func (f *EnumField) parse(p *Parser) error {
 	tok, lit := p.scanIgnoreWhitespace()
-	if tok != IDENT {
+	if tok != tIDENT {
 		return fmt.Errorf("found %q, expected identifier", lit)
 	}
 	tok, lit = p.scanIgnoreWhitespace()
-	if tok != EQUALS {
+	if tok != tEQUALS {
 		return fmt.Errorf("found %q, expected =", lit)
 	}
 	is := p.s.scanIntegerString()
@@ -34,15 +36,15 @@ func (f *EnumField) parse(p *Parser) error {
 	}
 	f.Integer = i
 	tok, lit = p.scanIgnoreWhitespace()
-	if tok == LEFTSQUARE {
+	if tok == tLEFTSQUARE {
 		o := new(Option)
 		err := o.parse(p)
 		if err != nil {
 			return err
 		}
 		f.ValueOption = o
 		tok, lit = p.scanIgnoreWhitespace()
-		if tok != RIGHTSQUARE {
+		if tok != tRIGHTSQUARE {
 			return fmt.Errorf("found %q, expected ]", lit)
 		}
 	}
@@ -51,21 +53,21 @@ func (f *EnumField) parse(p *Parser) error {
 
 func (e *Enum) parse(p *Parser) error {
 	tok, lit := p.scanIgnoreWhitespace()
-	if tok != IDENT {
+	if tok != tIDENT {
 		return fmt.Errorf("found %q, expected identifier", lit)
 	}
 	e.Name = lit
 	tok, lit = p.scanIgnoreWhitespace()
-	if tok != LEFTCURLY {
+	if tok != tLEFTCURLY {
 		return fmt.Errorf("found %q, expected {", lit)
 	}
 	for {
 		tok, lit = p.scanIgnoreWhitespace()
 		switch tok {
-		case RIGHTCURLY:
+		case tRIGHTCURLY:
 			goto done
-		case SEMICOLON:
-		case OPTION:
+		case tSEMICOLON:
+		case tOPTION:
 			v := new(Option)
 			err := v.parse(p)
 			if err != nil {
@@ -83,7 +85,7 @@ func (e *Enum) parse(p *Parser) error {
 		}
 	}
 done:
-	if tok != RIGHTCURLY {
+	if tok != tRIGHTCURLY {
 		return fmt.Errorf("found %q, expected }", lit)
 	}
 	return nil

diff --git a/enum_test.go b/enum_test.go
@@ -24,6 +24,7 @@ func TestEnum(t *testing.T) {
 
 func TestEnumWithBody(t *testing.T) {
 	proto := `
+// EnumAllowingAlias is part of TestEnumWithBody	
 enum EnumAllowingAlias {
   option allow_alias = true;
   UNKNOWN = 0;

diff --git a/field.go b/field.go
@@ -19,22 +19,22 @@ func (f *Field) parse(p *Parser) error {
 	for {
 		tok, lit := p.scanIgnoreWhitespace()
 		switch tok {
-		case IDENT:
+		case tIDENT:
 			// normal type?
 			if strings.Contains(typeTokens, lit) {
 				f.Type = lit
 				return parseNormalField(f, p)
 			}
 			//if tok == ONEOF {}
 			//if tok == ONEOFFIELD {}
-		case MESSAGE:
+		case tMESSAGE:
 			m := new(Message)
 			err := m.parse(p)
 			if err != nil {
 				return err
 			}
 			f.Messages = append(f.Messages, m)
-		case REPEATED:
+		case tREPEATED:
 			f.Repeated = true
 			return f.parse(p)
 		default:
@@ -48,12 +48,12 @@ done:
 // parseNormalField proceeds after reading the type of f.
 func parseNormalField(f *Field, p *Parser) error {
 	tok, lit := p.scanIgnoreWhitespace()
-	if tok != IDENT {
+	if tok != tIDENT {
 		return fmt.Errorf("found %q, expected identifier", lit)
 	}
 	f.Name = lit
 	tok, lit = p.scanIgnoreWhitespace()
-	if tok != EQUALS {
+	if tok != tEQUALS {
 		return fmt.Errorf("found %q, expected =", lit)
 	}
 	_, lit = p.scanIgnoreWhitespace()

diff --git a/import.go b/import.go
@@ -13,13 +13,13 @@ func (i *Import) parse(p *Parser) error {
 	tok, lit := p.scanIgnoreWhitespace()
 	i.Line = p.s.line
 	switch tok {
-	case WEAK:
+	case tWEAK:
 		i.Kind = lit
 		return i.parse(p)
-	case PUBLIC:
+	case tPUBLIC:
 		i.Kind = lit
 		return i.parse(p)
-	case QUOTE:
+	case tQUOTE:
 		i.Filename = p.s.scanUntil('"')
 	default:
 		return fmt.Errorf("found %q, expected weak|public|quoted identifier", lit)

diff --git a/message.go b/message.go
@@ -9,26 +9,34 @@ type Message struct {
 
 	Name   string
 	Fields []*Field
+	Enums  []*Enum
 }
 
 func (m *Message) parse(p *Parser) error {
 	tok, lit := p.scanIgnoreWhitespace()
-	if tok != IDENT {
+	if tok != tIDENT {
 		return fmt.Errorf("found %q, expected identifier", lit)
 	}
 	m.Name = lit
 	tok, lit = p.scanIgnoreWhitespace()
-	if tok != LEFTCURLY {
+	if tok != tLEFTCURLY {
 		return fmt.Errorf("found %q, expected {", lit)
 	}
 	for {
 		tok, lit = p.scanIgnoreWhitespace()
 		switch tok {
-		case COMMENT:
+		case tCOMMENT:
 			m.Comments = append(m.Comments, p.newComment(lit))
-		case RIGHTCURLY:
+		case tRIGHTCURLY:
 			goto done
-		case SEMICOLON:
+		case tSEMICOLON:
+		case tENUM:
+			e := new(Enum)
+			err := e.parse(p)
+			if err != nil {
+				return err
+			}
+			m.Enums = append(m.Enums, e)
 		default:
 			p.unscan()
 			f := new(Field)
@@ -40,7 +48,7 @@ func (m *Message) parse(p *Parser) error {
 		}
 	}
 done:
-	if tok != RIGHTCURLY {
+	if tok != tRIGHTCURLY {
 		return fmt.Errorf("found %q, expected }", lit)
 	}
 	return nil

diff --git a/option.go b/option.go
@@ -18,36 +18,36 @@ func (o *Option) accept(v Visitor) {
 func (o *Option) parse(p *Parser) error {
 	tok, lit := p.scanIgnoreWhitespace()
 	switch tok {
-	case IDENT:
+	case tIDENT:
 		o.Line = p.s.line
 		o.Name = lit
-	case LEFTPAREN:
+	case tLEFTPAREN:
 		tok, lit = p.scanIgnoreWhitespace()
-		if tok != IDENT {
+		if tok != tIDENT {
 			return fmt.Errorf("found %q, expected identifier", lit)
 		}
 		o.Name = lit
 		tok, lit = p.scanIgnoreWhitespace()
-		if tok != RIGHTPAREN {
+		if tok != tRIGHTPAREN {
 			return fmt.Errorf("found %q, expected )", lit)
 		}
 	default:
 		return fmt.Errorf("found %q, expected identifier or (", lit)
 	}
 	tok, lit = p.scanIgnoreWhitespace()
-	if tok != EQUALS {
+	if tok != tEQUALS {
 		return fmt.Errorf("found %q, expected =", lit)
 	}
 	tok, lit = p.scanIgnoreWhitespace()
-	if tok == QUOTE {
+	if tok == tQUOTE {
 		ident := p.s.scanUntil('"')
 		if len(ident) == 0 {
 			return fmt.Errorf("unexpected end of quoted string") // TODO create constant for this
 		}
 		o.String = ident
 		return nil
 	}
-	if TRUE == tok || FALSE == tok {
+	if tTRUE == tok || tFALSE == tok {
 		o.Boolean = lit == "true"
 	} else {
 		return fmt.Errorf("found %q, expected true or false", lit)

diff --git a/parser.go b/parser.go
@@ -45,7 +45,7 @@ func (p *Parser) scan() (tok token, lit string) {
 // scanIgnoreWhitespace scans the next non-whitespace token.
 func (p *Parser) scanIgnoreWhitespace() (tok token, lit string) {
 	tok, lit = p.scan()
-	if tok == WS {
+	if tok == tWS {
 		tok, lit = p.scan()
 	}
 	return

diff --git a/proto.go b/proto.go
@@ -27,41 +27,40 @@ func (c Comment) IsMultiline() bool {
 func (proto *Proto) parse(p *Parser) error {
 	tok, lit := p.scanIgnoreWhitespace()
 	switch tok {
-	case COMMENT:
+	case tCOMMENT:
 		proto.Comments = append(proto.Comments, p.newComment(lit))
-	case SYNTAX:
+	case tSYNTAX:
 		s := new(Syntax)
 		if err := s.parse(p); err != nil {
 			return err
 		}
 		proto.Syntax = s
-	case IMPORT:
+	case tIMPORT:
 		im := new(Import)
 		if err := im.parse(p); err != nil {
 			return err
 		}
 		proto.Imports = append(proto.Imports, im)
-	case ENUM:
+	case tENUM:
 		enum := new(Enum)
 		if err := enum.parse(p); err != nil {
 			return err
 		}
 		proto.Enums = append(proto.Enums, enum)
-	case SERVICE:
-		// TODO
+	case tSERVICE:
 		service := new(Service)
 		err := service.parse(p)
 		if err != nil {
 			return err
 		}
 		proto.Services = append(proto.Services, service)
-	case MESSAGE:
+	case tMESSAGE:
 		msg := new(Message)
 		if err := msg.parse(p); err != nil {
 			return err
 		}
 		proto.Messages = append(proto.Messages, msg)
-	case EOF:
+	case tEOF:
 		return nil
 	}
 	return proto.parse(p)