option work, add debug flag, compacter tests

.vscode/tasks.json

@@ -0,0 +1,9 @@
+{
+    // See https://go.microsoft.com/fwlink/?LinkId=733558
+    // for the documentation about the tasks.json format
+    "version": "0.1.0",
+    "command": "proto3fmt",
+    "isShellCommand": true,
+    "args": ["${file}"],
+    "showOutput": "always"
+}

cmd/proto3fmt/main.go

@@ -7,10 +7,18 @@ import (
 	"github.com/emicklei/proto3"
 )
 
-// go run *.go < example1.proto
-// go run *.go < example0.proto
+// go run *.go example1.proto
+// go run *.go example0.proto
 func main() {
-	p := proto3.NewParser(os.Stdin)
+	if len(os.Args) == 1 {
+		log.Fatal("Usage: proto3fmt my.proto")
+	}
+	i, err := os.Open(os.Args[1])
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer i.Close()
+	p := proto3.NewParser(i)
 	def, err := p.Parse()
 	if err != nil {
 		log.Fatalln("proto3fmt failed, on line", p.Line(), err)

enum.go

@@ -37,7 +37,7 @@ func (f *EnumField) parse(p *Parser) error {
 	f.Name = lit
 	tok, lit = p.scanIgnoreWhitespace()
 	if tok != tEQUALS {
-		return fmt.Errorf("found %q, expected =", lit)
+		return p.unexpected(lit, "=")
 	}
 	is := p.s.scanIntegerString()
 	i, err := strconv.Atoi(is)

enum_test.go

@@ -1,28 +1,8 @@
 package proto3
 
-import (
-	"strings"
-	"testing"
-)
+import "testing"
 
 func TestEnum(t *testing.T) {
-	proto := `enum EnumAllowingAlias {}`
-	p := NewParser(strings.NewReader(proto))
-	_, tok := p.scanIgnoreWhitespace() // consume first token
-	if got, want := tok, "enum"; got != want {
-		t.Errorf("got [%v] want [%v]", got, want)
-	}
-	e := new(Enum)
-	err := e.parse(p)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if got, want := e.Name, "EnumAllowingAlias"; got != want {
-		t.Errorf("got [%v] want [%v]", got, want)
-	}
-}
-
-func TestEnumWithBody(t *testing.T) {
 	proto := `
 // EnumAllowingAlias is part of TestEnumWithBody	
 enum EnumAllowingAlias {
@@ -31,7 +11,7 @@ enum EnumAllowingAlias {
   STARTED = 1;
   RUNNING = 2 [(custom_option) = "hello world"];
 }`
-	p := NewParser(strings.NewReader(proto))
+	p := newParserOn(proto)
 	pr, err := p.Parse()
 	if err != nil {
 		t.Fatal(err)

field.go

@@ -11,7 +11,7 @@ type Field struct {
 	Type     string
 	Repeated bool
 	Sequence int
-	Messages []*Message
+	Options  []*Option
 }
 
 // Accept dispatches the call to the visitor.
@@ -26,20 +26,13 @@ func (f *Field) parse(p *Parser) error {
 		case tIDENT:
 			f.Type = lit
 			return parseNormalField(f, p)
-		case tMESSAGE: // TODO here?
-			m := new(Message)
-			err := m.parse(p)
-			if err != nil {
-				return err
-			}
-			f.Messages = append(f.Messages, m)
 		case tREPEATED:
 			f.Repeated = true
 			return f.parse(p)
 		case tMAP:
 			tok, lit := p.scanIgnoreWhitespace()
 			if tLESS != tok {
-				return fmt.Errorf("found %q, expected <", lit)
+				return p.unexpected(lit, "<")
 			}
 			kvtypes := p.s.scanUntil('>')
 			f.Type = fmt.Sprintf("map<%s>", kvtypes)
@@ -52,22 +45,46 @@ done:
 	return nil
 }
 
-// parseNormalField proceeds after reading the type of f.
+// parseNormalField expects:
+// fieldName "=" fieldNumber [ "[" fieldOptions "]" ] ";
 func parseNormalField(f *Field, p *Parser) error {
 	tok, lit := p.scanIgnoreWhitespace()
 	if tok != tIDENT {
-		return fmt.Errorf("found %q, expected identifier", lit)
+		return p.unexpected(lit, "identifier")
 	}
 	f.Name = lit
 	tok, lit = p.scanIgnoreWhitespace()
 	if tok != tEQUALS {
-		return fmt.Errorf("found %q, expected =", lit)
+		return p.unexpected(lit, "=")
 	}
 	_, lit = p.scanIgnoreWhitespace()
 	i, err := strconv.Atoi(lit)
 	if err != nil {
-		return fmt.Errorf("found %q, expected sequence number", lit)
+		return p.unexpected(lit, "sequence number")
 	}
 	f.Sequence = i
+	// see if there are options
+	tok, lit = p.scanIgnoreWhitespace()
+	if tLEFTSQUARE != tok {
+		p.unscan()
+		return nil
+	}
+	for {
+		o := new(Option)
+		o.PartOfFieldOrEnum = true
+		err := o.parse(p)
+		if err != nil {
+			return err
+		}
+		f.Options = append(f.Options, o)
+
+		tok, lit = p.scanIgnoreWhitespace()
+		if tRIGHTSQUARE == tok {
+			break
+		}
+		if tCOMMA != tok {
+			return p.unexpected(lit, ",")
+		}
+	}
 	return nil
 }

field_test.go

@@ -1,13 +1,10 @@
 package proto3
 
-import (
-	"strings"
-	"testing"
-)
+import "testing"
 
-func TestRepeatedField(t *testing.T) {
-	proto := `repeated string lots = 1;`
-	p := NewParser(strings.NewReader(proto))
+func TestField(t *testing.T) {
+	proto := `repeated foo.bar lots = 1 [option1=a, option2=b, option3="happy"];`
+	p := newParserOn(proto)
 	f := new(Field)
 	err := f.parse(p)
 	if err != nil {
@@ -16,4 +13,39 @@ func TestRepeatedField(t *testing.T) {
 	if got, want := f.Repeated, true; got != want {
 		t.Errorf("got [%v] want [%v]", got, want)
 	}
+	if got, want := f.Type, "foo.bar"; got != want {
+		t.Errorf("got [%v] want [%v]", got, want)
+	}
+	if got, want := f.Name, "lots"; got != want {
+		t.Errorf("got [%v] want [%v]", got, want)
+	}
+	if got, want := len(f.Options), 3; got != want {
+		t.Errorf("got [%v] want [%v]", got, want)
+	}
+	if got, want := f.Options[0].Name, "option1"; got != want {
+		t.Errorf("got [%v] want [%v]", got, want)
+	}
+	if got, want := f.Options[0].Identifier, "a"; got != want {
+		t.Errorf("got [%v] want [%v]", got, want)
+	}
+	if got, want := f.Options[1].Name, "option2"; got != want {
+		t.Errorf("got [%v] want [%v]", got, want)
+	}
+	if got, want := f.Options[1].Identifier, "b"; got != want {
+		t.Errorf("got [%v] want [%v]", got, want)
+	}
+	if got, want := f.Options[2].String, "happy"; got != want {
+		t.Errorf("got [%v] want [%v]", got, want)
+	}
+}
+
+func TestFieldSyntaxErrors(t *testing.T) {
+	for i, each := range []string{
+		`repeatet foo.bar lots = 1;`,
+		`string lots == 1;`,
+	} {
+		if new(Field).parse(newParserOn(each)) == nil {
+			t.Errorf("%d: uncaught syntax error", i)
+		}
+	}
 }

import_test.go

@@ -1,13 +1,10 @@
 package proto3
 
-import (
-	"strings"
-	"testing"
-)
+import "testing"
 
 func TestParseImport(t *testing.T) {
 	proto := `import public "other.proto";`
-	p := NewParser(strings.NewReader(proto))
+	p := newParserOn(proto)
 	p.scanIgnoreWhitespace() // consume first token
 	i := new(Import)
 	err := i.parse(p)

message.go

@@ -38,6 +38,20 @@ func (m *Message) parse(p *Parser) error {
 				return err
 			}
 			m.Elements = append(m.Elements, e)
+		case tMESSAGE:
+			msg := new(Message)
+			err := msg.parse(p)
+			if err != nil {
+				return err
+			}
+			m.Elements = append(m.Elements, msg)
+		case tOPTION:
+			o := new(Option)
+			err := o.parse(p)
+			if err != nil {
+				return err
+			}
+			m.Elements = append(m.Elements, o)
 		case tONEOF:
 			o := new(Oneof)
 			err := o.parse(p)

message_test.go

@@ -1,33 +1,16 @@
 package proto3
 
-import (
-	"strings"
-	"testing"
-)
+import "testing"
 
 func TestMessage(t *testing.T) {
-	proto := `message AccountOut {}`
-	p := NewParser(strings.NewReader(proto))
-	p.scanIgnoreWhitespace() // consume first token
-	m := new(Message)
-	err := m.parse(p)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if got, want := m.Name, "AccountOut"; got != want {
-		t.Errorf("got [%v] want [%v]", got, want)
-	}
-}
-
-func TestMessageWithFieldsAndComments(t *testing.T) {
 	proto := `
 		message AccountOut {
 		// identifier
 		string id   = 1;
 		// size
 		int64 size = 2;
 	}`
-	p := NewParser(strings.NewReader(proto))
+	p := newParserOn(proto)
 	p.scanIgnoreWhitespace() // consume first token
 	m := new(Message)
 	err := m.parse(p)
@@ -51,7 +34,7 @@ func TestOneOf(t *testing.T) {
 		}	
 	}
 `
-	p := NewParser(strings.NewReader(proto))
+	p := newParserOn(proto)
 	p.scanIgnoreWhitespace() // consume first token
 	m := new(Message)
 	err := m.parse(p)

oneof.go

@@ -65,7 +65,7 @@ func (o *OneOfField) parse(p *Parser) error {
 	o.Name = lit
 	tok, lit = p.scanIgnoreWhitespace()
 	if tok != tEQUALS {
-		return fmt.Errorf("found %q, expected =", lit)
+		return p.unexpected(lit, "=")
 	}
 	_, lit = p.scanIgnoreWhitespace()
 	i, err := strconv.Atoi(lit)

oneof_test.go

@@ -1,16 +1,13 @@
 package proto3
 
-import (
-	"strings"
-	"testing"
-)
+import "testing"
 
 func TestOneof(t *testing.T) {
 	proto := `oneof foo {
     string name = 4;
     SubMessage sub_message = 9 [options=none];
 }`
-	p := NewParser(strings.NewReader(proto))
+	p := newParserOn(proto)
 	p.scanIgnoreWhitespace() // consume first token
 	o := new(Oneof)
 	err := o.parse(p)

option.go

@@ -4,41 +4,44 @@ import "fmt"
 
 // Option is a protoc compiler option
 type Option struct {
-	Line    int
-	Name    string
-	String  string
-	Boolean bool
+	Name              string
+	Identifier        string
+	String            string
+	Boolean           bool
+	PartOfFieldOrEnum bool
 }
 
 // Accept dispatches the call to the visitor.
 func (o *Option) Accept(v Visitor) {
 	v.VisitOption(o)
 }
 
+// parse reads an Option body
+// ( ident | "(" fullIdent ")" ) { "." ident } "=" constant ";"
 func (o *Option) parse(p *Parser) error {
 	tok, lit := p.scanIgnoreWhitespace()
 	switch tok {
 	case tIDENT:
-		o.Line = p.s.line
 		o.Name = lit
 	case tLEFTPAREN:
 		tok, lit = p.scanIgnoreWhitespace()
 		if tok != tIDENT {
-			return fmt.Errorf("found %q, expected identifier", lit)
+			return p.unexpected(lit, "identifier")
 		}
 		o.Name = lit
 		tok, lit = p.scanIgnoreWhitespace()
 		if tok != tRIGHTPAREN {
-			return fmt.Errorf("found %q, expected )", lit)
+			return p.unexpected(lit, ")")
 		}
 	default:
-		return fmt.Errorf("found %q, expected identifier or (", lit)
+		return p.unexpected(lit, "identifier or (")
 	}
 	tok, lit = p.scanIgnoreWhitespace()
 	if tok != tEQUALS {
-		return fmt.Errorf("found %q, expected =", lit)
+		return p.unexpected(lit, "=")
 	}
 	tok, lit = p.scanIgnoreWhitespace()
+	// stringLiteral?
 	if tok == tQUOTE {
 		ident := p.s.scanUntil('"')
 		if len(ident) == 0 {
@@ -47,10 +50,9 @@ func (o *Option) parse(p *Parser) error {
 		o.String = ident
 		return nil
 	}
-	if tTRUE == tok || tFALSE == tok {
-		o.Boolean = lit == "true"
-	} else {
-		return fmt.Errorf("found %q, expected true or false", lit)
+	if tIDENT != tok {
+		return p.unexpected(lit, "constant")
 	}
+	o.Identifier = lit
 	return nil
 }