From 7bc2316231e0836c243854c0c6e3d6c673ee974e Mon Sep 17 00:00:00 2001 From: Ernest Micklei Date: Sun, 29 Jan 2017 08:27:58 +0100 Subject: [PATCH] option work, add debug flag, compacter tests --- .vscode/tasks.json | 9 +++++++++ cmd/proto3fmt/main.go | 14 ++++++++++--- enum.go | 2 +- enum_test.go | 24 ++-------------------- field.go | 43 ++++++++++++++++++++++++++++------------ field_test.go | 46 ++++++++++++++++++++++++++++++++++++------- import_test.go | 7 ++----- message.go | 14 +++++++++++++ message_test.go | 23 +++------------------- oneof.go | 2 +- oneof_test.go | 7 ++----- option.go | 28 ++++++++++++++------------ option_test.go | 27 +++++-------------------- parser.go | 17 ++++++++++++++-- parser_test.go | 8 +++++++- scanner.go | 6 ++---- service_test.go | 18 ++--------------- syntax.go | 2 +- syntax_test.go | 5 ++--- token.go | 3 +-- 20 files changed, 164 insertions(+), 141 deletions(-) create mode 100644 .vscode/tasks.json diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..d1d41bc --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,9 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "0.1.0", + "command": "proto3fmt", + "isShellCommand": true, + "args": ["${file}"], + "showOutput": "always" +} \ No newline at end of file diff --git a/cmd/proto3fmt/main.go b/cmd/proto3fmt/main.go index 4d86f14..516fd77 100644 --- a/cmd/proto3fmt/main.go +++ b/cmd/proto3fmt/main.go @@ -7,10 +7,18 @@ import ( "github.com/emicklei/proto3" ) -// go run *.go < example1.proto -// go run *.go < example0.proto +// go run *.go example1.proto +// go run *.go example0.proto func main() { - p := proto3.NewParser(os.Stdin) + if len(os.Args) == 1 { + log.Fatal("Usage: proto3fmt my.proto") + } + i, err := os.Open(os.Args[1]) + if err != nil { + log.Fatal(err) + } + defer i.Close() + p := proto3.NewParser(i) def, err := p.Parse() if err != nil { log.Fatalln("proto3fmt failed, on line", p.Line(), err) diff --git a/enum.go b/enum.go index 7b76361..d3e55c0 100644 --- a/enum.go +++ b/enum.go @@ -37,7 +37,7 @@ func (f *EnumField) parse(p *Parser) error { f.Name = lit tok, lit = p.scanIgnoreWhitespace() if tok != tEQUALS { - return fmt.Errorf("found %q, expected =", lit) + return p.unexpected(lit, "=") } is := p.s.scanIntegerString() i, err := strconv.Atoi(is) diff --git a/enum_test.go b/enum_test.go index d2f9cd3..d1281fb 100644 --- a/enum_test.go +++ b/enum_test.go @@ -1,28 +1,8 @@ package proto3 -import ( - "strings" - "testing" -) +import "testing" func TestEnum(t *testing.T) { - proto := `enum EnumAllowingAlias {}` - p := NewParser(strings.NewReader(proto)) - _, tok := p.scanIgnoreWhitespace() // consume first token - if got, want := tok, "enum"; got != want { - t.Errorf("got [%v] want [%v]", got, want) - } - e := new(Enum) - err := e.parse(p) - if err != nil { - t.Fatal(err) - } - if got, want := e.Name, "EnumAllowingAlias"; got != want { - t.Errorf("got [%v] want [%v]", got, want) - } -} - -func TestEnumWithBody(t *testing.T) { proto := ` // EnumAllowingAlias is part of TestEnumWithBody enum EnumAllowingAlias { @@ -31,7 +11,7 @@ enum EnumAllowingAlias { STARTED = 1; RUNNING = 2 [(custom_option) = "hello world"]; }` - p := NewParser(strings.NewReader(proto)) + p := newParserOn(proto) pr, err := p.Parse() if err != nil { t.Fatal(err) diff --git a/field.go b/field.go index 7e14e73..610b212 100644 --- a/field.go +++ b/field.go @@ -11,7 +11,7 @@ type Field struct { Type string Repeated bool Sequence int - Messages []*Message + Options []*Option } // Accept dispatches the call to the visitor. @@ -26,20 +26,13 @@ func (f *Field) parse(p *Parser) error { case tIDENT: f.Type = lit return parseNormalField(f, p) - case tMESSAGE: // TODO here? - m := new(Message) - err := m.parse(p) - if err != nil { - return err - } - f.Messages = append(f.Messages, m) case tREPEATED: f.Repeated = true return f.parse(p) case tMAP: tok, lit := p.scanIgnoreWhitespace() if tLESS != tok { - return fmt.Errorf("found %q, expected <", lit) + return p.unexpected(lit, "<") } kvtypes := p.s.scanUntil('>') f.Type = fmt.Sprintf("map<%s>", kvtypes) @@ -52,22 +45,46 @@ done: return nil } -// parseNormalField proceeds after reading the type of f. +// parseNormalField expects: +// fieldName "=" fieldNumber [ "[" fieldOptions "]" ] "; func parseNormalField(f *Field, p *Parser) error { tok, lit := p.scanIgnoreWhitespace() if tok != tIDENT { - return fmt.Errorf("found %q, expected identifier", lit) + return p.unexpected(lit, "identifier") } f.Name = lit tok, lit = p.scanIgnoreWhitespace() if tok != tEQUALS { - return fmt.Errorf("found %q, expected =", lit) + return p.unexpected(lit, "=") } _, lit = p.scanIgnoreWhitespace() i, err := strconv.Atoi(lit) if err != nil { - return fmt.Errorf("found %q, expected sequence number", lit) + return p.unexpected(lit, "sequence number") } f.Sequence = i + // see if there are options + tok, lit = p.scanIgnoreWhitespace() + if tLEFTSQUARE != tok { + p.unscan() + return nil + } + for { + o := new(Option) + o.PartOfFieldOrEnum = true + err := o.parse(p) + if err != nil { + return err + } + f.Options = append(f.Options, o) + + tok, lit = p.scanIgnoreWhitespace() + if tRIGHTSQUARE == tok { + break + } + if tCOMMA != tok { + return p.unexpected(lit, ",") + } + } return nil } diff --git a/field_test.go b/field_test.go index a99950e..1d1a50f 100644 --- a/field_test.go +++ b/field_test.go @@ -1,13 +1,10 @@ package proto3 -import ( - "strings" - "testing" -) +import "testing" -func TestRepeatedField(t *testing.T) { - proto := `repeated string lots = 1;` - p := NewParser(strings.NewReader(proto)) +func TestField(t *testing.T) { + proto := `repeated foo.bar lots = 1 [option1=a, option2=b, option3="happy"];` + p := newParserOn(proto) f := new(Field) err := f.parse(p) if err != nil { @@ -16,4 +13,39 @@ func TestRepeatedField(t *testing.T) { if got, want := f.Repeated, true; got != want { t.Errorf("got [%v] want [%v]", got, want) } + if got, want := f.Type, "foo.bar"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } + if got, want := f.Name, "lots"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } + if got, want := len(f.Options), 3; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } + if got, want := f.Options[0].Name, "option1"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } + if got, want := f.Options[0].Identifier, "a"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } + if got, want := f.Options[1].Name, "option2"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } + if got, want := f.Options[1].Identifier, "b"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } + if got, want := f.Options[2].String, "happy"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } +} + +func TestFieldSyntaxErrors(t *testing.T) { + for i, each := range []string{ + `repeatet foo.bar lots = 1;`, + `string lots == 1;`, + } { + if new(Field).parse(newParserOn(each)) == nil { + t.Errorf("%d: uncaught syntax error", i) + } + } } diff --git a/import_test.go b/import_test.go index f48a8e0..cf4aada 100644 --- a/import_test.go +++ b/import_test.go @@ -1,13 +1,10 @@ package proto3 -import ( - "strings" - "testing" -) +import "testing" func TestParseImport(t *testing.T) { proto := `import public "other.proto";` - p := NewParser(strings.NewReader(proto)) + p := newParserOn(proto) p.scanIgnoreWhitespace() // consume first token i := new(Import) err := i.parse(p) diff --git a/message.go b/message.go index 90dc155..0081fb1 100644 --- a/message.go +++ b/message.go @@ -38,6 +38,20 @@ func (m *Message) parse(p *Parser) error { return err } m.Elements = append(m.Elements, e) + case tMESSAGE: + msg := new(Message) + err := msg.parse(p) + if err != nil { + return err + } + m.Elements = append(m.Elements, msg) + case tOPTION: + o := new(Option) + err := o.parse(p) + if err != nil { + return err + } + m.Elements = append(m.Elements, o) case tONEOF: o := new(Oneof) err := o.parse(p) diff --git a/message_test.go b/message_test.go index c3bc7e2..a0e1e8f 100644 --- a/message_test.go +++ b/message_test.go @@ -1,25 +1,8 @@ package proto3 -import ( - "strings" - "testing" -) +import "testing" func TestMessage(t *testing.T) { - proto := `message AccountOut {}` - p := NewParser(strings.NewReader(proto)) - p.scanIgnoreWhitespace() // consume first token - m := new(Message) - err := m.parse(p) - if err != nil { - t.Fatal(err) - } - if got, want := m.Name, "AccountOut"; got != want { - t.Errorf("got [%v] want [%v]", got, want) - } -} - -func TestMessageWithFieldsAndComments(t *testing.T) { proto := ` message AccountOut { // identifier @@ -27,7 +10,7 @@ func TestMessageWithFieldsAndComments(t *testing.T) { // size int64 size = 2; }` - p := NewParser(strings.NewReader(proto)) + p := newParserOn(proto) p.scanIgnoreWhitespace() // consume first token m := new(Message) err := m.parse(p) @@ -51,7 +34,7 @@ func TestOneOf(t *testing.T) { } } ` - p := NewParser(strings.NewReader(proto)) + p := newParserOn(proto) p.scanIgnoreWhitespace() // consume first token m := new(Message) err := m.parse(p) diff --git a/oneof.go b/oneof.go index 1eecfad..c3d7287 100644 --- a/oneof.go +++ b/oneof.go @@ -65,7 +65,7 @@ func (o *OneOfField) parse(p *Parser) error { o.Name = lit tok, lit = p.scanIgnoreWhitespace() if tok != tEQUALS { - return fmt.Errorf("found %q, expected =", lit) + return p.unexpected(lit, "=") } _, lit = p.scanIgnoreWhitespace() i, err := strconv.Atoi(lit) diff --git a/oneof_test.go b/oneof_test.go index 7349752..7321bc4 100644 --- a/oneof_test.go +++ b/oneof_test.go @@ -1,16 +1,13 @@ package proto3 -import ( - "strings" - "testing" -) +import "testing" func TestOneof(t *testing.T) { proto := `oneof foo { string name = 4; SubMessage sub_message = 9 [options=none]; }` - p := NewParser(strings.NewReader(proto)) + p := newParserOn(proto) p.scanIgnoreWhitespace() // consume first token o := new(Oneof) err := o.parse(p) diff --git a/option.go b/option.go index 6aaf0c0..4dcb6c6 100644 --- a/option.go +++ b/option.go @@ -4,10 +4,11 @@ import "fmt" // Option is a protoc compiler option type Option struct { - Line int - Name string - String string - Boolean bool + Name string + Identifier string + String string + Boolean bool + PartOfFieldOrEnum bool } // Accept dispatches the call to the visitor. @@ -15,30 +16,32 @@ func (o *Option) Accept(v Visitor) { v.VisitOption(o) } +// parse reads an Option body +// ( ident | "(" fullIdent ")" ) { "." ident } "=" constant ";" func (o *Option) parse(p *Parser) error { tok, lit := p.scanIgnoreWhitespace() switch tok { case tIDENT: - o.Line = p.s.line o.Name = lit case tLEFTPAREN: tok, lit = p.scanIgnoreWhitespace() if tok != tIDENT { - return fmt.Errorf("found %q, expected identifier", lit) + return p.unexpected(lit, "identifier") } o.Name = lit tok, lit = p.scanIgnoreWhitespace() if tok != tRIGHTPAREN { - return fmt.Errorf("found %q, expected )", lit) + return p.unexpected(lit, ")") } default: - return fmt.Errorf("found %q, expected identifier or (", lit) + return p.unexpected(lit, "identifier or (") } tok, lit = p.scanIgnoreWhitespace() if tok != tEQUALS { - return fmt.Errorf("found %q, expected =", lit) + return p.unexpected(lit, "=") } tok, lit = p.scanIgnoreWhitespace() + // stringLiteral? if tok == tQUOTE { ident := p.s.scanUntil('"') if len(ident) == 0 { @@ -47,10 +50,9 @@ func (o *Option) parse(p *Parser) error { o.String = ident return nil } - if tTRUE == tok || tFALSE == tok { - o.Boolean = lit == "true" - } else { - return fmt.Errorf("found %q, expected true or false", lit) + if tIDENT != tok { + return p.unexpected(lit, "constant") } + o.Identifier = lit return nil } diff --git a/option_test.go b/option_test.go index dabf695..da0caa6 100644 --- a/option_test.go +++ b/option_test.go @@ -1,30 +1,10 @@ package proto3 -import ( - "strings" - "testing" -) +import "testing" func TestOption(t *testing.T) { - proto := `option java_package = "com.example.foo";` - p := NewParser(strings.NewReader(proto)) - p.scanIgnoreWhitespace() // consume first token - o := new(Option) - err := o.parse(p) - if err != nil { - t.Fatal(err) - } - if got, want := o.Name, "java_package"; got != want { - t.Errorf("got [%v] want [%v]", got, want) - } - if got, want := o.String, "com.example.foo"; got != want { - t.Errorf("got [%v] want [%v]", got, want) - } -} - -func TestOptionFull(t *testing.T) { proto := `option (full.java_package) = "com.example.foo";` - p := NewParser(strings.NewReader(proto)) + p := newParserOn(proto) p.scanIgnoreWhitespace() // consume first token o := new(Option) err := o.parse(p) @@ -37,4 +17,7 @@ func TestOptionFull(t *testing.T) { if got, want := o.String, "com.example.foo"; got != want { t.Errorf("got [%v] want [%v]", got, want) } + if got, want := o.PartOfFieldOrEnum, false; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } } diff --git a/parser.go b/parser.go index c461bf0..5f9091c 100644 --- a/parser.go +++ b/parser.go @@ -1,6 +1,10 @@ package proto3 -import "io" +import ( + "fmt" + "io" + "runtime" +) // Parser represents a parser. type Parser struct { @@ -10,7 +14,7 @@ type Parser struct { lit string // last read literal n int // buffer size (max=1) } - comments []*Comment + debug bool } // NewParser returns a new instance of Parser. @@ -64,3 +68,12 @@ func (p *Parser) newComment(lit string) *Comment { // Line returns the line number on which the last token was read. func (p *Parser) Line() int { return p.s.line } + +func (p *Parser) unexpected(found, expected string) error { + debug := "" + if p.debug { + _, file, line, _ := runtime.Caller(1) + debug = fmt.Sprintf(" at %s:%d", file, line) + } + return fmt.Errorf("found %q, expected %s%s", found, expected, debug) +} diff --git a/parser_test.go b/parser_test.go index b76bb31..43b7af4 100644 --- a/parser_test.go +++ b/parser_test.go @@ -11,7 +11,7 @@ func TestParseComment(t *testing.T) { /* multi* */` - p := NewParser(strings.NewReader(proto)) + p := newParserOn(proto) pr, err := p.Parse() if err != nil { t.Fatal(err) @@ -20,3 +20,9 @@ func TestParseComment(t *testing.T) { t.Errorf("got [%v] want [%v]", got, want) } } + +func newParserOn(def string) *Parser { + p := NewParser(strings.NewReader(def)) + p.debug = true + return p +} diff --git a/scanner.go b/scanner.go index bf70688..debb437 100644 --- a/scanner.go +++ b/scanner.go @@ -63,6 +63,8 @@ func (s *scanner) scan() (tok token, lit string) { return tCOMMENT, s.scanComment() case '<': return tLESS, string(ch) + case ',': + return tCOMMA, string(ch) } return tILLEGAL, string(ch) } @@ -152,10 +154,6 @@ func (s *scanner) scanIdent() (tok token, lit string) { return tOPTION, buf.String() case "enum": return tENUM, buf.String() - case "true": - return tTRUE, buf.String() - case "false": - return tFALSE, buf.String() case "weak": return tWEAK, buf.String() case "public": diff --git a/service_test.go b/service_test.go index a1b7863..5146fe9 100644 --- a/service_test.go +++ b/service_test.go @@ -1,27 +1,13 @@ package proto3 -import ( - "strings" - "testing" -) +import "testing" func TestService(t *testing.T) { - proto := `service AccountService {}` - stmt, err := NewParser(strings.NewReader(proto)).Parse() - if err != nil { - t.Fatal(err) - } - if got, want := collect(stmt).Services()[0].Name, "AccountService"; got != want { - t.Errorf("got [%v] want [%v]", got, want) - } -} - -func TestServiceWithRPCs(t *testing.T) { proto := `service AccountService { rpc CreateAccount (CreateAccount) returns (ServiceFault) {} rpc GetAccount (Int64) returns (Account) {} }` - stmt, err := NewParser(strings.NewReader(proto)).Parse() + stmt, err := newParserOn(proto).Parse() if err != nil { t.Fatal(err) } diff --git a/syntax.go b/syntax.go index e7fb987..937ac8d 100644 --- a/syntax.go +++ b/syntax.go @@ -14,7 +14,7 @@ func (s *Syntax) Accept(v Visitor) { func (s *Syntax) parse(p *Parser) error { if tok, lit := p.scanIgnoreWhitespace(); tok != tEQUALS { - return fmt.Errorf("found %q, expected EQUALS", lit) + return p.unexpected(lit, "=") } if tok, lit := p.scanIgnoreWhitespace(); tok != tQUOTE { return fmt.Errorf("found %q, expected QUOTE", lit) diff --git a/syntax_test.go b/syntax_test.go index cd1cadf..c03075b 100644 --- a/syntax_test.go +++ b/syntax_test.go @@ -2,13 +2,12 @@ package proto3 import ( "strconv" - "strings" "testing" ) func TestSyntax(t *testing.T) { proto := `syntax = "proto3";` - p := NewParser(strings.NewReader(proto)) + p := newParserOn(proto) p.scanIgnoreWhitespace() // consume first token s := new(Syntax) err := s.parse(p) @@ -27,7 +26,7 @@ func TestCommentAroundSyntax(t *testing.T) { syntax = "proto3"; // comment3 // comment4 ` - p := NewParser(strings.NewReader(proto)) + p := newParserOn(proto) r, err := p.Parse() if err != nil { t.Fatal(err) diff --git a/token.go b/token.go index eb9d692..892f981 100644 --- a/token.go +++ b/token.go @@ -11,8 +11,6 @@ const ( // Literals tIDENT - tTRUE - tFALSE // Misc characters tSEMICOLON // ; @@ -26,6 +24,7 @@ const ( tRIGHTSQUARE // ] tCOMMENT // / tLESS // < + tCOMMA // , // Keywords tSYNTAX