From 0261da89e6fc989d59825977575f590d36ac8670 Mon Sep 17 00:00:00 2001 From: Ernest Micklei Date: Sat, 28 Jan 2017 00:09:30 +0100 Subject: [PATCH] enum, import, option. broke some tests Change-Id: Ifaf1e6157255a4f358c6c767a4000e8ddedec5c8 --- enum.go | 90 ++++++++++++++++++++++++++ enum_test.go | 41 ++++++++++++ import.go | 27 ++++++++ import_test.go | 23 +++++++ message.go | 1 + option.go | 34 ++++++---- option_test.go | 40 ++++++++++++ parser.go | 29 +-------- parser_test.go | 32 --------- proto.go | 16 ++++- example_proto_test.go => proto_test.go | 0 scanner.go | 35 +++++++--- scanner_test.go | 2 +- service.go | 1 + token.go | 2 + 15 files changed, 291 insertions(+), 82 deletions(-) create mode 100644 enum.go create mode 100644 enum_test.go create mode 100644 import.go create mode 100644 import_test.go create mode 100644 option_test.go rename example_proto_test.go => proto_test.go (100%) diff --git a/enum.go b/enum.go new file mode 100644 index 0000000..a611981 --- /dev/null +++ b/enum.go @@ -0,0 +1,90 @@ +package proto3parser + +import "fmt" + +type Enum struct { + Line int + Name string + Options []*Option + EnumFields []*EnumField +} + +type EnumField struct { + Name string + Constant string + ValueOption *Option +} + +func (f *EnumField) parse(p *Parser) error { + tok, lit := p.scanIgnoreWhitespace() + if tok != IDENT { + return fmt.Errorf("found %q, expected identifier", lit) + } + tok, lit = p.scanIgnoreWhitespace() + if tok != EQUALS { + return fmt.Errorf("found %q, expected =", lit) + } + ns := p.s.scanIntegerString() + if len(ns) != 0 { + f.Constant = ns + } else { + tok, lit = p.scanIgnoreWhitespace() + if tok != IDENT { + return fmt.Errorf("found %q, expected string", lit) + } + } + tok, lit = p.scanIgnoreWhitespace() + if tok == LEFTSQUARE { + o := new(Option) + err := o.parse(p) + if err != nil { + return err + } + f.ValueOption = o + tok, lit = p.scanIgnoreWhitespace() + if tok != RIGHTSQUARE { + return fmt.Errorf("found %q, expected ]", lit) + } + } + return nil +} + +func (e *Enum) parse(p *Parser) error { + tok, lit := p.scanIgnoreWhitespace() + if tok != IDENT { + return fmt.Errorf("found %q, expected identifier", lit) + } + e.Name = lit + tok, lit = p.scanIgnoreWhitespace() + if tok != LEFTCURLY { + return fmt.Errorf("found %q, expected {", lit) + } + for { + tok, lit = p.scanIgnoreWhitespace() + switch tok { + case RIGHTCURLY: + goto done + case SEMICOLON: + case OPTION: + v := new(Option) + err := v.parse(p) + if err != nil { + return err + } + e.Options = append(e.Options, v) + default: + p.unscan() + f := new(EnumField) + err := f.parse(p) + if err != nil { + return err + } + e.EnumFields = append(e.EnumFields, f) + } + } +done: + if tok != RIGHTCURLY { + return fmt.Errorf("found %q, expected }", lit) + } + return nil +} diff --git a/enum_test.go b/enum_test.go new file mode 100644 index 0000000..0b335fa --- /dev/null +++ b/enum_test.go @@ -0,0 +1,41 @@ +package proto3parser + +import ( + "strings" + "testing" +) + +func TestEnum(t *testing.T) { + proto := `enum EnumAllowingAlias {}` + p := NewParser(strings.NewReader(proto)) + _, tok := p.scanIgnoreWhitespace() // consume first token + if got, want := tok, "enum"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } + e := new(Enum) + err := e.parse(p) + if err != nil { + t.Fatal(err) + } + if got, want := e.Name, "EnumAllowingAlias"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } +} + +func TestEnumWithBody(t *testing.T) { + proto := ` +enum EnumAllowingAlias { + option allow_alias = true; + UNKNOWN = 0; + STARTED = 1; + RUNNING = 2 [(custom_option) = "hello world"]; +}` + p := NewParser(strings.NewReader(proto)) + pr, err := p.Parse() + if err != nil { + t.Fatal(err) + } + if got, want := len(pr.Enums), 1; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } +} diff --git a/import.go b/import.go new file mode 100644 index 0000000..87737ac --- /dev/null +++ b/import.go @@ -0,0 +1,27 @@ +package proto3parser + +import ( + "fmt" + "strings" +) + +type Import struct { + Line int + Filename string + Kind string // weak, public +} + +func (i *Import) parse(p *Parser) error { + tok, lit := p.scanIgnoreWhitespace() + if tok != IDENT || !strings.Contains("weak public", lit) { + return fmt.Errorf("found %q, expected kind (weak|public)", lit) + } + i.Line = p.s.line + i.Kind = lit + name := p.s.scanUntil('\n') + if len(name) == 0 { + return fmt.Errorf("unexpected end of quoted string") + } + i.Filename = name + return nil +} diff --git a/import_test.go b/import_test.go new file mode 100644 index 0000000..f4cf689 --- /dev/null +++ b/import_test.go @@ -0,0 +1,23 @@ +package proto3parser + +import ( + "strings" + "testing" +) + +func TestParseImport(t *testing.T) { + proto := `import public "other.proto";` + p := NewParser(strings.NewReader(proto)) + p.scanIgnoreWhitespace() // consume first token + i := new(Import) + err := i.parse(p) + if err != nil { + t.Fatal(err) + } + if got, want := i.Filename, "other.proto"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } + if got, want := i.Kind, "public"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } +} diff --git a/message.go b/message.go index 5dc9f15..526a5c5 100644 --- a/message.go +++ b/message.go @@ -6,6 +6,7 @@ import ( ) type Message struct { + Line int Name string Fields []*Field } diff --git a/option.go b/option.go index 65ef472..e8ede42 100644 --- a/option.go +++ b/option.go @@ -3,12 +3,12 @@ package proto3parser import "fmt" type Option struct { - Name string - Constant string + Name string + String string + Boolean bool } -func ParseOption(p *Parser) (*Option, error) { - o := new(Option) +func (o *Option) parse(p *Parser) error { tok, lit := p.scanIgnoreWhitespace() switch tok { case IDENT: @@ -16,24 +16,32 @@ func ParseOption(p *Parser) (*Option, error) { case LEFTPAREN: tok, lit = p.scanIgnoreWhitespace() if tok != IDENT { - return nil, fmt.Errorf("found %q, expected identifier", lit) + return fmt.Errorf("found %q, expected identifier", lit) } o.Name = lit tok, lit = p.scanIgnoreWhitespace() if tok != RIGHTPAREN { - return nil, fmt.Errorf("found %q, expected )", lit) + return fmt.Errorf("found %q, expected )", lit) } default: - return nil, fmt.Errorf("found %q, expected identifier or (", lit) + return fmt.Errorf("found %q, expected identifier or (", lit) } tok, lit = p.scanIgnoreWhitespace() if tok != EQUALS { - return nil, fmt.Errorf("found %q, expected =", lit) + return fmt.Errorf("found %q, expected =", lit) } - ident, err := p.scanQuotedIdent() - if err != nil { - return nil, err + tok, lit = p.scanIgnoreWhitespace() + if tok == QUOTE { + p.unscan() + ident := p.s.scanUntil('\n') + if len(ident) == 0 { + return fmt.Errorf("unexpected end of quoted string") // TODO create constant for this + } + o.String = ident + return nil + } + if TRUE == tok || FALSE == tok { + o.Boolean = lit == "true" } - o.Constant = ident - return o, nil + return nil } diff --git a/option_test.go b/option_test.go new file mode 100644 index 0000000..2fbb1ae --- /dev/null +++ b/option_test.go @@ -0,0 +1,40 @@ +package proto3parser + +import ( + "strings" + "testing" +) + +func TestOption(t *testing.T) { + proto := `option java_package = "com.example.foo";` + p := NewParser(strings.NewReader(proto)) + p.scanIgnoreWhitespace() // consume first token + o := new(Option) + err := o.parse(p) + if err != nil { + t.Fatal(err) + } + if got, want := o.Name, "java_package"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } + if got, want := o.String, "com.example.foo"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } +} + +func TestOptionFull(t *testing.T) { + proto := `option (full.java_package) = "com.example.foo";` + p := NewParser(strings.NewReader(proto)) + p.scanIgnoreWhitespace() // consume first token + o := new(Option) + err := o.parse(p) + if err != nil { + t.Fatal(err) + } + if got, want := o.Name, "full.java_package"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } + if got, want := o.String, "com.example.foo"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } +} diff --git a/parser.go b/parser.go index 9cb14b3..211005b 100644 --- a/parser.go +++ b/parser.go @@ -1,9 +1,6 @@ package proto3parser -import ( - "fmt" - "io" -) +import "io" // Parser represents a parser. type Parser struct { @@ -36,7 +33,7 @@ func (p *Parser) scan() (tok token, lit string) { } // Otherwise read the next token from the scanner. - tok, lit = p.s.Scan() + tok, lit = p.s.scan() // Save it to the buffer in case we unscan later. p.buf.tok, p.buf.lit = tok, lit @@ -55,25 +52,3 @@ func (p *Parser) scanIgnoreWhitespace() (tok token, lit string) { // unscan pushes the previously read token back onto the buffer. func (p *Parser) unscan() { p.buf.n = 1 } - -// scanQuotedIdent returns the identifier between 2 quotes. -func (p *Parser) scanQuotedIdent() (string, error) { - tok, lit := p.scanIgnoreWhitespace() - if tok != QUOTE { - return "", fmt.Errorf("found %q, expected \"", lit) - } - tok, lit = p.scanIgnoreWhitespace() - if tok != IDENT { - return "", fmt.Errorf("found %q, expected identifier", lit) - } - ident := lit - tok, lit = p.scanIgnoreWhitespace() - if tok != QUOTE { - return "", fmt.Errorf("found %q, expected \"", lit) - } - return ident, nil -} - -func (p *Parser) scanUntilLineEnd() string { - return p.s.scanUntilLineEnd() -} diff --git a/parser_test.go b/parser_test.go index 747f709..101b4ee 100644 --- a/parser_test.go +++ b/parser_test.go @@ -44,38 +44,6 @@ func TestServiceWithRPCs(t *testing.T) { } } -func TestOption(t *testing.T) { - proto := `option java_package = "com.example.foo";` - p := NewParser(strings.NewReader(proto)) - p.scanIgnoreWhitespace() // consume first token - o, err := ParseOption(p) - if err != nil { - t.Fatal(err) - } - if got, want := o.Name, "java_package"; got != want { - t.Errorf("got [%v] want [%v]", got, want) - } - if got, want := o.Constant, "com.example.foo"; got != want { - t.Errorf("got [%v] want [%v]", got, want) - } -} - -func TestOptionFull(t *testing.T) { - proto := `option (full.java_package) = "com.example.foo";` - p := NewParser(strings.NewReader(proto)) - p.scanIgnoreWhitespace() // consume first token - o, err := ParseOption(p) - if err != nil { - t.Fatal(err) - } - if got, want := o.Name, "full.java_package"; got != want { - t.Errorf("got [%v] want [%v]", got, want) - } - if got, want := o.Constant, "com.example.foo"; got != want { - t.Errorf("got [%v] want [%v]", got, want) - } -} - func TestCommentAroundSyntax(t *testing.T) { proto := ` // comment1 diff --git a/proto.go b/proto.go index ffa0762..60f6ddb 100644 --- a/proto.go +++ b/proto.go @@ -4,6 +4,8 @@ import "fmt" type Proto struct { Syntax string + Imports []*Import + Enums []*Enum Services []*Service Messages []*Message Comments []*Comment @@ -40,7 +42,7 @@ func parseProto(proto *Proto, p *Parser) error { switch tok { case COMMENT: proto.Comments = append(proto.Comments, &Comment{ - Line: p.s.Line() - 1, // line number before EOL was seen + Line: p.s.line - 1, // line number before EOL was seen Message: lit, }) case SYNTAX: @@ -49,6 +51,18 @@ func parseProto(proto *Proto, p *Parser) error { } else { proto.Syntax = syntax } + case IMPORT: + im := new(Import) + if err := im.parse(p); err != nil { + return err + } + proto.Imports = append(proto.Imports, im) + case ENUM: + enum := new(Enum) + if err := enum.parse(p); err != nil { + return err + } + proto.Enums = append(proto.Enums, enum) case SERVICE: if service, err := parseService(p); err != nil { return err diff --git a/example_proto_test.go b/proto_test.go similarity index 100% rename from example_proto_test.go rename to proto_test.go diff --git a/scanner.go b/scanner.go index 96cbd80..8a28367 100644 --- a/scanner.go +++ b/scanner.go @@ -20,11 +20,8 @@ func newScanner(r io.Reader) *scanner { return &scanner{r: bufio.NewReader(r)} } -// Line returns the current line being scanned -func (s *scanner) Line() int { return s.line } - -// Scan returns the next token and literal value. -func (s *scanner) Scan() (tok token, lit string) { +// scan returns the next token and literal value. +func (s *scanner) scan() (tok token, lit string) { // Read the next rune. ch := s.read() @@ -40,7 +37,7 @@ func (s *scanner) Scan() (tok token, lit string) { return s.scanIdent() } else if ch == '/' { if ch = s.read(); ch == '/' { - return COMMENT, s.scanUntilLineEnd() + return COMMENT, s.scanUntil('\n') } s.unread() s.unread() @@ -95,6 +92,26 @@ func (s *scanner) scanWhitespace() (tok token, lit string) { return WS, buf.String() } +func (s *scanner) scanIntegerString() string { + // Create a buffer and read the current character into it. + var buf bytes.Buffer + buf.WriteRune(s.read()) + + // Read every subsequent digit character into the buffer. + // Non-digiti characters and EOF will cause the loop to exit. + for { + if ch := s.read(); ch == eof { + break + } else if !isDigit(ch) { + s.unread() + break + } else { + _, _ = buf.WriteRune(ch) + } + } + return buf.String() +} + // scanIdent consumes the current rune and all contiguous ident runes. func (s *scanner) scanIdent() (tok token, lit string) { // Create a buffer and read the current character into it. @@ -134,6 +151,8 @@ func (s *scanner) scanIdent() (tok token, lit string) { return REPEATED, buf.String() case "OPTION": return OPTION, buf.String() + case "ENUM": + return ENUM, buf.String() } // Otherwise return as a regular identifier. @@ -168,8 +187,8 @@ func isDigit(ch rune) bool { return (ch >= '0' && ch <= '9') } // eof represents a marker rune for the end of the reader. var eof = rune(0) -// scanUntilLineEnd return the string up to (not including) a line end or EOF. -func (s *scanner) scanUntilLineEnd() string { +// scanUntil returns the string up to (not including) the rune or EOF. +func (s *scanner) scanUntil(ch rune) string { // Create a buffer and read the current character into it. var buf bytes.Buffer buf.WriteRune(s.read()) diff --git a/scanner_test.go b/scanner_test.go index a9aeb46..0b97ada 100644 --- a/scanner_test.go +++ b/scanner_test.go @@ -9,7 +9,7 @@ func TestScanUntilLineEnd(t *testing.T) { r := strings.NewReader(`hello world`) s := newScanner(r) - v := s.scanUntilLineEnd() + v := s.scanUntil('\n') if got, want := v, "hello"; got != want { t.Errorf("got [%v] want [%v]", got, want) } diff --git a/service.go b/service.go index 2c152f1..a7c35c6 100644 --- a/service.go +++ b/service.go @@ -6,6 +6,7 @@ import ( ) type Service struct { + Line int Name string RPCalls []*rpcall } diff --git a/token.go b/token.go index e68fe5f..4e15587 100644 --- a/token.go +++ b/token.go @@ -11,6 +11,8 @@ const ( // Literals IDENT // main + TRUE + FALSE // Misc characters SEMICOLON // ;