todo: work on context aware parsing

aligned.go

@@ -12,6 +12,7 @@ var (
 	alignedShortEquals = leftAligned("=")
 	alignedSpace       = leftAligned(" ")
 	alignedComma       = leftAligned(",")
+	alignedEmpty       = leftAligned("")
 )
 
 func leftAligned(src string) aligned  { return aligned{src, true} }

cmd/protofmt/main.go

@@ -23,4 +23,5 @@ func main() {
 		log.Fatalln("protofmt failed", err)
 	}
 	proto.NewFormatter(os.Stdout, "  ").Format(def)
+	//spew.Dump(def)
 }

cmd/protofmt/unformatted.proto

@@ -61,4 +61,11 @@ service SearchService { // comment
 enum Enum {}
 service Service {}
 message Message {}
-oneof Oneof {}
+
+// context aware
+enum enum {
+    enum = 0;
+}
+message message {
+    message message = 1;
+}

enum.go

@@ -1,9 +1,6 @@
 package proto
 
-import (
-	"fmt"
-	"strconv"
-)
+import "strconv"
 
 // Enum definition consists of a name and an enum body.
 type Enum struct {
@@ -41,16 +38,16 @@ func (f EnumField) columns() (cols []aligned) {
 func (f *EnumField) parse(p *Parser) error {
 	tok, lit := p.scanIgnoreWhitespace()
 	if tok != tIDENT {
-		return fmt.Errorf("found %q, expected identifier", lit)
+		return p.unexpected(lit, "enum field identifier", f)
 	}
 	f.Name = lit
 	tok, lit = p.scanIgnoreWhitespace()
 	if tok != tEQUALS {
-		return p.unexpected(lit, "=")
+		return p.unexpected(lit, "enum field =", f)
 	}
 	i, err := p.s.scanInteger()
 	if err != nil {
-		return fmt.Errorf("found %q, expected integer", err)
+		return p.unexpected(lit, "enum field integer", f)
 	}
 	f.Integer = i
 	tok, lit = p.scanIgnoreWhitespace()
@@ -64,21 +61,21 @@ func (f *EnumField) parse(p *Parser) error {
 		f.ValueOption = o
 		tok, lit = p.scanIgnoreWhitespace()
 		if tok != tRIGHTSQUARE {
-			return fmt.Errorf("found %q, expected ]", lit)
+			return p.unexpected(lit, "option closing ]", f)
 		}
 	}
 	return nil
 }
 
 func (e *Enum) parse(p *Parser) error {
-	tok, lit := p.scanIgnoreWhitespace()
+	tok, lit := p.s.scanIdent()
 	if tok != tIDENT {
-		return fmt.Errorf("found %q, expected identifier", lit)
+		return p.unexpected(lit, "enum identifier", e)
 	}
 	e.Name = lit
 	tok, lit = p.scanIgnoreWhitespace()
 	if tok != tLEFTCURLY {
-		return fmt.Errorf("found %q, expected {", lit)
+		return p.unexpected(lit, "enum opening {", e)
 	}
 	for {
 		tok, lit = p.scanIgnoreWhitespace()
@@ -107,7 +104,7 @@ func (e *Enum) parse(p *Parser) error {
 	}
 done:
 	if tok != tRIGHTCURLY {
-		return fmt.Errorf("found %q, expected }", lit)
+		return p.unexpected(lit, "enum closing }", e)
 	}
 	return nil
 }

field.go

@@ -73,16 +73,16 @@ done:
 func parseFieldAfterType(f *Field, p *Parser) error {
 	tok, lit := p.scanIgnoreWhitespace()
 	if tok != tIDENT {
-		return p.unexpected(lit, "identifier")
+		return p.unexpected(lit, "field identifier", f)
 	}
 	f.Name = lit
 	tok, lit = p.scanIgnoreWhitespace()
 	if tok != tEQUALS {
-		return p.unexpected(lit, "=")
+		return p.unexpected(lit, "field =", f)
 	}
 	i, err := p.s.scanInteger()
 	if err != nil {
-		return p.unexpected(lit, "sequence number")
+		return p.unexpected(lit, "field sequence number", f)
 	}
 	f.Sequence = i
 	// see if there are options
@@ -106,7 +106,7 @@ func parseFieldAfterType(f *Field, p *Parser) error {
 			break
 		}
 		if tCOMMA != tok {
-			return p.unexpected(lit, ",")
+			return p.unexpected(lit, "option ,", o)
 		}
 	}
 	return nil
@@ -132,25 +132,25 @@ func (f *MapField) Accept(v Visitor) {
 func (f *MapField) parse(p *Parser) error {
 	tok, lit := p.scanIgnoreWhitespace()
 	if tLESS != tok {
-		return p.unexpected(lit, "<")
+		return p.unexpected(lit, "map keyType <", f)
 	}
 	tok, lit = p.scanIgnoreWhitespace()
 	if tIDENT != tok {
-		return p.unexpected(lit, "identifier")
+		return p.unexpected(lit, "map identifier", f)
 	}
 	f.KeyType = lit
 	tok, lit = p.scanIgnoreWhitespace()
 	if tCOMMA != tok {
-		return p.unexpected(lit, ",")
+		return p.unexpected(lit, "map type separator ,", f)
 	}
 	tok, lit = p.scanIgnoreWhitespace()
 	if tIDENT != tok {
-		return p.unexpected(lit, "identifier")
+		return p.unexpected(lit, "map valueType identifier", f)
 	}
 	f.Type = lit
 	tok, lit = p.scanIgnoreWhitespace()
 	if tGREATER != tok {
-		return p.unexpected(lit, ">")
+		return p.unexpected(lit, "mak valueType >", f)
 	}
 	return parseFieldAfterType(f.Field, p)
 }

formatter_utils.go

@@ -90,22 +90,30 @@ func (f *Formatter) printAsGroups(list []Visitee) {
 	group := []columnsPrintable{}
 	lastGroupName := ""
 	for i := 0; i < len(list); i++ {
-		groupName := nameOfVisitee(list[i])
-		printable, isColumnsPrintable := list[i].(columnsPrintable)
+		each := list[i]
+		groupName := nameOfVisitee(each)
+		printable, isColumnsPrintable := each.(columnsPrintable)
 		if isColumnsPrintable {
-			if lastGroupName == groupName {
-				// collect in group
-				group = append(group, printable)
-			} else {
+			if lastGroupName != groupName {
 				// print current group
+				if len(group) > 0 {
+					f.printListOfColumns(group)
+					lastGroupName = groupName
+					// begin new group
+					group = []columnsPrintable{}
+				}
+			}
+			group = append(group, printable)
+		} else {
+			// not printable in group
+			// print current group
+			if len(group) > 0 {
 				f.printListOfColumns(group)
 				lastGroupName = groupName
 				// begin new group
-				group = []columnsPrintable{printable}
+				group = []columnsPrintable{}
 			}
-		} else {
-			// not printable in group
-			list[i].Accept(f)
+			each.Accept(f)
 		}
 	}
 	// print last group

import.go

@@ -22,8 +22,10 @@ func (i *Import) parse(p *Parser) error {
 		return i.parse(p)
 	case tQUOTE:
 		i.Filename = p.s.scanUntil('"')
+	case tSINGLEQUOTE:
+		i.Filename = p.s.scanUntil('\'')
 	default:
-		return p.unexpected(lit, "weak|public|quoted identifier")
+		return p.unexpected(lit, "import classifier weak|public|quoted", i)
 	}
 	return nil
 }

message.go

@@ -14,12 +14,12 @@ func (m *Message) Accept(v Visitor) {
 func (m *Message) parse(p *Parser) error {
 	tok, lit := p.scanIgnoreWhitespace()
 	if tok != tIDENT {
-		return p.unexpected(lit, "identifier")
+		return p.unexpected(lit, "message identifier", m)
 	}
 	m.Name = lit
 	tok, lit = p.scanIgnoreWhitespace()
 	if tok != tLEFTCURLY {
-		return p.unexpected(lit, "{")
+		return p.unexpected(lit, "message opening {", m)
 	}
 	for {
 		tok, lit = p.scanIgnoreWhitespace()
@@ -78,7 +78,7 @@ func (m *Message) parse(p *Parser) error {
 	}
 done:
 	if tok != tRIGHTCURLY {
-		return p.unexpected(lit, "}")
+		return p.unexpected(lit, "message closing }", m)
 	}
 	return nil
 }

oneof.go

@@ -60,17 +60,17 @@ func (o *OneOfField) Accept(v Visitor) {
 func (o *OneOfField) parse(p *Parser) error {
 	tok, lit := p.scanIgnoreWhitespace()
 	if tok != tIDENT {
-		return p.unexpected(lit, "identifier")
+		return p.unexpected(lit, "oneof field identifier", o)
 	}
 	o.Name = lit
 	tok, lit = p.scanIgnoreWhitespace()
 	if tok != tEQUALS {
-		return p.unexpected(lit, "=")
+		return p.unexpected(lit, "oneof field =", o)
 	}
 	_, lit = p.scanIgnoreWhitespace()
 	i, err := strconv.Atoi(lit)
 	if err != nil {
-		return p.unexpected(lit, "sequence number")
+		return p.unexpected(lit, "oneof sequence number", o)
 	}
 	o.Sequence = i
 	tok, lit = p.scanIgnoreWhitespace()

option.go

@@ -44,28 +44,28 @@ func (o *Option) parse(p *Parser) error {
 	case tLEFTPAREN:
 		tok, lit = p.scanIgnoreWhitespace()
 		if tok != tIDENT {
-			return p.unexpected(lit, "identifier")
+			return p.unexpected(lit, "option identifier", o)
 		}
 		o.Name = lit
 		tok, lit = p.scanIgnoreWhitespace()
 		if tok != tRIGHTPAREN {
-			return p.unexpected(lit, ")")
+			return p.unexpected(lit, "option closing )", o)
 		}
 	default:
-		return p.unexpected(lit, "identifier or (")
+		return p.unexpected(lit, "option identifier or (", o)
 	}
 	tok, lit = p.scanIgnoreWhitespace()
 	if tok == tDOT {
 		// extend identifier
 		tok, lit = p.scanIgnoreWhitespace()
 		if tok != tIDENT {
-			return p.unexpected(lit, "postfix identifier")
+			return p.unexpected(lit, "option postfix identifier", o)
 		}
 		o.Name = fmt.Sprintf("%s.%s", o.Name, lit)
 		tok, lit = p.scanIgnoreWhitespace()
 	}
 	if tok != tEQUALS {
-		return p.unexpected(lit, "=")
+		return p.unexpected(lit, "option constant =", o)
 	}
 	l := new(Literal)
 	if err := l.parse(p); err != nil {
@@ -95,7 +95,7 @@ func (l *Literal) parse(p *Parser) error {
 	if tok == tQUOTE {
 		ident := p.s.scanUntil('"')
 		if len(ident) == 0 {
-			return p.unexpected(lit, "quoted string")
+			return p.unexpected(lit, "literal quoted string", l)
 		}
 		l.Source, l.IsString = ident, true
 		return nil
@@ -104,7 +104,7 @@ func (l *Literal) parse(p *Parser) error {
 	if tok == tSINGLEQUOTE {
 		ident := p.s.scanUntil('\'')
 		if len(ident) == 0 {
-			return p.unexpected(lit, "single quoted string")
+			return p.unexpected(lit, "literal single quoted string", l)
 		}
 		l.Source, l.IsString = ident, true
 		return nil

parser.go

@@ -55,6 +55,12 @@ func (p *Parser) scanIgnoreWhitespace() (tok token, lit string) {
 	return
 }
 
+// scanIdent scans all whitespaces and scans the next non-whitespace identifier (not a keyword).
+func (p *Parser) scanIdent() (tok token, lit string) {
+	p.s.skipWhitespace()
+	return p.s.scanIdent()
+}
+
 // unscan pushes the previously read token back onto the buffer.
 func (p *Parser) unscan() { p.buf.n = 1 }
 
@@ -63,11 +69,11 @@ func (p *Parser) newComment(lit string) *Comment {
 	return &Comment{Message: lit}
 }
 
-func (p *Parser) unexpected(found, expected string) error {
+func (p *Parser) unexpected(found, expected string, obj interface{}) error {
 	debug := ""
 	if p.debug {
 		_, file, line, _ := runtime.Caller(1)
-		debug = fmt.Sprintf(" at %s:%d", file, line)
+		debug = fmt.Sprintf(" at %s:%d (with %#v)", file, line, obj)
 	}
 	return fmt.Errorf("found %q on line %d, expected %s%s", found, p.s.line, expected, debug)
 }
@@ -79,16 +85,16 @@ func (p *Parser) scanStringLiteral() (string, error) {
 	if tok == tQUOTE {
 		s := p.s.scanUntil('"')
 		if len(s) == 0 {
-			return "", p.unexpected(lit, "quoted string")
+			return "", p.unexpected(lit, "quoted string", nil)
 		}
 		return s, nil
 	}
 	if tok == tSINGLEQUOTE {
 		s := p.s.scanUntil('\'')
 		if len(s) == 0 {
-			return "", p.unexpected(lit, "single quoted string")
+			return "", p.unexpected(lit, "single quoted string", nil)
 		}
 		return s, nil
 	}
-	return "", p.unexpected(lit, "single or double quoted string")
+	return "", p.unexpected(lit, "single or double quoted string", nil)
 }

parser_test.go

@@ -26,3 +26,14 @@ func newParserOn(def string) *Parser {
 	p.debug = true
 	return p
 }
+
+func TestScanIdent(t *testing.T) {
+	p := NewParser(strings.NewReader(" message "))
+	tok, lit := p.scanIdent()
+	if got, want := tok, tIDENT; got != want {
+		t.Errorf("got [%v] want [%v]", got, want)
+	}
+	if got, want := lit, "message"; got != want {
+		t.Errorf("got [%v] want [%v]", got, want)
+	}
+}

proto.go

@@ -1,9 +1,6 @@
 package proto
 
-import (
-	"log"
-	"strings"
-)
+import "strings"
 
 // Proto represents a .proto definition
 type Proto struct {
@@ -76,11 +73,10 @@ func (proto *Proto) parse(p *Parser) error {
 			}
 			proto.Elements = append(proto.Elements, msg)
 		case tSEMICOLON:
-		default:
-			if p.debug {
-				log.Println("unhandled (1=EOF)", lit, tok)
-			}
+		case tEOF:
 			goto done
+		default:
+			return p.unexpected(lit, "comment|option|import|syntax|enum|service|package|message", p)
 		}
 	}
 done: