From e5da597215b29e8c3777c9a3b361208c8d48312f Mon Sep 17 00:00:00 2001 From: Ernest Micklei Date: Wed, 15 Feb 2017 13:09:20 +0100 Subject: [PATCH] proper parsing extensions, refactored parsing reserved Change-Id: Ia7ff015d8fb87ccebd02152a45533c5352509f04 --- enum_test.go | 9 +++-- extensions.go | 32 ++++++----------- extensions_test.go | 13 ++++--- formatter.go | 24 +++++++++---- range.go | 90 ++++++++++++++++++++++++++++++++++++++++++++++ range_test.go | 68 +++++++++++++++++++++++++++++++++++ reserved.go | 49 +++++-------------------- scanner.go | 16 ++++++--- scanner_test.go | 28 ++++++++++++--- 9 files changed, 241 insertions(+), 88 deletions(-) create mode 100644 range.go create mode 100644 range_test.go diff --git a/enum_test.go b/enum_test.go index 036856d..be37ff2 100644 --- a/enum_test.go +++ b/enum_test.go @@ -1,7 +1,7 @@ // Copyright (c) 2017 Ernest Micklei -// +// // MIT License -// +// // Permission is hereby granted, free of charge, to any person obtaining // a copy of this software and associated documentation files (the // "Software"), to deal in the Software without restriction, including @@ -9,10 +9,10 @@ // distribute, sublicense, and/or sell copies of the Software, and to // permit persons to whom the Software is furnished to do so, subject to // the following conditions: -// +// // The above copyright notice and this permission notice shall be // included in all copies or substantial portions of the Software. -// +// // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND @@ -59,5 +59,4 @@ enum EnumAllowingAlias { if got, want := ef3.ValueOption.Constant.Source, "hello world"; got != want { t.Errorf("got [%v] want [%v]", got, want) } - logformatted(t, pr.Elements[0]) } diff --git a/extensions.go b/extensions.go index 33b38ce..5a11e7e 100644 --- a/extensions.go +++ b/extensions.go @@ -1,7 +1,7 @@ // Copyright (c) 2017 Ernest Micklei -// +// // MIT License -// +// // Permission is hereby granted, free of charge, to any person obtaining // a copy of this software and associated documentation files (the // "Software"), to deal in the Software without restriction, including @@ -9,10 +9,10 @@ // distribute, sublicense, and/or sell copies of the Software, and to // permit persons to whom the Software is furnished to do so, subject to // the following conditions: -// +// // The above copyright notice and this permission notice shall be // included in all copies or substantial portions of the Software. -// +// // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND @@ -26,7 +26,7 @@ package proto // Extensions declare that a range of field numbers in a message are available for third-party extensions. // proto2 only type Extensions struct { - Ranges string + Ranges []Range Comment *Comment } @@ -40,22 +40,12 @@ func (e *Extensions) Accept(v Visitor) { v.VisitExtensions(e) } -// parse expects ident { messageBody +// parse expects ranges func (e *Extensions) parse(p *Parser) error { - // TODO proper range parsing - e.Ranges = p.s.scanUntil(';') - p.s.unread(';') // for reading inline comment - return nil -} - -// columns returns printable source tokens -func (e *Extensions) columns() (cols []aligned) { - cols = append(cols, - notAligned("extensions "), - leftAligned(e.Ranges), - alignedSemicolon) - if e.Comment != nil { - cols = append(cols, notAligned(" //"), notAligned(e.Comment.Message)) + list, err := parseRanges(p, e) + if err != nil { + return err } - return + e.Ranges = list + return nil } diff --git a/extensions_test.go b/extensions_test.go index 7f1314e..04cd866 100644 --- a/extensions_test.go +++ b/extensions_test.go @@ -1,7 +1,7 @@ // Copyright (c) 2017 Ernest Micklei -// +// // MIT License -// +// // Permission is hereby granted, free of charge, to any person obtaining // a copy of this software and associated documentation files (the // "Software"), to deal in the Software without restriction, including @@ -9,10 +9,10 @@ // distribute, sublicense, and/or sell copies of the Software, and to // permit persons to whom the Software is furnished to do so, subject to // the following conditions: -// +// // The above copyright notice and this permission notice shall be // included in all copies or substantial portions of the Software. -// +// // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND @@ -40,7 +40,10 @@ func TestExtensions(t *testing.T) { t.Fatal("extensions expected") } f := m.Elements[0].(*Extensions) - if got, want := f.Ranges, " 4, 20 to max"; got != want { + if got, want := len(f.Ranges), 2; got != want { + t.Fatalf("got [%d] want [%d]", got, want) + } + if got, want := f.Ranges[1].String(), "20 to max"; got != want { t.Errorf("got [%s] want [%s]", got, want) } if f.Comment == nil { diff --git a/formatter.go b/formatter.go index d9aeaf3..1e293ea 100644 --- a/formatter.go +++ b/formatter.go @@ -1,7 +1,7 @@ // Copyright (c) 2017 Ernest Micklei -// +// // MIT License -// +// // Permission is hereby granted, free of charge, to any person obtaining // a copy of this software and associated documentation files (the // "Software"), to deal in the Software without restriction, including @@ -9,10 +9,10 @@ // distribute, sublicense, and/or sell copies of the Software, and to // permit persons to whom the Software is furnished to do so, subject to // the following conditions: -// +// // The above copyright notice and this permission notice shall be // included in all copies or substantial portions of the Software. -// +// // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND @@ -155,14 +155,14 @@ func (f *Formatter) VisitReserved(r *Reserved) { if len(r.Ranges) > 0 { for i, each := range r.Ranges { if i > 0 { - io.WriteString(f.w, ",") + io.WriteString(f.w, ", ") } fmt.Fprintf(f.w, "%s", each.String()) } } else { for i, each := range r.FieldNames { if i > 0 { - io.WriteString(f.w, ",") + io.WriteString(f.w, ", ") } fmt.Fprintf(f.w, "%q", each) } @@ -197,4 +197,14 @@ func (f *Formatter) VisitGroup(g *Group) { } // VisitExtensions formats a proto2 Extensions. -func (f *Formatter) VisitExtensions(e *Extensions) {} +func (f *Formatter) VisitExtensions(e *Extensions) { + f.begin("extensions") + io.WriteString(f.w, "extensions ") + for i, each := range e.Ranges { + if i > 0 { + io.WriteString(f.w, ", ") + } + fmt.Fprintf(f.w, "%s", each.String()) + } + f.endWithComment(e.Comment) +} diff --git a/range.go b/range.go new file mode 100644 index 0000000..0f12844 --- /dev/null +++ b/range.go @@ -0,0 +1,90 @@ +// Copyright (c) 2017 Ernest Micklei +// +// MIT License +// +// Permission is hereby granted, free of charge, to any person obtaining +// a copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to +// permit persons to whom the Software is furnished to do so, subject to +// the following conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +package proto + +import ( + "fmt" + "strconv" +) + +// Range is to specify number intervals (with special end value "max") +type Range struct { + From, To int + Max bool +} + +// String return a single number if from = to. Returns to otherwise unless Max then return to max. +func (r Range) String() string { + if r.Max { + return fmt.Sprintf("%d to max", r.From) + } + if r.From == r.To { + return strconv.Itoa(r.From) + } + return fmt.Sprintf("%d to %d", r.From, r.To) +} + +// parseRanges is used to parse ranges for extensions and reserved +func parseRanges(p *Parser, n Visitee) (list []Range, err error) { + seenTo := false + for { + lit, isString := p.s.scanLiteral() + if isString { + return list, p.unexpected(lit, "integer, ", n) + } + switch lit { + case ",": + case "to": + seenTo = true + case ";": + p.s.unread(';') // allow for inline comment parsing + goto done + case "max": + if !seenTo { + return list, p.unexpected(lit, "to", n) + } + from := list[len(list)-1] + list = append(list[0:len(list)-1], Range{From: from.From, Max: true}) + default: + // must be number + i, err := strconv.Atoi(lit) + if err != nil { + return list, p.unexpected(lit, "range integer", n) + } + if seenTo { + // replace last two ranges with one + if len(list) < 1 { + p.unexpected(lit, "integer", n) + } + from := list[len(list)-1] + list = append(list[0:len(list)-1], Range{From: from.From, To: i}) + seenTo = false + } else { + list = append(list, Range{From: i, To: i}) + } + } + } +done: + return +} diff --git a/range_test.go b/range_test.go new file mode 100644 index 0000000..bc9a810 --- /dev/null +++ b/range_test.go @@ -0,0 +1,68 @@ +// Copyright (c) 2017 Ernest Micklei +// +// MIT License +// +// Permission is hereby granted, free of charge, to any person obtaining +// a copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to +// permit persons to whom the Software is furnished to do so, subject to +// the following conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +package proto + +import "testing" + +func TestParseRanges(t *testing.T) { + r := new(Reserved) + p := newParserOn(`reserved 2, 15, 9 to 11;`) + _, _ = p.scanIgnoreWhitespace() + ranges, err := parseRanges(p, r) + if err != nil { + t.Fatal(err) + } + if got, want := ranges[2].String(), "9 to 11"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } +} + +func TestParseRangesMax(t *testing.T) { + r := new(Extensions) + p := newParserOn(`extensions 3 to max;`) + _, _ = p.scanIgnoreWhitespace() + ranges, err := parseRanges(p, r) + if err != nil { + t.Fatal(err) + } + if got, want := ranges[0].String(), "3 to max"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } +} + +func TestParseRangesMultiToMax(t *testing.T) { + r := new(Extensions) + p := newParserOn(`extensions 1,2 to 5,6 to 9,10 to max;`) + _, _ = p.scanIgnoreWhitespace() + ranges, err := parseRanges(p, r) + if err != nil { + t.Fatal(err) + } + if got, want := len(ranges), 4; got != want { + t.Fatalf("got [%v] want [%v]", got, want) + } + if got, want := ranges[3].String(), "10 to max"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } +} diff --git a/reserved.go b/reserved.go index d3e6ae1..1210255 100644 --- a/reserved.go +++ b/reserved.go @@ -1,7 +1,7 @@ // Copyright (c) 2017 Ernest Micklei -// +// // MIT License -// +// // Permission is hereby granted, free of charge, to any person obtaining // a copy of this software and associated documentation files (the // "Software"), to deal in the Software without restriction, including @@ -9,10 +9,10 @@ // distribute, sublicense, and/or sell copies of the Software, and to // permit persons to whom the Software is furnished to do so, subject to // the following conditions: -// +// // The above copyright notice and this permission notice shall be // included in all copies or substantial portions of the Software. -// +// // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND @@ -23,11 +23,6 @@ package proto -import ( - "fmt" - "strconv" -) - // Reserved statements declare a range of field numbers or field names that cannot be used in a message. type Reserved struct { Ranges []Range @@ -40,26 +35,12 @@ func (r *Reserved) inlineComment(c *Comment) { r.Comment = c } -// Range is to specify number intervals -type Range struct { - From, To int -} - -// String return a single number if from = to. Returns to otherwise. -func (r Range) String() string { - if r.From == r.To { - return strconv.Itoa(r.From) - } - return fmt.Sprintf("%d to %d", r.From, r.To) -} - // Accept dispatches the call to the visitor. func (r *Reserved) Accept(v Visitor) { v.VisitReserved(r) } func (r *Reserved) parse(p *Parser) error { - seenRangeTo := false for { tok, lit := p.scanIgnoreWhitespace() if len(lit) == 0 { @@ -68,27 +49,13 @@ func (r *Reserved) parse(p *Parser) error { // first char that determined tok ch := []rune(lit)[0] if isDigit(ch) { - // use unread here because scanInteger does not look at buf + // use unread here because it could be start of ranges p.s.unread(ch) - i, err := p.s.scanInteger() + list, err := parseRanges(p, r) if err != nil { - return p.unexpected(lit, "reserved integer", r) - } - if seenRangeTo { - // replace last two ranges with one - if len(r.Ranges) < 1 { - p.unexpected(lit, "reserved integer", r) - } - from := r.Ranges[len(r.Ranges)-1] - r.Ranges = append(r.Ranges[0:len(r.Ranges)-1], Range{From: from.From, To: i}) - seenRangeTo = false - } else { - r.Ranges = append(r.Ranges, Range{From: i, To: i}) + return err } - continue - } - if tIDENT == tok && "to" == lit { - seenRangeTo = true + r.Ranges = list continue } if tQUOTE == tok || tSINGLEQUOTE == tok { diff --git a/scanner.go b/scanner.go index 25d1090..9cf2c20 100644 --- a/scanner.go +++ b/scanner.go @@ -1,7 +1,7 @@ // Copyright (c) 2017 Ernest Micklei -// +// // MIT License -// +// // Permission is hereby granted, free of charge, to any person obtaining // a copy of this software and associated documentation files (the // "Software"), to deal in the Software without restriction, including @@ -9,10 +9,10 @@ // distribute, sublicense, and/or sell copies of the Software, and to // permit persons to whom the Software is furnished to do so, subject to // the following conditions: -// +// // The above copyright notice and this permission notice shall be // included in all copies or substantial portions of the Software. -// +// // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND @@ -140,6 +140,9 @@ func (s *scanner) scanLiteral() (string, bool) { if '"' == ch { return s.scanUntil('"'), true } + if isLiteralTerminator(ch) { + return string(ch), false + } // Create a buffer and read the current character into it. var buf bytes.Buffer buf.WriteRune(ch) @@ -149,7 +152,7 @@ func (s *scanner) scanLiteral() (string, bool) { for { if ch := s.read(); ch == eof { break - } else if isWhitespace(ch) || strings.ContainsRune("[]();,", ch) { // TODO const? + } else if isWhitespace(ch) || isLiteralTerminator(ch) { s.unread(ch) break } else { @@ -267,6 +270,9 @@ func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && c // isDigit returns true if the rune is a digit. func isDigit(ch rune) bool { return (ch >= '0' && ch <= '9') } +// isLiteralTerminator returns true if the rune cannot be part of a literal. +func isLiteralTerminator(ch rune) bool { return strings.ContainsRune("[]();,", ch) } + // eof represents a marker rune for the end of the reader. var eof = rune(0) diff --git a/scanner_test.go b/scanner_test.go index 0f095a5..8de6a50 100644 --- a/scanner_test.go +++ b/scanner_test.go @@ -1,7 +1,7 @@ // Copyright (c) 2017 Ernest Micklei -// +// // MIT License -// +// // Permission is hereby granted, free of charge, to any person obtaining // a copy of this software and associated documentation files (the // "Software"), to deal in the Software without restriction, including @@ -9,10 +9,10 @@ // distribute, sublicense, and/or sell copies of the Software, and to // permit persons to whom the Software is furnished to do so, subject to // the following conditions: -// +// // The above copyright notice and this permission notice shall be // included in all copies or substantial portions of the Software. -// +// // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND @@ -113,3 +113,23 @@ func TestScanLiteral_float(t *testing.T) { t.Errorf("got [%v] want [%v]", got, want) } } + +// TODO table driven +func TestScanLiteral_statementEnd(t *testing.T) { + r := strings.NewReader(`;`) + s := newScanner(r) + v, _ := s.scanLiteral() + if got, want := v, ";"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } +} + +// TODO table driven +func TestScanLiteral_comma(t *testing.T) { + r := strings.NewReader(`,0`) + s := newScanner(r) + v, _ := s.scanLiteral() + if got, want := v, ","; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } +}