Don't include the trailing line break in Messages, Terms and Comments (…

…#153) Revert earlier changes which added the trailing newline to the definition of the Message, Term and Comment, as well as to the content of Comments. The trailing line break should not be part of the production, span nor the content. Make Junk subclass SyntaxNode directly, rather than Entry. Entry is now an abstract class for useful AST nodes carrying content: Messages, Terms and Comments. Remove the annotations field from Entry togather with the add_annotation method and move it directly onto Junk.
projectfluent · Jul 18, 2018 · 4bea761 · 4bea761
1 parent 1992137
commit 4bea761
Show file tree

Hide file tree

Showing 25 changed files with 109 additions and 223 deletions.
diff --git a/lib/combinators.mjs b/lib/combinators.mjs
@@ -91,7 +91,7 @@ export function maybe(parser) {
             .run(stream)
             .fold(
                 (value, tail) => new Success(value, tail),
-                (value, tail) => new Success(undefined, stream)));
+                (value, tail) => new Success(null, stream)));
 }
 
 export function append(p1, p2) {

diff --git a/spec/fluent.ebnf b/spec/fluent.ebnf
@@ -1,14 +1,14 @@
 
 /* An FTL file defines a Resource. */
-Resource            ::= (blank_line | Entry | junk_line)*
-Entry               ::= Message
-                      | Term
+Resource            ::= (Entry | blank_line | junk_line)*
+Entry               ::= (Message line_end)
+                      | (Term line_end)
                       | (ResourceComment | GroupComment | Comment)
-Message             ::= Comment? Identifier inline_space? "=" inline_space? ((Pattern Attribute*) | (Attribute+)) line_end
-Term                ::= Comment? TermIdentifier inline_space? "=" inline_space? Value Attribute* line_end
-Comment             ::= ("#" comment_line)+
-GroupComment        ::= ("##" comment_line)+
-ResourceComment     ::= ("###" comment_line)+
+Message             ::= Comment? Identifier inline_space? "=" inline_space? ((Pattern Attribute*) | (Attribute+))
+Term                ::= Comment? TermIdentifier inline_space? "=" inline_space? Value Attribute*
+Comment             ::= ("#" ("\u0020" /.*/)? line_end)+
+GroupComment        ::= ("##" ("\u0020" /.*/)? line_end)+
+ResourceComment     ::= ("###" ("\u0020" /.*/)? line_end)+
 
 /* Adjacent junk_lines should be joined into FTL.Junk during the AST
    construction. */
@@ -73,8 +73,6 @@ Function            ::= [A-Z] [A-Z_?-]*
 
 /* Tokens */
 identifier          ::= [a-zA-Z] [a-zA-Z0-9_-]*
-comment_line        ::= (line_end)
-                      | ("\u0020" /.*/ line_end)
 word                ::= (regular_char - backslash - "}" - "{" - "]" - "[")+
 
 /* Characters */

diff --git a/syntax/abstract.mjs b/syntax/abstract.mjs
@@ -82,6 +82,25 @@ export function list_into(Type) {
 
 export function into(Type) {
     switch (Type) {
+        case FTL.Comment:
+        case FTL.GroupComment:
+        case FTL.ResourceComment:
+            return content => {
+                if (content.endsWith("\n")) {
+                    if (content.endsWith("\r\n")) {
+                        var offset = -2;
+                    } else {
+                        var offset = -1;
+                    }
+                } else if (content.endsWith("\r")) {
+                    var offset = -1;
+                } else {
+                    // The comment ended with the EOF; don't trim it.
+                    return always(new Type(content));
+                }
+                // Trim the EOL from the end of the comment.
+                return always(new Type(content.slice(0, offset)));
+            };
         case FTL.Placeable:
             return expression => {
                 let invalid_expression_found =

diff --git a/syntax/ast.mjs b/syntax/ast.mjs
@@ -18,17 +18,8 @@ export class Resource extends SyntaxNode {
     }
 }
 
-export class Entry extends SyntaxNode {
-    constructor() {
-        super();
-        this.type = "Entry";
-        this.annotations = [];
-    }
-
-    addAnnotation(annot) {
-        this.annotations.push(annot);
-    }
-}
+// An abstract base class for useful elements of Resource.body.
+export class Entry extends SyntaxNode {}
 
 export class Message extends Entry {
     constructor(id, value = null, attributes = [], comment = null) {
@@ -245,12 +236,17 @@ export class Function extends Identifier {
     }
 }
 
-export class Junk extends Entry {
+export class Junk extends SyntaxNode {
     constructor(content) {
         super();
         this.type = "Junk";
+        this.annotations = [];
         this.content = content;
     }
+
+    addAnnotation(annot) {
+        this.annotations.push(annot);
+    }
 }
 
 export class Span extends BaseNode {

diff --git a/syntax/grammar.mjs b/syntax/grammar.mjs
@@ -14,16 +14,20 @@ export
 let Resource = defer(() =>
     repeat(
         either(
-            blank_line,
             Entry,
+            blank_line,
             junk_line))
     .chain(list_into(FTL.Resource)));
 
 export
 let Entry = defer(() =>
     either(
-        Message,
-        Term,
+        sequence(
+            Message,
+            line_end).map(element_at(0)),
+        sequence(
+            Term,
+            line_end).map(element_at(0)),
         either(
             ResourceComment,
             GroupComment,
@@ -42,8 +46,7 @@ let Message = defer(() =>
                 repeat(Attribute).abstract),
             sequence(
                 always(null).abstract,
-                repeat1(Attribute).abstract)),
-        line_end)
+                repeat1(Attribute).abstract)))
     .map(flatten(1))
     .map(keep_abstract)
     .chain(list_into(FTL.Message)));
@@ -56,17 +59,20 @@ let Term = defer(() =>
         string("="),
         maybe(inline_space),
         Value.abstract,
-        repeat(Attribute).abstract,
-        line_end)
+        repeat(Attribute).abstract)
     .map(keep_abstract)
     .chain(list_into(FTL.Term)));
 
 let Comment = defer(() =>
     repeat1(
         sequence(
             string("#"),
-            comment_line.abstract))
-    .map(flatten(1))
+            maybe(
+                sequence(
+                    string(" "),
+                    regex(/.*/).abstract)),
+            line_end.abstract))
+    .map(flatten(2))
     .map(keep_abstract)
     .map(join)
     .chain(into(FTL.Comment)));
@@ -75,8 +81,12 @@ let GroupComment = defer(() =>
     repeat1(
         sequence(
             string("##"),
-            comment_line.abstract))
-    .map(flatten(1))
+            maybe(
+                sequence(
+                    string(" "),
+                    regex(/.*/).abstract)),
+            line_end.abstract))
+    .map(flatten(2))
     .map(keep_abstract)
     .map(join)
     .chain(into(FTL.GroupComment)));
@@ -85,8 +95,12 @@ let ResourceComment = defer(() =>
     repeat1(
         sequence(
             string("###"),
-            comment_line.abstract))
-    .map(flatten(1))
+            maybe(
+                sequence(
+                    string(" "),
+                    regex(/.*/).abstract)),
+            line_end.abstract))
+    .map(flatten(2))
     .map(keep_abstract)
     .map(join)
     .chain(into(FTL.ResourceComment)));
@@ -378,17 +392,6 @@ let identifier =
     .map(flatten(1))
     .map(join);
 
-let comment_line = defer(() =>
-    either(
-        sequence(
-            line_end.abstract),
-        sequence(
-            string(" "),
-            regex(/.*/).abstract,
-            line_end.abstract))
-    .map(keep_abstract)
-    .map(join));
-
 let word = defer(() =>
     repeat1(
         and(