Fix CRLF newlines being consumed by the parser #35

The issue is caused by newline special case CRLF being ignored by parser, when the line is empty.
Ygg01 · Oct 23, 2023 · 084f6dd · 084f6dd
1 parent e575574
commit 084f6dd
Showing 1 changed file with 24 additions and 10 deletions.
diff --git a/Linguini.Syntax/Parser/LinguiniParser.cs b/Linguini.Syntax/Parser/LinguiniParser.cs
@@ -20,6 +20,7 @@ namespace Linguini.Syntax.Parser
     public class LinguiniParser
     {
         private readonly ZeroCopyReader _reader;
+        private const string CR = "\n";
 
         /// <summary>
         /// Set input to <c>string</c>
@@ -526,6 +527,16 @@ private bool TryGetPattern(out Pattern? pattern, out ParseError? error)
                             ));
                         }
                     }
+                    // In case an empty newline is emitted, we create an artificial token to represent LINEFEED (`\n`)
+                    else if (text.Start == text.End && text.TerminationReason == TextElementTermination.CRLF)
+                    {
+                        elements.Add(new TextElementPlaceholder(
+                            0,
+                            0,
+                            indent,
+                            textElementRole,
+                            true));
+                    }
 
                     textElementRole = text.TerminationReason switch
                     {
@@ -560,29 +571,32 @@ private bool TryGetPattern(out Pattern? pattern, out ParseError? error)
                             {
                                 if (commonIndent == null)
                                 {
-                                    start = start + indent;
+                                    start += indent;
                                 }
                                 else
                                 {
-                                    start = start + Math.Min(indent, commonIndent.Value);
+                                    start += Math.Min(indent, commonIndent.Value);
                                 }
                             }
 
                             ReadOnlyMemory<char> value;
-                            if (textLiteral.MissingEol)
+                            if (textLiteral.MissingEol && textLiteral.Start == textLiteral.End)
+                            {
+                                value = CR.AsMemory();
+                            }
+                            else if (textLiteral.MissingEol && textLiteral.Start != textLiteral.End)
                             {
-                                var str = _reader.ReadSlice(start, end) + "\n";
+                                var str = _reader.ReadSlice(start, end) + CR;
                                 value = str.AsMemory();
                             }
                             else
                             {
                                 value = _reader.ReadSlice(start, end);
                             }
-                            // var value = _reader.ReadSlice(start, end);
 
                             if (lastNonBlank == i)
                             {
-#if NET5_0_OR_GREATER 
+#if NET5_0_OR_GREATER
                                 value = value.TrimEnd();
 #else
                                 value = value.TrimEndPolyFill();
@@ -650,10 +664,10 @@ private bool TryGetTextSlice([NotNullWhen(true)] out TextSlice? textElement, out
                         textElementType,
                         TextElementTermination.CRLF
                     );
-                    
+
                     _reader.Position += 2;
                     _reader.Row += 1;
-                    
+
                     error = null;
                     return true;
                 }
@@ -987,7 +1001,7 @@ private bool TryGetInlineExpression(bool onlyLiteral, [NotNullWhen(true)] out II
                                 {
                                     _reader.Position += 2;
                                 }
-                                else if ('u'== c)
+                                else if ('u' == c)
                                 {
                                     _reader.Position += 2;
                                     if (!TrySkipUnicodeSequence(4, out error))
@@ -996,7 +1010,7 @@ private bool TryGetInlineExpression(bool onlyLiteral, [NotNullWhen(true)] out II
                                         return false;
                                     }
                                 }
-                                else if ('U'== c)
+                                else if ('U' == c)
                                 {
                                     _reader.Position += 2;
                                     if (!TrySkipUnicodeSequence(6, out error))