[.NET] GherkinLine: explicit struct Enumerator

Author: obligaron

dotnet/Gherkin.Specs/Tokens/TestTokenFormatter.cs

@@ -7,7 +7,8 @@ public string FormatToken(Token token)
         if (token.IsEOF)
             return "EOF";
 
-        string stepTypeText;
+        string stepTypeText = string.Empty;
+        string matchedItemsText = null;
         switch (token.MatchedType)
         {
             case TokenType.FeatureLine:
@@ -22,13 +23,14 @@ public string FormatToken(Token token)
                 var tokenType = token.MatchedGherkinDialect.GetStepKeywordType(token.MatchedKeyword);
                 stepTypeText = $"({tokenType})";
                 break;
-            default:
-                stepTypeText = "";
+            case TokenType.TagLine:
+                matchedItemsText = string.Join(",", token.Line.GetTags().Select(i => i.Column + ":" + i.Text));
+                break;
+            case TokenType.TableRow:
+                matchedItemsText = string.Join(",", token.Line.GetTableCells().Select(i => i.Column + ":" + i.Text));
                 break;
         }
 
-        var matchedItemsText = token.MatchedItems == null ? "" : string.Join(",", token.MatchedItems.Select(i => i.Column + ":" + i.Text));
-
         return $"({token.Location.Line}:{token.Location.Column}){token.MatchedType}:{stepTypeText}{token.MatchedKeyword}/{token.MatchedText}/{matchedItemsText}";
     }
 }

dotnet/Gherkin/AstBuilder.cs

@@ -262,7 +262,7 @@ private IEnumerable<Tag> GetTags(AstNode node)
         var tags = new List<Tag>();
         foreach (var line in tagsNode.GetTokens(TokenType.TagLine))
         {
-            foreach (var matchedItem in line.MatchedItems)
+            foreach (var matchedItem in line.Line.GetTags())
                 tags.Add(CreateTag(GetLocation(line, matchedItem.Column), matchedItem.Text, tagsNode));
         }
         return tags;
@@ -277,7 +277,7 @@ private List<TableRow> GetTableRows(AstNode node)
         {
             var rowLocation = GetLocation(rowToken);
             var cells = new List<TableCell>();
-            foreach (var cellItem in rowToken.MatchedItems)
+            foreach (var cellItem in rowToken.Line.GetTableCells())
                 cells.Add(CreateTableCell(GetLocation(rowToken, cellItem.Column), cellItem.Text));
             if (firstRow)
             {

dotnet/Gherkin/GherkinLine.cs

@@ -1,4 +1,5 @@
 using Gherkin.Ast;
+using System.Collections;
 
 namespace Gherkin;
 
@@ -97,69 +98,131 @@ public string GetRestTrimmed(int length)
         return lineText.Substring(trimmedStartIndex + length).Trim();
     }
 
-    /// <summary>
-    /// Tries parsing the line as a tag list, and returns the tags wihtout the leading '@' characters.
-    /// </summary>
-    /// <returns>(position,text) pairs, position is 0-based index</returns>
-    public IEnumerable<GherkinLineSpan> GetTags()
+    public readonly struct TagsEnumerable : IEnumerable<GherkinLineSpan>
     {
-        string uncommentedLine = lineText;
-        var commentIndex = lineText.IndexOf(GherkinLanguageConstants.COMMENT_PREFIX[0], trimmedStartIndex);
-        while (commentIndex >= 0)
+        readonly int lineNumber;
+        readonly string uncommentedLine;
+        readonly int position;
+        public TagsEnumerable(int lineNumber, string lineText, int trimmedStartIndex)
         {
-            if (commentIndex == 0)
-                yield break;
-            if (Array.IndexOf(inlineWhitespaceChars, lineText[commentIndex - 1]) != -1)
+            this.lineNumber = lineNumber;
+            uncommentedLine = lineText;
+            var commentIndex = lineText.IndexOf(GherkinLanguageConstants.COMMENT_PREFIX[0], trimmedStartIndex);
+            while (commentIndex >= 0)
             {
-                uncommentedLine = uncommentedLine.Substring(0, commentIndex);
-                break;
+                if (commentIndex == 0)
+                {
+                    position = -1;
+                    return;
+                }
+                if (Array.IndexOf(inlineWhitespaceChars, lineText[commentIndex - 1]) != -1)
+                {
+                    uncommentedLine = uncommentedLine.Substring(0, commentIndex);
+                    break;
+                }
+                commentIndex = lineText.IndexOf(GherkinLanguageConstants.COMMENT_PREFIX[0], commentIndex + 1);
             }
-            commentIndex = lineText.IndexOf(GherkinLanguageConstants.COMMENT_PREFIX[0], commentIndex + 1);
+            position = uncommentedLine.IndexOf(GherkinLanguageConstants.TAG_PREFIX[0], trimmedStartIndex);
         }
-        int position = uncommentedLine.IndexOf(GherkinLanguageConstants.TAG_PREFIX[0], trimmedStartIndex);
-        while (position >= 0)
-        { 
-            int nextPos = uncommentedLine.IndexOf(GherkinLanguageConstants.TAG_PREFIX[0], position + 1);
-            int endPos;
-            if (nextPos > 0)
-                endPos = nextPos - 1;
-            else
-                endPos = uncommentedLine.Length - 1;
-
-            while (endPos > position && Array.IndexOf(inlineWhitespaceChars, lineText[endPos]) != -1) // TrimEnd
-                endPos -= 1;
-
-            int length = endPos - position + 1;
-            if (length <= 1)
+
+        public TagsEnumerator GetEnumerator() => new TagsEnumerator(lineNumber, uncommentedLine, position);
+
+        IEnumerator<GherkinLineSpan> IEnumerable<GherkinLineSpan>.GetEnumerator() => GetEnumerator();
+        IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
+    }
+
+    public struct TagsEnumerator : IEnumerator<GherkinLineSpan>
+    {
+        readonly int lineNumber;
+        readonly string uncommentedLine;
+        int position;
+
+        public TagsEnumerator(int lineNumber, string uncommentedLine, int position) : this()
+        {
+            this.lineNumber = lineNumber;
+            this.uncommentedLine = uncommentedLine;
+            this.position = position;
+        }
+
+        public GherkinLineSpan Current { readonly get; private set; }
+        readonly object IEnumerator.Current => Current;
+
+        public bool MoveNext()
+        {
+            while (position >= 0)
             {
-                position = nextPos;
-                continue;
-            }
+                int nextPos = uncommentedLine.IndexOf(GherkinLanguageConstants.TAG_PREFIX[0], position + 1);
+                int endPos;
+                if (nextPos > 0)
+                    endPos = nextPos - 1;
+                else
+                    endPos = uncommentedLine.Length - 1;
+
+                while (endPos > position && Array.IndexOf(inlineWhitespaceChars, uncommentedLine[endPos]) != -1) // TrimEnd
+                    endPos -= 1;
+
+                int length = endPos - position + 1;
+                if (length <= 1)
+                {
+                    position = nextPos;
+                    continue;
+                }
 
-            var tagName = lineText.Substring(position, length);
+                var tagName = uncommentedLine.Substring(position, length);
 
-            if (tagName.IndexOfAny(inlineWhitespaceChars) >= 0)
-                throw new InvalidTagException("A tag may not contain whitespace", new Location(LineNumber, position + 1));
+                if (tagName.IndexOfAny(inlineWhitespaceChars) >= 0)
+                    throw new InvalidTagException("A tag may not contain whitespace", new Location(lineNumber, position + 1));
 
-            yield return new GherkinLineSpan(position + 1, tagName);
+                Current = new GherkinLineSpan(position + 1, tagName);
+                position = nextPos;
+                return true;
+            }
 
-            position = nextPos;
+            Current = default;
+            return false;
+        }
+
+        readonly void IDisposable.Dispose()
+        {
+            // nothing to do
         }
+
+        void IEnumerator.Reset() => throw new NotImplementedException();
     }
 
     /// <summary>
-    /// Tries parsing the line as table row and returns the trimmed cell values.
+    /// Tries parsing the line as a tag list, and returns the tags wihtout the leading '@' characters.
     /// </summary>
     /// <returns>(position,text) pairs, position is 0-based index</returns>
-    public IEnumerable<GherkinLineSpan> GetTableCells()
+    public TagsEnumerable GetTags() => new TagsEnumerable(LineNumber, lineText, trimmedStartIndex);
+
+    public readonly struct TableCellsEnumerable(string lineText, int startPos) : IEnumerable<GherkinLineSpan>
     {
-        bool isFirstRow = true;
+        public TableCellsEnumerator GetEnumerator() => new TableCellsEnumerator(lineText, startPos);
 
-        string cell = null;
-        int startPos = trimmedStartIndex;
-        int pos = startPos;
+        IEnumerator<GherkinLineSpan> IEnumerable<GherkinLineSpan>.GetEnumerator() => GetEnumerator();
+        IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
+    }
+
+    public struct TableCellsEnumerator : IEnumerator<GherkinLineSpan>
+    {
+        readonly string lineText;
+        int startPos;
+        int pos;
+        bool isFirstRow;
 
-        static void EnsureCellText(ref string cell, string lineText, ref int startPos, int pos, bool trim)
+        public TableCellsEnumerator(string lineText, int startPos)
+        {
+            this.lineText = lineText;
+            this.startPos = startPos;
+            this.pos = startPos;
+            this.isFirstRow = true;
+        }
+
+        public GherkinLineSpan Current { readonly get; private set; }
+        readonly object IEnumerator.Current => Current;
+
+        void EnsureCellText(ref string cell, bool trim)
         {
             if (cell is not null)
             {
@@ -178,53 +241,76 @@ static void EnsureCellText(ref string cell, string lineText, ref int startPos, i
             cell = lineText.Substring(startPos, trimedPos - startPos + 1);
         }
 
-        while (pos < lineText.Length)
+        public bool MoveNext()
         {
-            char c = lineText[pos];
-            pos++;
-            if (c == GherkinLanguageConstants.TABLE_CELL_SEPARATOR_CHAR)
-            {
-                if (isFirstRow)
-                    isFirstRow = false;
-                else
-                {
-                    EnsureCellText(ref cell, lineText, ref startPos, pos, true);
+            string cell = null;
 
-                    yield return new GherkinLineSpan(startPos + 1, cell);
-                }
-                cell = null;
-                startPos = pos;
-            }
-            else if (c == GherkinLanguageConstants.TABLE_CELL_ESCAPE_CHAR)
+            while (pos < lineText.Length)
             {
-                EnsureCellText(ref cell, lineText, ref startPos, pos, false);
-                if ((pos + 1) < lineText.Length)
+                char c = lineText[pos];
+                pos++;
+                if (c == GherkinLanguageConstants.TABLE_CELL_SEPARATOR_CHAR)
                 {
-                    c = lineText[pos];
-                    pos++;
-                    if (c == GherkinLanguageConstants.TABLE_CELL_NEWLINE_ESCAPE)
+                    if (isFirstRow)
                     {
-                        cell += "\n";
+                        isFirstRow = false;
+                        startPos = pos;
                     }
                     else
                     {
-                        if (c != GherkinLanguageConstants.TABLE_CELL_SEPARATOR_CHAR && c != GherkinLanguageConstants.TABLE_CELL_ESCAPE_CHAR)
+                        EnsureCellText(ref cell, true);
+
+                        Current = new GherkinLineSpan(startPos + 1, cell);
+                        startPos = pos;
+                        return true;
+                    }
+                }
+                else if (c == GherkinLanguageConstants.TABLE_CELL_ESCAPE_CHAR)
+                {
+                    EnsureCellText(ref cell, false);
+                    if ((pos + 1) < lineText.Length)
+                    {
+                        c = lineText[pos];
+                        pos++;
+                        if (c == GherkinLanguageConstants.TABLE_CELL_NEWLINE_ESCAPE)
                         {
-                            cell += GherkinLanguageConstants.TABLE_CELL_ESCAPE_CHAR;
+                            cell += "\n";
                         }
-                        cell += c;
+                        else
+                        {
+                            if (c != GherkinLanguageConstants.TABLE_CELL_SEPARATOR_CHAR && c != GherkinLanguageConstants.TABLE_CELL_ESCAPE_CHAR)
+                            {
+                                cell += GherkinLanguageConstants.TABLE_CELL_ESCAPE_CHAR;
+                            }
+                            cell += c;
+                        }
+                    }
+                    else
+                    {
+                        cell += GherkinLanguageConstants.TABLE_CELL_ESCAPE_CHAR;
                     }
                 }
                 else
                 {
-                    cell += GherkinLanguageConstants.TABLE_CELL_ESCAPE_CHAR;
+                    if (cell is not null)
+                        cell += c;
                 }
             }
-            else
-            {
-                if (cell is not null)
-                    cell += c;
-            }
+
+            return false;
         }
+
+        readonly void IDisposable.Dispose()
+        {
+            // nothing to do
+        }
+
+        void IEnumerator.Reset() => throw new NotImplementedException();
     }
+
+    /// <summary>
+    /// Tries parsing the line as table row and returns the trimmed cell values.
+    /// </summary>
+    /// <returns>(position,text) pairs, position is 0-based index</returns>
+    public TableCellsEnumerable GetTableCells() => new TableCellsEnumerable(lineText, trimmedStartIndex);
 }

dotnet/Gherkin/Token.cs

@@ -20,7 +20,6 @@ public Token(Location location)
     public TokenType MatchedType { get; set; }
     public string MatchedKeyword { get; set; }
     public string MatchedText { get; set; }
-    public IEnumerable<GherkinLineSpan> MatchedItems { get; set; }
     public int MatchedIndent { get; set; }
     public GherkinDialect MatchedGherkinDialect { get; set; }
     public Location Location { get; set; }

dotnet/Gherkin/TokenMatcher.cs

@@ -32,12 +32,11 @@ public void Reset()
             currentDialect = dialectProvider.DefaultDialect;
     }
 
-    protected virtual void SetTokenMatched(Token token, TokenType matchedType, string text = null, string keyword = null, int? indent = null, IEnumerable<GherkinLineSpan> items = null)
+    protected virtual void SetTokenMatched(Token token, TokenType matchedType, string text = null, string keyword = null, int? indent = null)
     {
         token.MatchedType = matchedType;
         token.MatchedKeyword = keyword;
         token.MatchedText = text;
-        token.MatchedItems = items;
         token.MatchedGherkinDialect = CurrentDialect;
         token.MatchedIndent = indent ?? (token.IsEOF ? 0 : token.Line.Indent);
         token.Location = new Ast.Location(token.Location.Line, token.MatchedIndent + 1);
@@ -113,7 +112,7 @@ public bool Match_TagLine(Token token)
     {
         if (token.Line.StartsWith(GherkinLanguageConstants.TAG_PREFIX))
         {
-            SetTokenMatched(token, TokenType.TagLine, items: token.Line.GetTags());
+            SetTokenMatched(token, TokenType.TagLine);
             return true;
         }
         return false;
@@ -212,7 +211,7 @@ public bool Match_TableRow(Token token)
     {
         if (token.Line.StartsWith(GherkinLanguageConstants.TABLE_CELL_SEPARATOR))
         {
-            SetTokenMatched(token, TokenType.TableRow, items: token.Line.GetTableCells());
+            SetTokenMatched(token, TokenType.TableRow);
             return true;
         }
         return false;