Fix perf of multiline semantic tokens, and LF handling of same

src/Razor/benchmarks/Microsoft.AspNetCore.Razor.Microbenchmarks/LanguageServer/RazorSemanticTokensBenchmark.cs

@@ -45,6 +45,9 @@ public class RazorSemanticTokensBenchmark : RazorLanguageServerBenchmarkBase
 
     private string TargetPath { get; set; }
 
+    [ParamsAllValues]
+    public bool WithMultiLineComment { get; set; }
+
     [GlobalSetup(Target = nameof(RazorSemanticTokensRangeAsync))]
     public async Task InitializeRazorSemanticAsync()
     {
@@ -53,8 +56,10 @@ public async Task InitializeRazorSemanticAsync()
         var projectRoot = Path.Combine(RepoRoot, "src", "Razor", "test", "testapps", "ComponentApp");
         ProjectFilePath = Path.Combine(projectRoot, "ComponentApp.csproj");
         PagesDirectory = Path.Combine(projectRoot, "Components", "Pages");
-        var filePath = Path.Combine(PagesDirectory, $"SemanticTokens.razor");
-        TargetPath = "/Components/Pages/SemanticTokens.razor";
+
+        var fileName = WithMultiLineComment ? "SemanticTokens_LargeMultiLineComment" : "SemanticTokens";
+        var filePath = Path.Combine(PagesDirectory, $"{fileName}.razor");
+        TargetPath = $"/Components/Pages/{fileName}.razor";
 
         var documentUri = new Uri(filePath);
         var documentSnapshot = GetDocumentSnapshot(ProjectFilePath, filePath, TargetPath);
@@ -94,11 +99,6 @@ await RazorSemanticTokenService.GetSemanticTokensAsync(
             textDocumentIdentifier, Range, DocumentContext, SemanticTokensLegend, cancellationToken).ConfigureAwait(false);
     }
 
-    private static LspServices GetLspServices()
-    {
-        throw new NotImplementedException();
-    }
-
     private async Task UpdateDocumentAsync(int newVersion, IDocumentSnapshot documentSnapshot, CancellationToken cancellationToken)
     {
         await ProjectSnapshotManagerDispatcher.RunOnDispatcherThreadAsync(

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Semantic/Services/TagHelperSemanticRangeVisitor.cs

@@ -8,7 +8,6 @@
 using Microsoft.AspNetCore.Razor.Language.Components;
 using Microsoft.AspNetCore.Razor.Language.Syntax;
 using Microsoft.AspNetCore.Razor.LanguageServer.Extensions;
-using Microsoft.AspNetCore.Razor.LanguageServer.Semantic.Models;
 using Microsoft.CodeAnalysis.Razor.Workspaces.Extensions;
 using Microsoft.VisualStudio.LanguageServer.Protocol;
 
@@ -485,21 +484,40 @@ private void AddSemanticRange(SyntaxNode node, int semanticKind)
             var childNodes = node.ChildNodes();
             if (childNodes.Count == 0)
             {
-                var content = node.GetContent();
-                var lines = content.Split(new string[] { Environment.NewLine }, StringSplitOptions.None);
                 var charPosition = range.Start.Character;
-                for (var i = 0; i < lines.Length; i++)
+                var lineStartAbsoluteIndex = node.SpanStart - charPosition;
+                for (var lineNumber = range.Start.Line; lineNumber <= range.End.Line; lineNumber++)
                 {
-                    var startPosition = new Position(range.Start.Line + i, charPosition);
-                    var endPosition = new Position(range.Start.Line + i, charPosition + lines[i].Length);
+                    var startPosition = new Position(lineNumber, charPosition);
+
+                    // NOTE: GetLineLength includes newlines but we don't report tokens for newlines so
+                    // need to account for them.
+                    var lineLength = source.Lines.GetLineLength(lineNumber);
+
+                    // For the last line, we end where the syntax tree tells us to. For all other lines, we end at the
+                    // last non-newline character
+                    var endChar = lineNumber == range.End.Line
+                       ? range.End.Character
+                       : GetLastNonWhitespaceCharacterOffset(source, lineStartAbsoluteIndex, lineLength);
+
+                    // Make sure we move our line start index pointer on, before potentially breaking out of the loop
+                    lineStartAbsoluteIndex += lineLength;
+                    charPosition = 0;
+
+                    // No tokens for blank lines
+                    if (endChar == 0)
+                    {
+                        continue;
+                    }
+
+                    var endPosition = new Position(lineNumber, endChar);
                     var lineRange = new Range
                     {
                         Start = startPosition,
                         End = endPosition
                     };
                     var semantic = new SemanticRange(semanticKind, lineRange, modifier: 0);
                     AddRange(semantic);
-                    charPosition = 0;
                 }
             }
             else
@@ -533,5 +551,21 @@ void AddRange(SemanticRange semanticRange)
                 _semanticRanges.Add(semanticRange);
             }
         }
+
+        static int GetLastNonWhitespaceCharacterOffset(RazorSourceDocument source, int lineStartAbsoluteIndex, int lineLength)
+        {
+            // lineStartAbsoluteIndex + lineLength is the first character of the next line, so move back one to get to the end of the line
+            lineLength--;
+
+            var lineEndAbsoluteIndex = lineStartAbsoluteIndex + lineLength;
+            if (lineEndAbsoluteIndex == 0 || lineLength == 0)
+            {
+                return lineLength;
+            }
+
+            return source[lineEndAbsoluteIndex - 1] is '\n' or '\r'
+                ? lineLength - 1
+                : lineLength;
+        }
     }
 }

src/Razor/test/Microsoft.AspNetCore.Razor.LanguageServer.Test/Semantic/RazorSemanticTokenInfoServiceTest.cs

@@ -7,6 +7,7 @@
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Razor.LanguageServer.Common;
 using Microsoft.AspNetCore.Razor.LanguageServer.Formatting;
+using Microsoft.AspNetCore.Razor.Test.Common;
 using Microsoft.AspNetCore.Razor.Test.Common.Mef;
 using Microsoft.CommonLanguageServerProtocol.Framework;
 using Microsoft.VisualStudio.LanguageServer.Protocol;
@@ -617,6 +618,35 @@ public async Task GetSemanticTokens_Razor_MultiLineCommentMidlineAsync()
         await AssertSemanticTokensAsync(documentText, isRazorFile: false, razorRange, csharpTokens: csharpTokens);
     }
 
+    [Fact]
+    public async Task GetSemanticTokens_Razor_MultiLineCommentWithBlankLines()
+    {
+        var documentText =
+            """
+                @* kdl
+
+                skd
+
+                        sdfasdfasdf
+                slf*@
+                """;
+
+        var razorRange = GetRange(documentText);
+        var csharpTokens = await GetCSharpSemanticTokensResponseAsync(documentText, razorRange, isRazorFile: false);
+        await AssertSemanticTokensAsync(documentText, isRazorFile: false, razorRange, csharpTokens: csharpTokens);
+    }
+
+    [Fact]
+    [WorkItem("https://github.com/dotnet/razor/issues/8176")]
+    public async Task GetSemanticTokens_Razor_MultiLineCommentWithBlankLines_LF()
+    {
+        var documentText = "@* kdl\n\nskd\n    \n        sdfasdfasdf\nslf*@";
+
+        var razorRange = GetRange(documentText);
+        var csharpTokens = await GetCSharpSemanticTokensResponseAsync(documentText, razorRange, isRazorFile: false);
+        await AssertSemanticTokensAsync(documentText, isRazorFile: false, razorRange, csharpTokens: csharpTokens);
+    }
+
     [Fact]
     public async Task GetSemanticTokens_Razor_MultiLineCommentAsync()
     {

src/Razor/test/Microsoft.AspNetCore.Razor.LanguageServer.Test/Semantic/TestFiles/RazorSemanticTokenInfoServiceTest/GetSemanticTokens_Razor_MultiLineCommentWithBlankLines.semantic.txt

@@ -0,0 +1,10 @@
+//line,characterPos,length,tokenType,modifier
+0 0 1 razorCommentTransition 0
+0 1 1 razorCommentStar 0
+0 1 4 razorComment 0
+2 0 3 razorComment 0
+1 0 4 razorComment 0
+1 0 19 razorComment 0
+1 0 3 razorComment 0
+0 3 1 razorCommentStar 0
+0 1 1 razorCommentTransition 0

src/Razor/test/Microsoft.AspNetCore.Razor.LanguageServer.Test/Semantic/TestFiles/RazorSemanticTokenInfoServiceTest/GetSemanticTokens_Razor_MultiLineCommentWithBlankLines_LF.semantic.txt

@@ -0,0 +1,10 @@
+//line,characterPos,length,tokenType,modifier
+0 0 1 razorCommentTransition 0
+0 1 1 razorCommentStar 0
+0 1 4 razorComment 0
+2 0 3 razorComment 0
+1 0 4 razorComment 0
+1 0 19 razorComment 0
+1 0 3 razorComment 0
+0 3 1 razorCommentStar 0
+0 1 1 razorCommentTransition 0

src/Razor/test/testapps/ComponentApp/Components/Pages/SemanticTokens_LargeMultiLineComment.razor

@@ -0,0 +1,51 @@
+@page "/semantic"
+
+<p>Words!</p>
+
+<p>@* this is 
+a multi
+line comment *@</p>
+
+<NavMenu 
+    Collapse="true"
+ />
+
+<NavMenu />
+
+@*<div class="top-row pl-4 navbar navbar-dark">
+    <a class="navbar-brand" href="">ComponentApp</a>
+    <button class="navbar-toggler" @onclick="@ToggleNavMenu">
+        <span class="navbar-toggler-icon"></span>
+    </button>
+</div>
+
+<div class="@NavMenuCssClass" @onclick="@ToggleNavMenu">
+    <ul class="nav flex-column">
+        <li class="nav-item px-3">
+            <NavLink class="nav-link" href="" Match="NavLinkMatch.All">
+                <span class="oi oi-home" aria-hidden="true"></span> Home
+            </NavLink>
+        </li>
+        <li class="nav-item px-3">
+            <NavLink class="nav-link" href="counter">
+                <span class="oi oi-plus" aria-hidden="true"></span> Counter
+            </NavLink>
+        </li>
+        <li class="nav-item px-3">
+            <NavLink class="nav-link" href="fetchdata">
+                <span class="oi oi-list-rich" aria-hidden="true"></span> Fetch data
+            </NavLink>
+        </li>
+    </ul>
+</div>
+
+@functions {
+    bool collapseNavMenu = true;
+
+    string NavMenuCssClass => collapseNavMenu ? "collapse" : null;
+
+    void ToggleNavMenu()
+    {
+        collapseNavMenu = !collapseNavMenu;
+    }
+}*@