diff --git a/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/SemanticTokens/AbstractRazorSemanticTokensInfoService.cs b/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/SemanticTokens/AbstractRazorSemanticTokensInfoService.cs index 7719d377ab1..d6416188253 100644 --- a/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/SemanticTokens/AbstractRazorSemanticTokensInfoService.cs +++ b/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/SemanticTokens/AbstractRazorSemanticTokensInfoService.cs @@ -295,15 +295,21 @@ private static int[] ConvertSemanticRangesToSemanticTokensData( var isFirstRange = true; var index = 0; SemanticRange previousRange = default; + var i = 0; foreach (var range in semanticRanges) { - if (TryWriteToken(range, previousRange, isFirstRange, sourceText, tokens.AsSpan(index, TokenSize))) + var nextRange = semanticRanges.Count > i + 1 + ? semanticRanges[i + 1] + : default; + + if (TryWriteToken(range, previousRange, nextRange, isFirstRange, sourceText, tokens.AsSpan(index, TokenSize))) { index += TokenSize; previousRange = range; } isFirstRange = false; + i++; } // The common case is that the ConvertIntoDataArray calls didn't find any overlap, and we can just directly use the @@ -319,12 +325,33 @@ private static int[] ConvertSemanticRangesToSemanticTokensData( static bool TryWriteToken( SemanticRange currentRange, SemanticRange previousRange, + SemanticRange nextRange, bool isFirstRange, SourceText sourceText, Span destination) { Debug.Assert(destination.Length == TokenSize); + // Due to the fact that Razor ranges can supersede C# ranges, we can end up with C# whitespace ranges we've + // added, that we not don't want after further processing, so check for that, and skip emitting those ranges. + if (currentRange.IsCSharpWhitespace) + { + // If the previous range is on the same line, and from Razor, then we don't want to emit this. + // This happens when we have leftover whitespace from between two C# ranges, that were superseded by Razor ranges. + if (previousRange.FromRazor && + currentRange.StartLine == previousRange.EndLine) + { + return false; + } + + // If the next range is Razor, then it's leftover whitespace before C#, that was superseded by Razor, so don't emit. + if (nextRange.FromRazor && + currentRange.StartCharacter == 0) + { + return false; + } + } + /* * In short, each token takes 5 integers to represent, so a specific token `i` in the file consists of the following array indices: * - at index `5*i` - `deltaLine`: token line number, relative to the previous token @@ -366,16 +393,6 @@ static bool TryWriteToken( deltaStart = currentRange.StartCharacter; } - // If this is a C# whitespace range, and the previous range is on the same line, and from Razor - // then we don't want to emit this. This happens when we have leftover whitespace from between - // two C# ranges, that were superseded by Razor ranges. - if (currentRange.IsCSharpWhitespace && - previousRange.FromRazor && - currentRange.StartLine == previousRange.EndLine) - { - return false; - } - destination[0] = deltaLine; destination[1] = deltaStart; diff --git a/src/Razor/test/Microsoft.VisualStudio.LanguageServices.Razor.Test/TestFiles/SemanticTokens/RazorComponents.txt b/src/Razor/test/Microsoft.VisualStudio.LanguageServices.Razor.Test/TestFiles/SemanticTokens/RazorComponents.txt index 79373585584..0d84e48c57c 100644 --- a/src/Razor/test/Microsoft.VisualStudio.LanguageServices.Razor.Test/TestFiles/SemanticTokens/RazorComponents.txt +++ b/src/Razor/test/Microsoft.VisualStudio.LanguageServices.Razor.Test/TestFiles/SemanticTokens/RazorComponents.txt @@ -41,6 +41,26 @@ Line Δ, Char Δ, Length, Type, Modifier(s), Text 0 1 1 markupTagDelimiter [] [/] 0 1 47 razorComponentElement [] [Microsoft.AspNetCore.Components.Forms.InputText] 0 47 1 markupTagDelimiter [] [>] +2 0 1 markupTagDelimiter [] [<] +0 1 9 razorComponentElement [] [InputText] +1 4 5 razorComponentAttribute [] [Value] +0 5 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 9 markupAttributeValue [] [someValue] +0 9 1 markupAttributeQuote [] ["] +1 4 8 markupAttribute [] [CssClass] +0 8 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 8 markupAttributeValue [] [my-class] +0 8 1 markupAttributeQuote [] ["] +1 4 11 razorComponentAttribute [] [DisplayName] +0 11 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 2 markupAttributeValue [] [My] +0 2 6 markupAttributeValue [] [ Input] +0 6 1 markupAttributeQuote [] ["] +0 2 1 markupTagDelimiter [] [/] +0 1 1 markupTagDelimiter [] [>] 2 0 1 razorTransition [] [@] 0 1 6 keyword [] [typeof] 0 6 1 punctuation [] [(] diff --git a/src/Razor/test/Microsoft.VisualStudio.LanguageServices.Razor.Test/TestFiles/SemanticTokens/RazorComponents_misc_file.txt b/src/Razor/test/Microsoft.VisualStudio.LanguageServices.Razor.Test/TestFiles/SemanticTokens/RazorComponents_misc_file.txt index dfeceace673..f0d3dabb229 100644 --- a/src/Razor/test/Microsoft.VisualStudio.LanguageServices.Razor.Test/TestFiles/SemanticTokens/RazorComponents_misc_file.txt +++ b/src/Razor/test/Microsoft.VisualStudio.LanguageServices.Razor.Test/TestFiles/SemanticTokens/RazorComponents_misc_file.txt @@ -41,6 +41,26 @@ Line Δ, Char Δ, Length, Type, Modifier(s), Text 0 1 1 markupTagDelimiter [] [/] 0 1 47 markupElement [] [Microsoft.AspNetCore.Components.Forms.InputText] 0 47 1 markupTagDelimiter [] [>] +2 0 1 markupTagDelimiter [] [<] +0 1 9 markupElement [] [InputText] +1 4 5 markupAttribute [] [Value] +0 5 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 9 markupAttributeValue [] [someValue] +0 9 1 markupAttributeQuote [] ["] +1 4 8 markupAttribute [] [CssClass] +0 8 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 8 markupAttributeValue [] [my-class] +0 8 1 markupAttributeQuote [] ["] +1 4 11 markupAttribute [] [DisplayName] +0 11 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 2 markupAttributeValue [] [My] +0 2 6 markupAttributeValue [] [ Input] +0 6 1 markupAttributeQuote [] ["] +0 2 1 markupTagDelimiter [] [/] +0 1 1 markupTagDelimiter [] [>] 2 0 1 razorTransition [] [@] 0 1 6 keyword [] [typeof] 0 6 1 punctuation [] [(] diff --git a/src/Razor/test/Microsoft.VisualStudio.LanguageServices.Razor.Test/TestFiles/SemanticTokens/RazorComponents_with_background.txt b/src/Razor/test/Microsoft.VisualStudio.LanguageServices.Razor.Test/TestFiles/SemanticTokens/RazorComponents_with_background.txt index 70e0ef908b9..b8e373d267f 100644 --- a/src/Razor/test/Microsoft.VisualStudio.LanguageServices.Razor.Test/TestFiles/SemanticTokens/RazorComponents_with_background.txt +++ b/src/Razor/test/Microsoft.VisualStudio.LanguageServices.Razor.Test/TestFiles/SemanticTokens/RazorComponents_with_background.txt @@ -41,6 +41,26 @@ Line Δ, Char Δ, Length, Type, Modifier(s), Text 0 1 1 markupTagDelimiter [] [/] 0 1 47 razorComponentElement [] [Microsoft.AspNetCore.Components.Forms.InputText] 0 47 1 markupTagDelimiter [] [>] +2 0 1 markupTagDelimiter [] [<] +0 1 9 razorComponentElement [] [InputText] +1 4 5 razorComponentAttribute [] [Value] +0 5 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 9 markupAttributeValue [] [someValue] +0 9 1 markupAttributeQuote [] ["] +1 4 8 markupAttribute [] [CssClass] +0 8 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 8 markupAttributeValue [] [my-class] +0 8 1 markupAttributeQuote [] ["] +1 4 11 razorComponentAttribute [] [DisplayName] +0 11 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 2 markupAttributeValue [] [My] +0 2 6 markupAttributeValue [] [ Input] +0 6 1 markupAttributeQuote [] ["] +0 2 1 markupTagDelimiter [] [/] +0 1 1 markupTagDelimiter [] [>] 2 0 1 razorTransition [razorCode] [@] 0 1 6 keyword [razorCode] [typeof] 0 6 1 punctuation [razorCode] [(] diff --git a/src/Razor/test/Microsoft.VisualStudio.LanguageServices.Razor.Test/TestFiles/SemanticTokens/RazorComponents_with_background_misc_file.txt b/src/Razor/test/Microsoft.VisualStudio.LanguageServices.Razor.Test/TestFiles/SemanticTokens/RazorComponents_with_background_misc_file.txt index 7ae31783f99..4b70a3752bc 100644 --- a/src/Razor/test/Microsoft.VisualStudio.LanguageServices.Razor.Test/TestFiles/SemanticTokens/RazorComponents_with_background_misc_file.txt +++ b/src/Razor/test/Microsoft.VisualStudio.LanguageServices.Razor.Test/TestFiles/SemanticTokens/RazorComponents_with_background_misc_file.txt @@ -41,6 +41,26 @@ Line Δ, Char Δ, Length, Type, Modifier(s), Text 0 1 1 markupTagDelimiter [] [/] 0 1 47 markupElement [] [Microsoft.AspNetCore.Components.Forms.InputText] 0 47 1 markupTagDelimiter [] [>] +2 0 1 markupTagDelimiter [] [<] +0 1 9 markupElement [] [InputText] +1 4 5 markupAttribute [] [Value] +0 5 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 9 markupAttributeValue [] [someValue] +0 9 1 markupAttributeQuote [] ["] +1 4 8 markupAttribute [] [CssClass] +0 8 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 8 markupAttributeValue [] [my-class] +0 8 1 markupAttributeQuote [] ["] +1 4 11 markupAttribute [] [DisplayName] +0 11 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 2 markupAttributeValue [] [My] +0 2 6 markupAttributeValue [] [ Input] +0 6 1 markupAttributeQuote [] ["] +0 2 1 markupTagDelimiter [] [/] +0 1 1 markupTagDelimiter [] [>] 2 0 1 razorTransition [razorCode] [@] 0 1 6 keyword [razorCode] [typeof] 0 6 1 punctuation [razorCode] [(] diff --git a/src/Razor/test/Microsoft.VisualStudioCode.RazorExtension.Test/Endpoints/Shared/CohostSemanticTokensRangeEndpointTest.cs b/src/Razor/test/Microsoft.VisualStudioCode.RazorExtension.Test/Endpoints/Shared/CohostSemanticTokensRangeEndpointTest.cs index bfec9c8c0ea..90bdc8130a1 100644 --- a/src/Razor/test/Microsoft.VisualStudioCode.RazorExtension.Test/Endpoints/Shared/CohostSemanticTokensRangeEndpointTest.cs +++ b/src/Razor/test/Microsoft.VisualStudioCode.RazorExtension.Test/Endpoints/Shared/CohostSemanticTokensRangeEndpointTest.cs @@ -30,6 +30,11 @@ public async Task RazorComponents(bool colorBackground, bool miscellaneousFile) + + @typeof(InputText).ToString() @typeof(Microsoft.AspNetCore.Components.Forms.InputText).ToString() """; diff --git a/src/Razor/test/Microsoft.VisualStudioCode.RazorExtension.Test/TestFiles/SemanticTokens/RazorComponents.txt b/src/Razor/test/Microsoft.VisualStudioCode.RazorExtension.Test/TestFiles/SemanticTokens/RazorComponents.txt index 729ec4f8992..9874150f958 100644 --- a/src/Razor/test/Microsoft.VisualStudioCode.RazorExtension.Test/TestFiles/SemanticTokens/RazorComponents.txt +++ b/src/Razor/test/Microsoft.VisualStudioCode.RazorExtension.Test/TestFiles/SemanticTokens/RazorComponents.txt @@ -41,6 +41,26 @@ Line Δ, Char Δ, Length, Type, Modifier(s), Text 0 1 1 markupTagDelimiter [] [/] 0 1 47 razorComponentElement [] [Microsoft.AspNetCore.Components.Forms.InputText] 0 47 1 markupTagDelimiter [] [>] +2 0 1 markupTagDelimiter [] [<] +0 1 9 razorComponentElement [] [InputText] +1 4 5 razorComponentAttribute [] [Value] +0 5 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 9 markupAttributeValue [] [someValue] +0 9 1 markupAttributeQuote [] ["] +1 4 8 markupAttribute [] [CssClass] +0 8 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 8 markupAttributeValue [] [my-class] +0 8 1 markupAttributeQuote [] ["] +1 4 11 razorComponentAttribute [] [DisplayName] +0 11 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 2 markupAttributeValue [] [My] +0 2 6 markupAttributeValue [] [ Input] +0 6 1 markupAttributeQuote [] ["] +0 2 1 markupTagDelimiter [] [/] +0 1 1 markupTagDelimiter [] [>] 2 0 1 razorTransition [] [@] 0 1 6 keyword [] [typeof] 0 6 1 punctuation [] [(] diff --git a/src/Razor/test/Microsoft.VisualStudioCode.RazorExtension.Test/TestFiles/SemanticTokens/RazorComponents_misc_file.txt b/src/Razor/test/Microsoft.VisualStudioCode.RazorExtension.Test/TestFiles/SemanticTokens/RazorComponents_misc_file.txt index 344aa6e5a99..22e3a323587 100644 --- a/src/Razor/test/Microsoft.VisualStudioCode.RazorExtension.Test/TestFiles/SemanticTokens/RazorComponents_misc_file.txt +++ b/src/Razor/test/Microsoft.VisualStudioCode.RazorExtension.Test/TestFiles/SemanticTokens/RazorComponents_misc_file.txt @@ -41,6 +41,26 @@ Line Δ, Char Δ, Length, Type, Modifier(s), Text 0 1 1 markupTagDelimiter [] [/] 0 1 47 markupElement [] [Microsoft.AspNetCore.Components.Forms.InputText] 0 47 1 markupTagDelimiter [] [>] +2 0 1 markupTagDelimiter [] [<] +0 1 9 markupElement [] [InputText] +1 4 5 markupAttribute [] [Value] +0 5 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 9 markupAttributeValue [] [someValue] +0 9 1 markupAttributeQuote [] ["] +1 4 8 markupAttribute [] [CssClass] +0 8 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 8 markupAttributeValue [] [my-class] +0 8 1 markupAttributeQuote [] ["] +1 4 11 markupAttribute [] [DisplayName] +0 11 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 2 markupAttributeValue [] [My] +0 2 6 markupAttributeValue [] [ Input] +0 6 1 markupAttributeQuote [] ["] +0 2 1 markupTagDelimiter [] [/] +0 1 1 markupTagDelimiter [] [>] 2 0 1 razorTransition [] [@] 0 1 6 keyword [] [typeof] 0 6 1 punctuation [] [(] diff --git a/src/Razor/test/Microsoft.VisualStudioCode.RazorExtension.Test/TestFiles/SemanticTokens/RazorComponents_with_background.txt b/src/Razor/test/Microsoft.VisualStudioCode.RazorExtension.Test/TestFiles/SemanticTokens/RazorComponents_with_background.txt index 1b8e3c533bd..f461b335ffa 100644 --- a/src/Razor/test/Microsoft.VisualStudioCode.RazorExtension.Test/TestFiles/SemanticTokens/RazorComponents_with_background.txt +++ b/src/Razor/test/Microsoft.VisualStudioCode.RazorExtension.Test/TestFiles/SemanticTokens/RazorComponents_with_background.txt @@ -41,6 +41,26 @@ Line Δ, Char Δ, Length, Type, Modifier(s), Text 0 1 1 markupTagDelimiter [] [/] 0 1 47 razorComponentElement [] [Microsoft.AspNetCore.Components.Forms.InputText] 0 47 1 markupTagDelimiter [] [>] +2 0 1 markupTagDelimiter [] [<] +0 1 9 razorComponentElement [] [InputText] +1 4 5 razorComponentAttribute [] [Value] +0 5 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 9 markupAttributeValue [] [someValue] +0 9 1 markupAttributeQuote [] ["] +1 4 8 markupAttribute [] [CssClass] +0 8 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 8 markupAttributeValue [] [my-class] +0 8 1 markupAttributeQuote [] ["] +1 4 11 razorComponentAttribute [] [DisplayName] +0 11 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 2 markupAttributeValue [] [My] +0 2 6 markupAttributeValue [] [ Input] +0 6 1 markupAttributeQuote [] ["] +0 2 1 markupTagDelimiter [] [/] +0 1 1 markupTagDelimiter [] [>] 2 0 1 razorTransition [razorCode] [@] 0 1 6 keyword [razorCode] [typeof] 0 6 1 punctuation [razorCode] [(] diff --git a/src/Razor/test/Microsoft.VisualStudioCode.RazorExtension.Test/TestFiles/SemanticTokens/RazorComponents_with_background_misc_file.txt b/src/Razor/test/Microsoft.VisualStudioCode.RazorExtension.Test/TestFiles/SemanticTokens/RazorComponents_with_background_misc_file.txt index d8b85cd0a86..eb59bf0f92a 100644 --- a/src/Razor/test/Microsoft.VisualStudioCode.RazorExtension.Test/TestFiles/SemanticTokens/RazorComponents_with_background_misc_file.txt +++ b/src/Razor/test/Microsoft.VisualStudioCode.RazorExtension.Test/TestFiles/SemanticTokens/RazorComponents_with_background_misc_file.txt @@ -41,6 +41,26 @@ Line Δ, Char Δ, Length, Type, Modifier(s), Text 0 1 1 markupTagDelimiter [] [/] 0 1 47 markupElement [] [Microsoft.AspNetCore.Components.Forms.InputText] 0 47 1 markupTagDelimiter [] [>] +2 0 1 markupTagDelimiter [] [<] +0 1 9 markupElement [] [InputText] +1 4 5 markupAttribute [] [Value] +0 5 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 9 markupAttributeValue [] [someValue] +0 9 1 markupAttributeQuote [] ["] +1 4 8 markupAttribute [] [CssClass] +0 8 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 8 markupAttributeValue [] [my-class] +0 8 1 markupAttributeQuote [] ["] +1 4 11 markupAttribute [] [DisplayName] +0 11 1 markupOperator [] [=] +0 1 1 markupAttributeQuote [] ["] +0 1 2 markupAttributeValue [] [My] +0 2 6 markupAttributeValue [] [ Input] +0 6 1 markupAttributeQuote [] ["] +0 2 1 markupTagDelimiter [] [/] +0 1 1 markupTagDelimiter [] [>] 2 0 1 razorTransition [razorCode] [@] 0 1 6 keyword [razorCode] [typeof] 0 6 1 punctuation [razorCode] [(]