Add a couple of "stress tests" to the integration test project

azure-pipelines-conditional-integration.yml

@@ -18,6 +18,8 @@ variables:
   value: false
 - name: Codeql.SkipTaskAutoInjection
   value: true
+- name: _IntegrationTestsRunningInCI
+  value: true
 
 trigger: none
 

azure-pipelines-integration-dartlab.yml

@@ -35,6 +35,8 @@ variables:
   value: false
 - name: Codeql.SkipTaskAutoInjection
   value: true
+- name: _IntegrationTestsRunningInCI
+  value: true
 
 stages:
 - template: \stages\visual-studio\agent.yml@DartLabTemplates

azure-pipelines.yml

@@ -20,6 +20,8 @@ variables:
   value: false
 - name: Codeql.SkipTaskAutoInjection
   value: true
+- name: _IntegrationTestsRunningInCI
+  value: true
 
 trigger:
   batch: true

src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/AbstractIntegrationTest.cs

@@ -31,6 +31,8 @@ public abstract class AbstractIntegrationTest : AbstractIdeIntegrationTest
 {
     protected CancellationToken ControlledHangMitigatingCancellationToken => HangMitigatingCancellationToken;
 
+    protected virtual bool AllowDebugFails => false;
+
     public override async Task InitializeAsync()
     {
         // Not sure why the module initializer doesn't seem to work for integration tests
@@ -41,11 +43,14 @@ public override async Task InitializeAsync()
 
     public override void Dispose()
     {
-        var fails = ThrowingTraceListener.Fails;
-        Assert.False(fails.Length > 0, $"""
-            Expected 0 Debug.Fail calls. Actual:
-            {string.Join(Environment.NewLine, fails)}
-            """);
+        if (!AllowDebugFails)
+        {
+            var fails = ThrowingTraceListener.Fails;
+            Assert.False(fails.Length > 0, $"""
+                Expected 0 Debug.Fail calls. Actual:
+                {string.Join(Environment.NewLine, fails)}
+                """);
+        }
 
         base.Dispose();
     }

src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/AbstractRazorEditorTest.cs

@@ -32,6 +32,10 @@ public abstract class AbstractRazorEditorTest(ITestOutputHelper testOutput) : Ab
 
     protected virtual string TargetFrameworkElement => $"""<TargetFramework>{TargetFramework}</TargetFramework>""";
 
+    protected virtual string ProjectZipFile => "Microsoft.VisualStudio.Razor.IntegrationTests.TestFiles.BlazorProject.zip";
+
+    private protected virtual ILogger Logger => _testLogger.AssumeNotNull();
+
     protected string ProjectFilePath => _projectFilePath.AssumeNotNull();
 
     public override async Task InitializeAsync()
@@ -96,19 +100,20 @@ private async Task<string> CreateAndOpenBlazorProjectAsync(CancellationToken can
 
         var solutionPath = CreateTemporaryPath();
 
-        var resourceName = "Microsoft.VisualStudio.Razor.IntegrationTests.TestFiles.BlazorProject.zip";
-        using var zipStream = typeof(AbstractRazorEditorTest).Assembly.GetManifestResourceStream(resourceName);
+        using var zipStream = typeof(AbstractRazorEditorTest).Assembly.GetManifestResourceStream(ProjectZipFile);
         using var zip = new ZipArchive(zipStream);
         zip.ExtractToDirectory(solutionPath);
 
         var slnFile = Directory.EnumerateFiles(solutionPath, "*.sln").Single();
-        var projectFile = Directory.EnumerateFiles(solutionPath, "*.csproj", SearchOption.AllDirectories).Single();
 
-        PrepareProjectForFirstOpen(projectFile);
+        foreach (var projectFile in Directory.EnumerateFiles(solutionPath, "*.csproj", SearchOption.AllDirectories))
+        {
+            PrepareProjectForFirstOpen(projectFile);
+        }
 
         await TestServices.SolutionExplorer.OpenSolutionAsync(slnFile, cancellationToken);
 
-        return projectFile;
+        return Directory.EnumerateFiles(solutionPath, $"{RazorProjectConstants.BlazorProjectName}.csproj", SearchOption.AllDirectories).Single();
     }
 
     protected virtual void PrepareProjectForFirstOpen(string projectFileName)

src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/InProcess/EditorInProcess_Classification.cs

@@ -20,8 +20,10 @@ internal partial class EditorInProcess
     /// </summary>
     /// <param name="cancellationToken">A cancellation token.</param>
     /// <param name="count">The number of the given classification to expect.</param>
+    /// <param name="exact">Whether to wait for exactly <paramref name="count"/> classifications.</param>
     /// <returns>A <see cref="Task"/> which completes when classification is "ready".</returns>
-    public Task WaitForComponentClassificationAsync(CancellationToken cancellationToken, int count = 1) => WaitForSemanticClassificationAsync("RazorComponentElement", cancellationToken, count);
+    public Task WaitForComponentClassificationAsync(CancellationToken cancellationToken, int count = 1, bool exact = false)
+        => WaitForSemanticClassificationAsync("RazorComponentElement", cancellationToken, count, exact);
 
     /// <summary>
     /// Waits for any semantic classifications to be available on the active TextView, and for at least one of the
@@ -30,8 +32,9 @@ internal partial class EditorInProcess
     /// <param name="cancellationToken">A cancellation token.</param>
     /// <param name="expectedClassification">The classification to wait for, if any.</param>
     /// <param name="count">The number of the given classification to expect.</param>
+    /// <param name="exact">Whether to wait for exactly <paramref name="count"/> classifications.</param>
     /// <returns>A <see cref="Task"/> which completes when classification is "ready".</returns>
-    public async Task WaitForSemanticClassificationAsync(string expectedClassification, CancellationToken cancellationToken, int count = 1)
+    public async Task WaitForSemanticClassificationAsync(string expectedClassification, CancellationToken cancellationToken, int count = 1, bool exact = false)
     {
         var textView = await TestServices.Editor.GetActiveTextViewAsync(cancellationToken);
         var classifier = await GetClassifierAsync(textView, cancellationToken);
@@ -42,7 +45,7 @@ public async Task WaitForSemanticClassificationAsync(string expectedClassificati
         classifier.ClassificationChanged += Classifier_ClassificationChanged;
 
         // Check that we're not ALREADY changed
-        if (HasClassification(classifier, textView, expectedClassification, count))
+        if (HasClassification(classifier, textView, expectedClassification, count, exact))
         {
             semaphore.Release();
             classifier.ClassificationChanged -= Classifier_ClassificationChanged;
@@ -60,13 +63,13 @@ public async Task WaitForSemanticClassificationAsync(string expectedClassificati
 
         void Classifier_ClassificationChanged(object sender, ClassificationChangedEventArgs e)
         {
-            if (HasClassification(classifier, textView, expectedClassification, count))
+            if (HasClassification(classifier, textView, expectedClassification, count, exact))
             {
                 semaphore.Release();
             }
         }
 
-        static bool HasClassification(IClassifier classifier, ITextView textView, string expectedClassification, int count)
+        static bool HasClassification(IClassifier classifier, ITextView textView, string expectedClassification, int count, bool exact)
         {
             var classifications = GetClassifications(classifier, textView);
 
@@ -80,7 +83,8 @@ static bool HasClassification(IClassifier classifier, ITextView textView, string
                 }
             }
 
-            return found >= count;
+            return found == count ||
+                (!exact && found > count);
         }
 
         static bool ClassificationMatches(string expectedClassification, IClassificationType classificationType)

src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/InProcess/EditorInProcess_Commands.cs

@@ -58,7 +58,8 @@ public async Task InvokeRenameAsync(CancellationToken cancellationToken)
         var commandId = VSStd2KCmdID.RENAME;
 
         // Rename seems to be extra-succeptable to COM exceptions
-        await Helper.RetryAsync<bool?>(async (cancellationToken) => {
+        await Helper.RetryAsync<bool?>(async (cancellationToken) =>
+        {
             await ExecuteCommandAsync(commandGuid, (uint)commandId, cancellationToken);
             return true;
         }, TimeSpan.FromSeconds(1), cancellationToken);
@@ -69,15 +70,18 @@ public async Task CloseCodeFileAsync(string projectName, string relativeFilePath
         await CloseFileAsync(projectName, relativeFilePath, VSConstants.LOGVIEWID.Code_guid, saveFile, cancellationToken);
     }
 
-    public async Task CloseCurrentlyFocusedWindowAsync(CancellationToken cancellationToken)
+    public async Task CloseCurrentlyFocusedWindowAsync(CancellationToken cancellationToken, bool save = false)
     {
         await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken);
 
         var monitorSelection = await GetRequiredGlobalServiceAsync<SVsShellMonitorSelection, IVsMonitorSelection>(cancellationToken);
         ErrorHandler.ThrowOnFailure(monitorSelection.GetCurrentElementValue((uint)VSSELELEMID.SEID_WindowFrame, out var windowFrameObj));
         var windowFrame = (IVsWindowFrame)windowFrameObj;
 
-        ErrorHandler.ThrowOnFailure(windowFrame.CloseFrame((uint)__FRAMECLOSE.FRAMECLOSE_NoSave));
+        var closeFlags = save
+            ? __FRAMECLOSE.FRAMECLOSE_SaveIfDirty
+            : __FRAMECLOSE.FRAMECLOSE_NoSave;
+        ErrorHandler.ThrowOnFailure(windowFrame.CloseFrame((uint)closeFlags));
     }
 
     private async Task ExecuteCommandAsync(Guid commandGuid, uint commandId, CancellationToken cancellationToken)

src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/InProcess/EditorInProcess_LightBulb.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT license. See License.txt in the project root for license information.
 
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.VisualStudio.Language.Intellisense;
@@ -24,6 +25,15 @@ public async Task DismissLightBulbSessionAsync(CancellationToken cancellationTok
         broker.DismissSession(view);
     }
 
+    public async Task InvokeCodeActionAsync(string codeActionTitle, CancellationToken cancellationToken)
+    {
+        var codeActions = await ShowLightBulbAsync(cancellationToken);
+
+        var codeAction = codeActions.First(a => a.Actions.Single().DisplayText == codeActionTitle).Actions.Single();
+
+        await InvokeCodeActionAsync(codeAction, cancellationToken);
+    }
+
     public async Task<IEnumerable<SuggestedActionSet>> InvokeCodeActionListAsync(CancellationToken cancellationToken)
     {
         var lightbulbs = await ShowLightBulbAsync(cancellationToken);

src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/ManualRunOnlyIdeFactAttribute.cs

@@ -0,0 +1,19 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT license. See License.txt in the project root for license information.
+
+using System;
+using Xunit;
+
+namespace Microsoft.VisualStudio.Razor.IntegrationTests;
+
+[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
+public class ManualRunOnlyIdeFactAttribute : IdeFactAttribute
+{
+    public ManualRunOnlyIdeFactAttribute()
+    {
+        if (Environment.GetEnvironmentVariable("_IntegrationTestsRunningInCI") is not null)
+        {
+            Skip = "This test can only run manually";
+        }
+    }
+}

src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/StressTests/AbstractStressTest.cs

@@ -0,0 +1,49 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT license. See License.txt in the project root for license information.
+
+using System;
+using System.Diagnostics;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.CodeAnalysis.Razor.Logging;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Microsoft.VisualStudio.Razor.IntegrationTests;
+
+[IdeSettings(MinVersion = VisualStudioVersion.VS2022, RootSuffix = "RoslynDev", MaxAttempts = 1)]
+public abstract class AbstractStressTest(ITestOutputHelper testOutputHelper) : AbstractRazorEditorTest(testOutputHelper)
+{
+    protected override bool AllowDebugFails => true;
+
+    protected Task RunStressTestAsync(Func<int, CancellationToken, Task> iterationFunc)
+        => RunStressTestAsync(iterationFunc, TimeSpan.FromHours(1), TimeSpan.FromMinutes(1));
+
+    protected async Task RunStressTestAsync(Func<int, CancellationToken, Task> iterationFunc, TimeSpan maxRunTime, TimeSpan iterationTimeout)
+    {
+        var min = long.MaxValue;
+        var max = long.MinValue;
+        var avg = 0L;
+
+        var i = 0;
+        var start = DateTime.Now;
+        while (DateTime.Now.Subtract(maxRunTime) < start)
+        {
+            var iterationStart = Stopwatch.GetTimestamp();
+            Logger.LogInformation($"**** Test iteration started: #{i} at {DateTime.Now}");
+
+            using var iterationCts = new CancellationTokenSource(iterationTimeout);
+            var iterationToken = iterationCts.Token;
+
+            await iterationFunc(i, iterationToken);
+            i++;
+
+            var duration = Stopwatch.GetTimestamp() - iterationStart;
+            min = Math.Min(min, duration);
+            max = Math.Max(max, duration);
+            avg = ((avg * (i - 1)) + duration) / i;
+            Logger.LogInformation($"**** Test iteration finished: #{i} in {TimeSpan.FromTicks(duration).TotalMilliseconds}ms");
+            Logger.LogInformation($"**** Test iteration duration: min={TimeSpan.FromTicks(min).TotalMilliseconds}ms, max={TimeSpan.FromTicks(max).TotalMilliseconds}ms, avg={TimeSpan.FromTicks(avg).TotalMilliseconds}ms");
+        }
+    }
+}

src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/StressTests/RCLStressTests.cs

@@ -0,0 +1,82 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT license. See License.txt in the project root for license information.
+
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.WebTools.Languages.Shared.Editor.EditorHelpers;
+using Xunit.Abstractions;
+
+namespace Microsoft.VisualStudio.Razor.IntegrationTests;
+
+public class RCLStressTests(ITestOutputHelper testOutputHelper) : AbstractStressTest(testOutputHelper)
+{
+    protected override string TargetFramework => "net9.0";
+
+    protected override string ProjectZipFile => "Microsoft.VisualStudio.Razor.IntegrationTests.TestFiles.BlazorProjectWithRCL.zip";
+
+    [ManualRunOnlyIdeFact]
+    public async Task AddAndRemoveComponentInRCL()
+    {
+        await TestServices.SolutionExplorer.OpenFileAsync("RazorClassLibrary", @"Components\RCLComponent.razor", ControlledHangMitigatingCancellationToken);
+
+        await TestServices.Editor.PlaceCaretAsync("<div", charsOffset: -1, ControlledHangMitigatingCancellationToken);
+
+        await RunStressTestAsync(RunIterationAsync);
+
+        async Task RunIterationAsync(int index, CancellationToken cancellationToken)
+        {
+            await TestServices.Editor.InsertTextAsync($"<h1>Iteration {index}</h1>{Environment.NewLine}", cancellationToken);
+
+            await TestServices.Editor.PlaceCaretAsync("h1", charsOffset: -1, cancellationToken);
+
+            await TestServices.Editor.WaitForComponentClassificationAsync(cancellationToken, count: 1, exact: true);
+
+            await TestServices.Editor.InvokeCodeActionAsync("Extract element to new component", cancellationToken);
+
+            await TestServices.Editor.WaitForActiveWindowByFileAsync("Component.razor", cancellationToken);
+
+            await TestServices.Editor.PlaceCaretAsync("<h1", charsOffset: -1, cancellationToken);
+
+            // TODO: Remove once https://github.com/dotnet/razor/issues/11478 is fixed
+            await TestServices.Editor.InsertTextAsync($"@namespace MyCoolNamespace{Environment.NewLine}{Environment.NewLine}", cancellationToken);
+
+            var componentFileName = (await TestServices.Editor.GetActiveTextViewAsync(cancellationToken)).TextBuffer.GetFileName();
+
+            await TestServices.Editor.CloseCurrentlyFocusedWindowAsync(cancellationToken, save: true);
+
+            await TestServices.Editor.WaitForActiveWindowByFileAsync("RCLComponent.razor", cancellationToken);
+
+            await TestServices.Editor.WaitForComponentClassificationAsync(cancellationToken, count: 2, exact: true);
+
+            await TestServices.SolutionExplorer.OpenFileAsync(RazorProjectConstants.BlazorProjectName, RazorProjectConstants.IndexRazorFile, ControlledHangMitigatingCancellationToken);
+
+            await TestServices.Editor.PlaceCaretAsync("h1", charsOffset: -1, cancellationToken);
+
+            await TestServices.Editor.InvokeDeleteLineAsync(cancellationToken);
+
+            await TestServices.Editor.InsertTextAsync($"<Component />{Environment.NewLine}", cancellationToken);
+
+            await TestServices.Editor.WaitForComponentClassificationAsync(cancellationToken, count: 5, exact: true);
+
+            File.Delete(componentFileName);
+
+            await TestServices.Editor.WaitForComponentClassificationAsync(cancellationToken, count: 4, exact: true);
+
+            await TestServices.Editor.PlaceCaretAsync("<Component />", charsOffset: -1, cancellationToken);
+
+            await TestServices.Editor.InvokeDeleteLineAsync(cancellationToken);
+
+            await TestServices.Editor.InsertTextAsync($"<h1>Iteration {index}</h1>{Environment.NewLine}", cancellationToken);
+
+            await TestServices.SolutionExplorer.OpenFileAsync("RazorClassLibrary", @"Components\RCLComponent.razor", ControlledHangMitigatingCancellationToken);
+
+            await TestServices.Editor.WaitForComponentClassificationAsync(cancellationToken, count: 1, exact: true);
+
+            await TestServices.Editor.PlaceCaretAsync("<Component />", charsOffset: -1, cancellationToken);
+
+            await TestServices.Editor.InvokeDeleteLineAsync(cancellationToken);
+        }
+    }
+}