diff --git a/azure-pipelines-conditional-integration.yml b/azure-pipelines-conditional-integration.yml index 12272bd7526..72194970581 100644 --- a/azure-pipelines-conditional-integration.yml +++ b/azure-pipelines-conditional-integration.yml @@ -18,6 +18,8 @@ variables: value: false - name: Codeql.SkipTaskAutoInjection value: true +- name: _IntegrationTestsRunningInCI + value: true trigger: none diff --git a/azure-pipelines-integration-dartlab.yml b/azure-pipelines-integration-dartlab.yml index a8f362cf9d2..1d3ca2df260 100644 --- a/azure-pipelines-integration-dartlab.yml +++ b/azure-pipelines-integration-dartlab.yml @@ -35,6 +35,8 @@ variables: value: false - name: Codeql.SkipTaskAutoInjection value: true +- name: _IntegrationTestsRunningInCI + value: true stages: - template: \stages\visual-studio\agent.yml@DartLabTemplates diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f7dbd5c10e0..2bd257f1a7d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -20,6 +20,8 @@ variables: value: false - name: Codeql.SkipTaskAutoInjection value: true +- name: _IntegrationTestsRunningInCI + value: true trigger: batch: true diff --git a/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/AbstractIntegrationTest.cs b/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/AbstractIntegrationTest.cs index 65bd85d6d6a..d3b38fa46c9 100644 --- a/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/AbstractIntegrationTest.cs +++ b/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/AbstractIntegrationTest.cs @@ -31,6 +31,8 @@ public abstract class AbstractIntegrationTest : AbstractIdeIntegrationTest { protected CancellationToken ControlledHangMitigatingCancellationToken => HangMitigatingCancellationToken; + protected virtual bool AllowDebugFails => false; + public override async Task InitializeAsync() { // Not sure why the module initializer doesn't seem to work for integration tests @@ -41,11 +43,14 @@ public override async Task InitializeAsync() public override void Dispose() { - var fails = ThrowingTraceListener.Fails; - Assert.False(fails.Length > 0, $""" - Expected 0 Debug.Fail calls. Actual: - {string.Join(Environment.NewLine, fails)} - """); + if (!AllowDebugFails) + { + var fails = ThrowingTraceListener.Fails; + Assert.False(fails.Length > 0, $""" + Expected 0 Debug.Fail calls. Actual: + {string.Join(Environment.NewLine, fails)} + """); + } base.Dispose(); } diff --git a/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/AbstractRazorEditorTest.cs b/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/AbstractRazorEditorTest.cs index a2017ae8d96..157d6f375d5 100644 --- a/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/AbstractRazorEditorTest.cs +++ b/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/AbstractRazorEditorTest.cs @@ -32,6 +32,10 @@ public abstract class AbstractRazorEditorTest(ITestOutputHelper testOutput) : Ab protected virtual string TargetFrameworkElement => $"""{TargetFramework}"""; + protected virtual string ProjectZipFile => "Microsoft.VisualStudio.Razor.IntegrationTests.TestFiles.BlazorProject.zip"; + + private protected virtual ILogger Logger => _testLogger.AssumeNotNull(); + protected string ProjectFilePath => _projectFilePath.AssumeNotNull(); public override async Task InitializeAsync() @@ -96,19 +100,20 @@ private async Task CreateAndOpenBlazorProjectAsync(CancellationToken can var solutionPath = CreateTemporaryPath(); - var resourceName = "Microsoft.VisualStudio.Razor.IntegrationTests.TestFiles.BlazorProject.zip"; - using var zipStream = typeof(AbstractRazorEditorTest).Assembly.GetManifestResourceStream(resourceName); + using var zipStream = typeof(AbstractRazorEditorTest).Assembly.GetManifestResourceStream(ProjectZipFile); using var zip = new ZipArchive(zipStream); zip.ExtractToDirectory(solutionPath); var slnFile = Directory.EnumerateFiles(solutionPath, "*.sln").Single(); - var projectFile = Directory.EnumerateFiles(solutionPath, "*.csproj", SearchOption.AllDirectories).Single(); - PrepareProjectForFirstOpen(projectFile); + foreach (var projectFile in Directory.EnumerateFiles(solutionPath, "*.csproj", SearchOption.AllDirectories)) + { + PrepareProjectForFirstOpen(projectFile); + } await TestServices.SolutionExplorer.OpenSolutionAsync(slnFile, cancellationToken); - return projectFile; + return Directory.EnumerateFiles(solutionPath, $"{RazorProjectConstants.BlazorProjectName}.csproj", SearchOption.AllDirectories).Single(); } protected virtual void PrepareProjectForFirstOpen(string projectFileName) diff --git a/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/InProcess/EditorInProcess_Classification.cs b/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/InProcess/EditorInProcess_Classification.cs index 65a1d6bb8d4..2ec7e310e94 100644 --- a/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/InProcess/EditorInProcess_Classification.cs +++ b/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/InProcess/EditorInProcess_Classification.cs @@ -20,8 +20,10 @@ internal partial class EditorInProcess /// /// A cancellation token. /// The number of the given classification to expect. + /// Whether to wait for exactly classifications. /// A which completes when classification is "ready". - public Task WaitForComponentClassificationAsync(CancellationToken cancellationToken, int count = 1) => WaitForSemanticClassificationAsync("RazorComponentElement", cancellationToken, count); + public Task WaitForComponentClassificationAsync(CancellationToken cancellationToken, int count = 1, bool exact = false) + => WaitForSemanticClassificationAsync("RazorComponentElement", cancellationToken, count, exact); /// /// Waits for any semantic classifications to be available on the active TextView, and for at least one of the @@ -30,8 +32,9 @@ internal partial class EditorInProcess /// A cancellation token. /// The classification to wait for, if any. /// The number of the given classification to expect. + /// Whether to wait for exactly classifications. /// A which completes when classification is "ready". - public async Task WaitForSemanticClassificationAsync(string expectedClassification, CancellationToken cancellationToken, int count = 1) + public async Task WaitForSemanticClassificationAsync(string expectedClassification, CancellationToken cancellationToken, int count = 1, bool exact = false) { var textView = await TestServices.Editor.GetActiveTextViewAsync(cancellationToken); var classifier = await GetClassifierAsync(textView, cancellationToken); @@ -42,7 +45,7 @@ public async Task WaitForSemanticClassificationAsync(string expectedClassificati classifier.ClassificationChanged += Classifier_ClassificationChanged; // Check that we're not ALREADY changed - if (HasClassification(classifier, textView, expectedClassification, count)) + if (HasClassification(classifier, textView, expectedClassification, count, exact)) { semaphore.Release(); classifier.ClassificationChanged -= Classifier_ClassificationChanged; @@ -60,13 +63,13 @@ public async Task WaitForSemanticClassificationAsync(string expectedClassificati void Classifier_ClassificationChanged(object sender, ClassificationChangedEventArgs e) { - if (HasClassification(classifier, textView, expectedClassification, count)) + if (HasClassification(classifier, textView, expectedClassification, count, exact)) { semaphore.Release(); } } - static bool HasClassification(IClassifier classifier, ITextView textView, string expectedClassification, int count) + static bool HasClassification(IClassifier classifier, ITextView textView, string expectedClassification, int count, bool exact) { var classifications = GetClassifications(classifier, textView); @@ -80,7 +83,8 @@ static bool HasClassification(IClassifier classifier, ITextView textView, string } } - return found >= count; + return found == count || + (!exact && found > count); } static bool ClassificationMatches(string expectedClassification, IClassificationType classificationType) diff --git a/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/InProcess/EditorInProcess_Commands.cs b/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/InProcess/EditorInProcess_Commands.cs index c7db5bf074b..7988fe46388 100644 --- a/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/InProcess/EditorInProcess_Commands.cs +++ b/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/InProcess/EditorInProcess_Commands.cs @@ -58,7 +58,8 @@ public async Task InvokeRenameAsync(CancellationToken cancellationToken) var commandId = VSStd2KCmdID.RENAME; // Rename seems to be extra-succeptable to COM exceptions - await Helper.RetryAsync(async (cancellationToken) => { + await Helper.RetryAsync(async (cancellationToken) => + { await ExecuteCommandAsync(commandGuid, (uint)commandId, cancellationToken); return true; }, TimeSpan.FromSeconds(1), cancellationToken); @@ -69,7 +70,7 @@ public async Task CloseCodeFileAsync(string projectName, string relativeFilePath await CloseFileAsync(projectName, relativeFilePath, VSConstants.LOGVIEWID.Code_guid, saveFile, cancellationToken); } - public async Task CloseCurrentlyFocusedWindowAsync(CancellationToken cancellationToken) + public async Task CloseCurrentlyFocusedWindowAsync(CancellationToken cancellationToken, bool save = false) { await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken); @@ -77,7 +78,10 @@ public async Task CloseCurrentlyFocusedWindowAsync(CancellationToken cancellatio ErrorHandler.ThrowOnFailure(monitorSelection.GetCurrentElementValue((uint)VSSELELEMID.SEID_WindowFrame, out var windowFrameObj)); var windowFrame = (IVsWindowFrame)windowFrameObj; - ErrorHandler.ThrowOnFailure(windowFrame.CloseFrame((uint)__FRAMECLOSE.FRAMECLOSE_NoSave)); + var closeFlags = save + ? __FRAMECLOSE.FRAMECLOSE_SaveIfDirty + : __FRAMECLOSE.FRAMECLOSE_NoSave; + ErrorHandler.ThrowOnFailure(windowFrame.CloseFrame((uint)closeFlags)); } private async Task ExecuteCommandAsync(Guid commandGuid, uint commandId, CancellationToken cancellationToken) diff --git a/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/InProcess/EditorInProcess_LightBulb.cs b/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/InProcess/EditorInProcess_LightBulb.cs index eee2a507be4..58ba58d39b3 100644 --- a/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/InProcess/EditorInProcess_LightBulb.cs +++ b/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/InProcess/EditorInProcess_LightBulb.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See License.txt in the project root for license information. using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.VisualStudio.Language.Intellisense; @@ -24,6 +25,15 @@ public async Task DismissLightBulbSessionAsync(CancellationToken cancellationTok broker.DismissSession(view); } + public async Task InvokeCodeActionAsync(string codeActionTitle, CancellationToken cancellationToken) + { + var codeActions = await ShowLightBulbAsync(cancellationToken); + + var codeAction = codeActions.First(a => a.Actions.Single().DisplayText == codeActionTitle).Actions.Single(); + + await InvokeCodeActionAsync(codeAction, cancellationToken); + } + public async Task> InvokeCodeActionListAsync(CancellationToken cancellationToken) { var lightbulbs = await ShowLightBulbAsync(cancellationToken); diff --git a/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/ManualRunOnlyIdeFactAttribute.cs b/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/ManualRunOnlyIdeFactAttribute.cs new file mode 100644 index 00000000000..1df19f73f22 --- /dev/null +++ b/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/ManualRunOnlyIdeFactAttribute.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT license. See License.txt in the project root for license information. + +using System; +using Xunit; + +namespace Microsoft.VisualStudio.Razor.IntegrationTests; + +[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] +public class ManualRunOnlyIdeFactAttribute : IdeFactAttribute +{ + public ManualRunOnlyIdeFactAttribute() + { + if (Environment.GetEnvironmentVariable("_IntegrationTestsRunningInCI") is not null) + { + Skip = "This test can only run manually"; + } + } +} diff --git a/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/StressTests/AbstractStressTest.cs b/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/StressTests/AbstractStressTest.cs new file mode 100644 index 00000000000..68701bbf841 --- /dev/null +++ b/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/StressTests/AbstractStressTest.cs @@ -0,0 +1,49 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT license. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.CodeAnalysis.Razor.Logging; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.VisualStudio.Razor.IntegrationTests; + +[IdeSettings(MinVersion = VisualStudioVersion.VS2022, RootSuffix = "RoslynDev", MaxAttempts = 1)] +public abstract class AbstractStressTest(ITestOutputHelper testOutputHelper) : AbstractRazorEditorTest(testOutputHelper) +{ + protected override bool AllowDebugFails => true; + + protected Task RunStressTestAsync(Func iterationFunc) + => RunStressTestAsync(iterationFunc, TimeSpan.FromHours(1), TimeSpan.FromMinutes(1)); + + protected async Task RunStressTestAsync(Func iterationFunc, TimeSpan maxRunTime, TimeSpan iterationTimeout) + { + var min = long.MaxValue; + var max = long.MinValue; + var avg = 0L; + + var i = 0; + var start = DateTime.Now; + while (DateTime.Now.Subtract(maxRunTime) < start) + { + var iterationStart = Stopwatch.GetTimestamp(); + Logger.LogInformation($"**** Test iteration started: #{i} at {DateTime.Now}"); + + using var iterationCts = new CancellationTokenSource(iterationTimeout); + var iterationToken = iterationCts.Token; + + await iterationFunc(i, iterationToken); + i++; + + var duration = Stopwatch.GetTimestamp() - iterationStart; + min = Math.Min(min, duration); + max = Math.Max(max, duration); + avg = ((avg * (i - 1)) + duration) / i; + Logger.LogInformation($"**** Test iteration finished: #{i} in {TimeSpan.FromTicks(duration).TotalMilliseconds}ms"); + Logger.LogInformation($"**** Test iteration duration: min={TimeSpan.FromTicks(min).TotalMilliseconds}ms, max={TimeSpan.FromTicks(max).TotalMilliseconds}ms, avg={TimeSpan.FromTicks(avg).TotalMilliseconds}ms"); + } + } +} diff --git a/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/StressTests/RCLStressTests.cs b/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/StressTests/RCLStressTests.cs new file mode 100644 index 00000000000..c5ed75e06bf --- /dev/null +++ b/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/StressTests/RCLStressTests.cs @@ -0,0 +1,82 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT license. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.WebTools.Languages.Shared.Editor.EditorHelpers; +using Xunit.Abstractions; + +namespace Microsoft.VisualStudio.Razor.IntegrationTests; + +public class RCLStressTests(ITestOutputHelper testOutputHelper) : AbstractStressTest(testOutputHelper) +{ + protected override string TargetFramework => "net9.0"; + + protected override string ProjectZipFile => "Microsoft.VisualStudio.Razor.IntegrationTests.TestFiles.BlazorProjectWithRCL.zip"; + + [ManualRunOnlyIdeFact] + public async Task AddAndRemoveComponentInRCL() + { + await TestServices.SolutionExplorer.OpenFileAsync("RazorClassLibrary", @"Components\RCLComponent.razor", ControlledHangMitigatingCancellationToken); + + await TestServices.Editor.PlaceCaretAsync("Iteration {index}{Environment.NewLine}", cancellationToken); + + await TestServices.Editor.PlaceCaretAsync("h1", charsOffset: -1, cancellationToken); + + await TestServices.Editor.WaitForComponentClassificationAsync(cancellationToken, count: 1, exact: true); + + await TestServices.Editor.InvokeCodeActionAsync("Extract element to new component", cancellationToken); + + await TestServices.Editor.WaitForActiveWindowByFileAsync("Component.razor", cancellationToken); + + await TestServices.Editor.PlaceCaretAsync("{Environment.NewLine}", cancellationToken); + + await TestServices.Editor.WaitForComponentClassificationAsync(cancellationToken, count: 5, exact: true); + + File.Delete(componentFileName); + + await TestServices.Editor.WaitForComponentClassificationAsync(cancellationToken, count: 4, exact: true); + + await TestServices.Editor.PlaceCaretAsync("", charsOffset: -1, cancellationToken); + + await TestServices.Editor.InvokeDeleteLineAsync(cancellationToken); + + await TestServices.Editor.InsertTextAsync($"

Iteration {index}

{Environment.NewLine}", cancellationToken); + + await TestServices.SolutionExplorer.OpenFileAsync("RazorClassLibrary", @"Components\RCLComponent.razor", ControlledHangMitigatingCancellationToken); + + await TestServices.Editor.WaitForComponentClassificationAsync(cancellationToken, count: 1, exact: true); + + await TestServices.Editor.PlaceCaretAsync("", charsOffset: -1, cancellationToken); + + await TestServices.Editor.InvokeDeleteLineAsync(cancellationToken); + } + } +} diff --git a/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/StressTests/StressTests.cs b/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/StressTests/StressTests.cs new file mode 100644 index 00000000000..dcf91f27ad4 --- /dev/null +++ b/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/StressTests/StressTests.cs @@ -0,0 +1,59 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT license. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.WebTools.Languages.Shared.Editor.EditorHelpers; +using Xunit.Abstractions; + +namespace Microsoft.VisualStudio.Razor.IntegrationTests; + +public class StressTests(ITestOutputHelper testOutputHelper) : AbstractStressTest(testOutputHelper) +{ + [ManualRunOnlyIdeFact] + public async Task AddAndRemoveComponent() + { + await TestServices.SolutionExplorer.OpenFileAsync(RazorProjectConstants.BlazorProjectName, RazorProjectConstants.CounterRazorFile, ControlledHangMitigatingCancellationToken); + + await TestServices.Editor.PlaceCaretAsync("h1", charsOffset: -1, ControlledHangMitigatingCancellationToken); + + await TestServices.Editor.InvokeDeleteLineAsync(ControlledHangMitigatingCancellationToken); + + await RunStressTestAsync(RunIterationAsync); + + async Task RunIterationAsync(int index, CancellationToken cancellationToken) + { + await TestServices.Editor.InsertTextAsync($"

Iteration {index}

{Environment.NewLine}", cancellationToken); + + await TestServices.Editor.PlaceCaretAsync("h1", charsOffset: -1, cancellationToken); + + await TestServices.Editor.WaitForComponentClassificationAsync(cancellationToken, count: 2, exact: true); + + await TestServices.Editor.InvokeCodeActionAsync("Extract element to new component", cancellationToken); + + await TestServices.Editor.WaitForActiveWindowByFileAsync("Component.razor", cancellationToken); + + var componentFileName = (await TestServices.Editor.GetActiveTextViewAsync(cancellationToken)).TextBuffer.GetFileName(); + + await TestServices.Editor.CloseCurrentlyFocusedWindowAsync(cancellationToken, save: true); + + await TestServices.Editor.WaitForActiveWindowByFileAsync("Counter.razor", cancellationToken); + + await TestServices.Editor.WaitForComponentClassificationAsync(cancellationToken, count: 3, exact: true); + + await Task.Delay(500); + + File.Delete(componentFileName); + + await TestServices.Editor.WaitForComponentClassificationAsync(cancellationToken, count: 2, exact: true); + + await Task.Delay(500); + + await TestServices.Editor.PlaceCaretAsync("Component", charsOffset: -1, cancellationToken); + + await TestServices.Editor.InvokeDeleteLineAsync(cancellationToken); + } + } +} diff --git a/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/TestFiles/BlazorProjectWithRCL.zip b/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/TestFiles/BlazorProjectWithRCL.zip new file mode 100644 index 00000000000..f05439c499d Binary files /dev/null and b/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/TestFiles/BlazorProjectWithRCL.zip differ