diff --git a/src/Dotnet.Watch/Watch/HotReload/CompilationHandler.cs b/src/Dotnet.Watch/Watch/HotReload/CompilationHandler.cs index 91d14e2bc234..85e120eafafc 100644 --- a/src/Dotnet.Watch/Watch/HotReload/CompilationHandler.cs +++ b/src/Dotnet.Watch/Watch/HotReload/CompilationHandler.cs @@ -22,6 +22,7 @@ internal sealed class CompilationHandler : IDisposable /// /// Lock to synchronize: /// + /// /// /// private readonly object _runningProjectsAndUpdatesGuard = new(); @@ -33,6 +34,16 @@ internal sealed class CompilationHandler : IDisposable private ImmutableDictionary> _runningProjects = ImmutableDictionary>.Empty.WithComparers(PathUtilities.OSSpecificPathComparer); + /// + /// Maps to the list of active restart operations for the project. + /// The of the project instance is added whenever a process crashes (terminated with non-zero exit code) + /// and the corresponding is removed from . + /// + /// When a file change is observed whose containing project is listed here, the associated relaunch operations are executed. + /// + private ImmutableDictionary> _activeProjectRelaunchOperations + = ImmutableDictionary>.Empty.WithComparers(PathUtilities.OSSpecificPathComparer); + /// /// All updates that were attempted. Includes updates whose application failed. /// @@ -145,10 +156,19 @@ public async ValueTask StartSessionAsync(CancellationToken cancellationToken) await previousOnExit(processId, exitCode); } - // Remove the running project if it has been published to _runningProjects (if it hasn't exited during initialization): - if (publishedRunningProject != null && RemoveRunningProject(publishedRunningProject)) + if (publishedRunningProject != null) { - await publishedRunningProject.DisposeAsync(isExiting: true); + var relaunch = + !cancellationToken.IsCancellationRequested && + !publishedRunningProject.Options.IsMainProject && + exitCode.HasValue && + exitCode.Value != 0; + + // Remove the running project if it has been published to _runningProjects (if it hasn't exited during initialization): + if (RemoveRunningProject(publishedRunningProject, relaunch)) + { + await publishedRunningProject.DisposeAsync(isExiting: true); + } } }; @@ -222,12 +242,7 @@ await await clients.ApplyManagedCodeUpdatesAsync( // Only add the running process after it has been up-to-date. // This will prevent new updates being applied before we have applied all the previous updates. - if (!_runningProjects.TryGetValue(projectPath, out var projectInstances)) - { - projectInstances = []; - } - - _runningProjects = _runningProjects.SetItem(projectPath, projectInstances.Add(runningProject)); + _runningProjects = _runningProjects.Add(projectPath, runningProject); // transfer ownership to _runningProjects publishedRunningProject = runningProject; @@ -390,26 +405,59 @@ public async ValueTask GetManagedCodeUpdatesAsync( } } - public async ValueTask ApplyManagedCodeAndStaticAssetUpdatesAsync( + public async ValueTask ApplyManagedCodeAndStaticAssetUpdatesAndRelaunchAsync( IReadOnlyList managedCodeUpdates, IReadOnlyDictionary> staticAssetUpdates, + ImmutableArray changedFiles, + LoadedProjectGraph projectGraph, Stopwatch stopwatch, CancellationToken cancellationToken) { var applyTasks = new List(); ImmutableDictionary> projectsToUpdate = []; - if (managedCodeUpdates is not []) + IReadOnlyList relaunchOperations; + lock (_runningProjectsAndUpdatesGuard) { - lock (_runningProjectsAndUpdatesGuard) - { - // Adding the updates makes sure that all new processes receive them before they are added to running processes. - _previousUpdates = _previousUpdates.AddRange(managedCodeUpdates); + // Adding the updates makes sure that all new processes receive them before they are added to running processes. + _previousUpdates = _previousUpdates.AddRange(managedCodeUpdates); - // Capture the set of processes that do not have the currently calculated deltas yet. - projectsToUpdate = _runningProjects; - } + // Capture the set of processes that do not have the currently calculated deltas yet. + projectsToUpdate = _runningProjects; + + // Determine relaunch operations at the same time as we capture running processes, + // so that these sets are consistent even if another process crashes while doing so. + relaunchOperations = GetRelaunchOperations_NoLock(changedFiles, projectGraph); + } + + // Relaunch projects after _previousUpdates were updated above. + // Ensures that the current and previous updates will be applied as initial updates to the newly launched processes. + // We also capture _runningProjects above, before launching new ones, so that the current updates are not applied twice to the relaunched processes. + // Static asset changes do not need to be updated in the newly launched processes since the application will read their updated content once it launches. + // Fire and forget. + foreach (var relaunchOperation in relaunchOperations) + { + // fire and forget: + _ = Task.Run(async () => + { + try + { + await relaunchOperation.Invoke(cancellationToken); + } + catch (OperationCanceledException) + { + // nop + } + catch (Exception e) + { + // Handle all exceptions since this is a fire-and-forget task. + _context.Logger.LogError("Failed to relaunch: {Exception}", e.ToString()); + } + }, cancellationToken); + } + if (managedCodeUpdates is not []) + { // Apply changes to all running projects, even if they do not have a static project dependency on any project that changed. // The process may load any of the binaries using MEF or some other runtime dependency loader. @@ -470,14 +518,14 @@ async Task CompleteApplyOperationAsync() projectsToUpdate.Select(e => e.Value.First().Options.Representation).Concat( staticAssetUpdates.Select(e => e.Key.Options.Representation))); } + catch (OperationCanceledException) + { + // nop + } catch (Exception e) { // Handle all exceptions since this is a fire-and-forget task. - - if (e is not OperationCanceledException) - { - _context.Logger.LogError("Failed to apply managedCodeUpdates: {Exception}", e.ToString()); - } + _context.Logger.LogError("Failed to apply managedCodeUpdates: {Exception}", e.ToString()); } } } @@ -531,8 +579,7 @@ private async ValueTask DisplayResultsAsync(HotReloadService.Updates updates, Im ReportRudeEdits(); // report or clear diagnostics in the browser UI - await ForEachProjectAsync( - _runningProjects, + await _runningProjects.ForEachValueAsync( (project, cancellationToken) => project.Clients.ReportCompilationErrorsInApplicationAsync([.. errorsToDisplayInApp], cancellationToken).AsTask() ?? Task.CompletedTask, cancellationToken); @@ -865,37 +912,74 @@ await Task.WhenAll( _context.Logger.Log(MessageDescriptor.ProjectsRestarted, projectsToRestart.Count); } - private bool RemoveRunningProject(RunningProject project) + private bool RemoveRunningProject(RunningProject project, bool relaunch) { var projectPath = project.ProjectNode.ProjectInstance.FullPath; - return UpdateRunningProjects(runningProjectsByPath => + lock (_runningProjectsAndUpdatesGuard) { - if (!runningProjectsByPath.TryGetValue(projectPath, out var runningInstances)) + var newRunningProjects = _runningProjects.Remove(projectPath, project); + if (newRunningProjects == _runningProjects) + { + return false; + } + + if (relaunch) { - return runningProjectsByPath; + // Create re-launch operation for each instance that crashed + // even if other instances of the project are still running. + _activeProjectRelaunchOperations = _activeProjectRelaunchOperations.Add(projectPath, project.GetRelaunchOperation()); } - var updatedRunningProjects = runningInstances.Remove(project); - return updatedRunningProjects is [] - ? runningProjectsByPath.Remove(projectPath) - : runningProjectsByPath.SetItem(projectPath, updatedRunningProjects); - }); + _runningProjects = newRunningProjects; + } + + if (relaunch) + { + project.ClientLogger.Log(MessageDescriptor.ProcessCrashedAndWillBeRelaunched); + } + + return true; } - private bool UpdateRunningProjects(Func>, ImmutableDictionary>> updater) + private IReadOnlyList GetRelaunchOperations_NoLock(IReadOnlyList changedFiles, LoadedProjectGraph projectGraph) { - lock (_runningProjectsAndUpdatesGuard) + if (_activeProjectRelaunchOperations.IsEmpty) { - var newRunningProjects = updater(_runningProjects); - if (newRunningProjects != _runningProjects) + return []; + } + + var relaunchOperations = new List(); + foreach (var changedFile in changedFiles) + { + foreach (var containingProjectPath in changedFile.Item.ContainingProjectPaths) { - _runningProjects = newRunningProjects; - return true; - } + if (!projectGraph.Map.TryGetValue(containingProjectPath, out var containingProjectNodes)) + { + // Shouldn't happen. + Logger.LogWarning("Project '{Path}' not found in the project graph.", containingProjectPath); + continue; + } + + // Relaunch all projects whose dependency is affected by this file change. + foreach (var ancestor in containingProjectNodes[0].GetAncestorsAndSelf()) + { + var ancestorPath = ancestor.ProjectInstance.FullPath; + if (_activeProjectRelaunchOperations.TryGetValue(ancestorPath, out var operations)) + { + relaunchOperations.AddRange(operations); + _activeProjectRelaunchOperations = _activeProjectRelaunchOperations.Remove(ancestorPath); - return false; + if (_activeProjectRelaunchOperations.IsEmpty) + { + break; + } + } + } + } } + + return relaunchOperations; } public bool TryGetRunningProject(string projectPath, out ImmutableArray projects) @@ -906,9 +990,6 @@ public bool TryGetRunningProject(string projectPath, out ImmutableArray> projects, Func action, CancellationToken cancellationToken) - => Task.WhenAll(projects.SelectMany(entry => entry.Value).Select(project => action(project, cancellationToken))).WaitAsync(cancellationToken); - private static ImmutableArray ToManagedCodeUpdates(IEnumerable updates) => [.. updates.Select(update => new HotReloadManagedCodeUpdate(update.ModuleId, update.MetadataDelta, update.ILDelta, update.PdbDelta, update.UpdatedTypes, update.RequiredCapabilities))]; diff --git a/src/Dotnet.Watch/Watch/HotReload/HotReloadDotNetWatcher.cs b/src/Dotnet.Watch/Watch/HotReload/HotReloadDotNetWatcher.cs index 8f82392d5872..92ce3027396f 100644 --- a/src/Dotnet.Watch/Watch/HotReload/HotReloadDotNetWatcher.cs +++ b/src/Dotnet.Watch/Watch/HotReload/HotReloadDotNetWatcher.cs @@ -79,6 +79,7 @@ public async Task WatchAsync(CancellationToken shutdownCancellationToken) using var iterationCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(shutdownCancellationToken, forceRestartCancellationSource.Token); var iterationCancellationToken = iterationCancellationSource.Token; + var suppressWaitForFileChange = false; EvaluationResult? evaluationResult = null; RunningProject? mainRunningProject = null; IRuntimeProcessLauncher? runtimeProcessLauncher = null; @@ -280,11 +281,7 @@ await compilationHandler.GetManagedCodeUpdatesAsync( // Apply updates only after dependencies have been deployed, // so that updated code doesn't attempt to access the dependency before it has been deployed. - if (updates.ManagedCodeUpdates.Count > 0 || updates.StaticAssetsToUpdate.Count > 0) - { - await compilationHandler.ApplyManagedCodeAndStaticAssetUpdatesAsync(updates.ManagedCodeUpdates, updates.StaticAssetsToUpdate, stopwatch, iterationCancellationToken); - } - + await compilationHandler.ApplyManagedCodeAndStaticAssetUpdatesAndRelaunchAsync(updates.ManagedCodeUpdates, updates.StaticAssetsToUpdate, changedFiles, evaluationResult.ProjectGraph, stopwatch, iterationCancellationToken); if (updates.ProjectsToRestart is not []) { await compilationHandler.RestartPeripheralProjectsAsync(updates.ProjectsToRestart, shutdownCancellationToken); @@ -400,6 +397,10 @@ async Task> CaptureChangedFilesSnapshot(IReadOnlyLis { // start next iteration unless shutdown is requested } + catch (Exception) when (!(suppressWaitForFileChange = true)) + { + // unreachable + } finally { // stop watching file changes: @@ -438,7 +439,7 @@ async Task> CaptureChangedFilesSnapshot(IReadOnlyLis { _context.Logger.Log(MessageDescriptor.Restarting); } - else if (mainRunningProject?.IsRestarting != true) + else if (mainRunningProject?.IsRestarting != true && !suppressWaitForFileChange) { using var shutdownOrForcedRestartSource = CancellationTokenSource.CreateLinkedTokenSource(shutdownCancellationToken, forceRestartCancellationSource.Token); await WaitForFileChangeBeforeRestarting(fileWatcher, evaluationResult, shutdownOrForcedRestartSource.Token); diff --git a/src/Dotnet.Watch/Watch/Process/RunningProject.cs b/src/Dotnet.Watch/Watch/Process/RunningProject.cs index ca1498aecbc5..c39dd4e541ae 100644 --- a/src/Dotnet.Watch/Watch/Process/RunningProject.cs +++ b/src/Dotnet.Watch/Watch/Process/RunningProject.cs @@ -94,5 +94,13 @@ public async ValueTask RestartAsync(CancellationToken cancellationToken) await restartOperation(cancellationToken); ClientLogger.Log(MessageDescriptor.ProjectRestarted); } + + public RestartOperation GetRelaunchOperation() + => new(async cancellationToken => + { + ClientLogger.Log(MessageDescriptor.ProjectRelaunching); + await restartOperation(cancellationToken); + ClientLogger.Log(MessageDescriptor.ProjectRelaunched); + }); } } diff --git a/src/Dotnet.Watch/Watch/UI/IReporter.cs b/src/Dotnet.Watch/Watch/UI/IReporter.cs index 4e2c7a0f9359..09bcb2feba1f 100644 --- a/src/Dotnet.Watch/Watch/UI/IReporter.cs +++ b/src/Dotnet.Watch/Watch/UI/IReporter.cs @@ -191,6 +191,9 @@ public static MessageDescriptor GetDescriptor(EventId id) public static readonly MessageDescriptor> RestartingProjectsNotification = CreateNotification>(); public static readonly MessageDescriptor ProjectRestarting = Create("Restarting ...", Emoji.Watch, LogLevel.Debug); public static readonly MessageDescriptor ProjectRestarted = Create("Restarted", Emoji.Watch, LogLevel.Debug); + public static readonly MessageDescriptor ProjectRelaunching = Create("Relaunching ...", Emoji.Watch, LogLevel.Information); + public static readonly MessageDescriptor ProjectRelaunched = Create("Relaunched", Emoji.Watch, LogLevel.Debug); + public static readonly MessageDescriptor ProcessCrashedAndWillBeRelaunched = Create("Process crashed and will be relaunched on file change", Emoji.Watch, LogLevel.Debug); public static readonly MessageDescriptor ProjectDependenciesDeployed = Create("Project dependencies deployed ({0})", Emoji.HotReload, LogLevel.Debug); public static readonly MessageDescriptor FixBuildError = Create("Fix the error to continue or press Ctrl+C to exit.", Emoji.Watch, LogLevel.Warning); public static readonly MessageDescriptor WaitingForChanges = Create("Waiting for changes", Emoji.Watch, LogLevel.Information); diff --git a/src/Dotnet.Watch/Watch/Utilities/ImmutableDictionaryExtensions.cs b/src/Dotnet.Watch/Watch/Utilities/ImmutableDictionaryExtensions.cs new file mode 100644 index 000000000000..fc37c72eb732 --- /dev/null +++ b/src/Dotnet.Watch/Watch/Utilities/ImmutableDictionaryExtensions.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; + +namespace Microsoft.DotNet.Watch; + +internal static class ImmutableDictionaryExtensions +{ + public static ImmutableDictionary> Add(this ImmutableDictionary> dictionary, TKey key, TValue value) + where TKey : notnull + { + if (!dictionary.TryGetValue(key, out var items)) + { + items = []; + } + + return dictionary.SetItem(key, items.Add(value)); + } + + public static ImmutableDictionary> Remove(this ImmutableDictionary> dictionary, TKey key, TValue value) + where TKey : notnull + { + if (!dictionary.TryGetValue(key, out var items)) + { + return dictionary; + } + + var updatedItems = items.Remove(value); + if (items == updatedItems) + { + return dictionary; + } + + return updatedItems is [] + ? dictionary.Remove(key) + : dictionary.SetItem(key, updatedItems); + } + + public static Task ForEachValueAsync(this ImmutableDictionary> dictionary, Func action, CancellationToken cancellationToken) + where TKey : notnull + => Task.WhenAll(dictionary.SelectMany(entry => entry.Value).Select(project => action(project, cancellationToken))).WaitAsync(cancellationToken); +} diff --git a/src/Dotnet.Watch/dotnet-watch/Properties/launchSettings.json b/src/Dotnet.Watch/dotnet-watch/Properties/launchSettings.json index b60b37dc0855..0ff2c6a27f47 100644 --- a/src/Dotnet.Watch/dotnet-watch/Properties/launchSettings.json +++ b/src/Dotnet.Watch/dotnet-watch/Properties/launchSettings.json @@ -3,7 +3,7 @@ "dotnet-watch": { "commandName": "Project", "commandLineArgs": "-bl", - "workingDirectory": "$(RepoRoot)src\\Assets\\TestProjects\\BlazorWasmWithLibrary\\blazorwasm", + "workingDirectory": "D:\\Temp\\app2", "environmentVariables": { "DOTNET_WATCH_DEBUG_SDK_DIRECTORY": "$(RepoRoot)artifacts\\bin\\redist\\$(Configuration)\\dotnet\\sdk\\$(Version)", "DCP_IDE_REQUEST_TIMEOUT_SECONDS": "100000", diff --git a/test/dotnet-watch.Tests/HotReload/RuntimeProcessLauncherTests.cs b/test/dotnet-watch.Tests/HotReload/RuntimeProcessLauncherTests.cs index f5ea15cb1218..59ca84b94159 100644 --- a/test/dotnet-watch.Tests/HotReload/RuntimeProcessLauncherTests.cs +++ b/test/dotnet-watch.Tests/HotReload/RuntimeProcessLauncherTests.cs @@ -193,7 +193,6 @@ public async Task UpdateAppliedToNewProcesses(bool sharedOutput) var serviceDirB = Path.Combine(testAsset.Path, "ServiceB"); var serviceProjectB = Path.Combine(serviceDirB, "B.csproj"); var libDir = Path.Combine(testAsset.Path, "Lib"); - var libProject = Path.Combine(libDir, "Lib.csproj"); var libSource = Path.Combine(libDir, "Lib.cs"); await using var w = CreateInProcWatcher(testAsset, ["--non-interactive", "--project", hostProject], workingDirectory); @@ -285,7 +284,6 @@ public async Task HostRestart(UpdateLocation updateLocation) var hostDir = Path.Combine(testAsset.Path, "Host"); var hostProject = Path.Combine(hostDir, "Host.csproj"); var hostProgram = Path.Combine(hostDir, "Program.cs"); - var libProject = Path.Combine(testAsset.Path, "Lib2", "Lib2.csproj"); var lib = Path.Combine(testAsset.Path, "Lib2", "Lib2.cs"); await using var w = CreateInProcWatcher(testAsset, args: ["--project", hostProject], workingDirectory); @@ -412,4 +410,82 @@ public async Task RudeEditInProjectWithoutRunningProcess() Log("Waiting for verbose rude edit reported ..."); await applyUpdateVerbose.WaitAsync(w.ShutdownSource.Token); } + + [Fact] + public async Task RelaunchOnCrash() + { + var testAsset = CopyTestAsset("WatchAppMultiProc"); + + var workingDirectory = testAsset.Path; + var hostDir = Path.Combine(testAsset.Path, "Host"); + var hostProject = Path.Combine(hostDir, "Host.csproj"); + var serviceDirA = Path.Combine(testAsset.Path, "ServiceA"); + var serviceProjectA = Path.Combine(serviceDirA, "A.csproj"); + var libDir = Path.Combine(testAsset.Path, "Lib"); + var libSource = Path.Combine(libDir, "Lib.cs"); + + await using var w = CreateInProcWatcher(testAsset, ["--project", hostProject], workingDirectory); + + var waitingForChanges = w.Observer.RegisterSemaphore(MessageDescriptor.WaitingForChanges); + var processCrashedAndWillBeRelaunched = w.Observer.RegisterSemaphore(MessageDescriptor.ProcessCrashedAndWillBeRelaunched); + var projectRelaunched = w.Observer.RegisterSemaphore(MessageDescriptor.ProjectRelaunched); + + var hasCrashed = new SemaphoreSlim(initialCount: 0); + var hasUpdate = new SemaphoreSlim(initialCount: 0); + w.Reporter.OnProcessOutput += line => + { + if (line.Content.Contains("")) + { + hasCrashed.Release(); + } + else if (line.Content.Contains("")) + { + hasUpdate.Release(); + } + }; + + w.Start(); + + // let the host process start: + Log("Waiting for changes..."); + await waitingForChanges.WaitAsync(w.ShutdownSource.Token); + + // service should have been created before Hot Reload session started: + Assert.NotNull(w.Service); + + await w.Service.Launch(serviceProjectA, workingDirectory, w.ShutdownSource.Token); + + UpdateSourceFile(libSource, """ + using System; + + public class Lib + { + public static void Common() + => throw new Exception(""); + } + """); + + Log("Waiting in output ..."); + await hasCrashed.WaitAsync(w.ShutdownSource.Token); + + Log("Waiting for process crashed ..."); + await processCrashedAndWillBeRelaunched.WaitAsync(w.ShutdownSource.Token); + + // file change triggers relaunch: + UpdateSourceFile(libSource,""" + using System; + + public class Lib + { + public static void Common() + => Console.WriteLine(""); + } + """); + + Log("Waiting for A to relaunch ..."); + await projectRelaunched.WaitAsync(w.ShutdownSource.Token); + + Log("Waiting for in output ..."); + await hasUpdate.WaitAsync(w.ShutdownSource.Token); + } }