Add the ability to stop the Host when one of its BackgroundService instances errors

src/libraries/Microsoft.Extensions.Hosting.Abstractions/ref/Microsoft.Extensions.Hosting.Abstractions.cs

@@ -23,6 +23,11 @@ public virtual void Dispose() { }
         public virtual System.Threading.Tasks.Task StartAsync(System.Threading.CancellationToken cancellationToken) { throw null; }
         public virtual System.Threading.Tasks.Task StopAsync(System.Threading.CancellationToken cancellationToken) { throw null; }
     }
+    public enum BackgroundServiceExceptionBehavior
+    {
+        Ignore,
+        StopHost
+    }
     [System.ObsoleteAttribute("This type is obsolete and will be removed in a future version. The recommended alternative is Microsoft.Extensions.Hosting.Environments.", false)]
     public static partial class EnvironmentName
     {

src/libraries/Microsoft.Extensions.Hosting.Abstractions/src/BackgroundService.cs

@@ -75,7 +75,6 @@ public virtual async Task StopAsync(CancellationToken cancellationToken)
                 // Wait until the task completes or the stop token triggers
                 await Task.WhenAny(_executeTask, Task.Delay(Timeout.Infinite, cancellationToken)).ConfigureAwait(false);
             }
-
         }
 
         public virtual void Dispose()

src/libraries/Microsoft.Extensions.Hosting.Abstractions/src/BackgroundServiceExceptionBehavior.cs

@@ -0,0 +1,29 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Microsoft.Extensions.Hosting
+{
+    /// <summary>
+    /// Specifies a behavior that the <see cref="IHost"/> will honor if an
+    /// unhandled exception occurs in one of its <see cref="BackgroundService"/> instances.
+    /// </summary>
+    public enum BackgroundServiceExceptionBehavior
+    {
+        /// <summary>
+        /// Ignore exceptions thrown in <see cref="BackgroundService"/>.
+        /// </summary>
+        /// <remarks>
+        /// If a <see cref="BackgroundService"/> throws an exception, the <see cref="IHost"/> will ignore it.
+        /// The <see cref="BackgroundService"/> is not restarted.
+        /// </remarks>
+        Ignore = 0,
+
+        /// <summary>
+        /// Stops the <see cref="IHost"/> instance, and terminates the process.
+        /// </summary>
+        /// <remarks>
+        /// If a <see cref="BackgroundService"/> throws an exception, the <see cref="IHost"/> will terminate the process.
+        /// </remarks>
+        StopHost = 1
+    }
+}

src/libraries/Microsoft.Extensions.Hosting/ref/Microsoft.Extensions.Hosting.cs

@@ -56,6 +56,7 @@ public partial class HostOptions
     {
         public HostOptions() { }
         public System.TimeSpan ShutdownTimeout { get { throw null; } set { } }
+        public Microsoft.Extensions.Hosting.BackgroundServiceExceptionBehavior BackgroundServiceExceptionBehavior { get { throw null; } set { } }
     }
 }
 namespace Microsoft.Extensions.Hosting.Internal

src/libraries/Microsoft.Extensions.Hosting/src/HostOptions.cs

@@ -17,6 +17,12 @@ public class HostOptions
         /// </summary>
         public TimeSpan ShutdownTimeout { get; set; } = TimeSpan.FromSeconds(5);
 
+        /// <summary>
+        /// The behavior the <see cref="IHost"/> will follow when any of
+        /// its <see cref="BackgroundService"/> instances throw an unhandled exception.
+        /// </summary>
+        public BackgroundServiceExceptionBehavior BackgroundServiceExceptionBehavior { get; set; }
+
         internal void Initialize(IConfiguration configuration)
         {
             var timeoutSeconds = configuration["shutdownTimeoutSeconds"];

src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs

@@ -61,12 +61,15 @@ public async Task StartAsync(CancellationToken cancellationToken = default)
 
             foreach (IHostedService hostedService in _hostedServices)
             {
-                // Fire IHostedService.Start
-                await hostedService.StartAsync(combinedCancellationToken).ConfigureAwait(false);
-
                 if (hostedService is BackgroundService backgroundService)
                 {
-                    _ = HandleBackgroundException(backgroundService);
+                    _ = TryStartBackgroundServiceAsync(
+                        backgroundService, combinedCancellationToken);
+                }
+                else
+                {
+                    // Fire IHostedService.Start
+                    await hostedService.StartAsync(combinedCancellationToken).ConfigureAwait(false);
                 }
             }
 
@@ -76,15 +79,23 @@ public async Task StartAsync(CancellationToken cancellationToken = default)
             _logger.Started();
         }
 
-        private async Task HandleBackgroundException(BackgroundService backgroundService)
+        private async Task TryStartBackgroundServiceAsync(
+            BackgroundService backgroundService, CancellationToken cancellationToken)
         {
             try
             {
+                // Fire IHostedService.Start
+                await backgroundService.StartAsync(cancellationToken).ConfigureAwait(false);
                 await backgroundService.ExecuteTask.ConfigureAwait(false);
             }
             catch (Exception ex)
             {
                 _logger.BackgroundServiceFaulted(ex);
+                if (_options is { BackgroundServiceExceptionBehavior: BackgroundServiceExceptionBehavior.StopHost })
+                {
+                    Environment.FailFast(
+                      SR.Format(SR.BackgroundServiceExceptionStoppedHost, ex));
+                }
             }
         }
 

src/libraries/Microsoft.Extensions.Hosting/src/Resources/Strings.resx

@@ -2,16 +2,16 @@
 <root>
   <!-- 
     Microsoft ResX Schema 
-
+    
     Version 2.0
-
-    The primary goals of this format is to allow a simple XML format
+    
+    The primary goals of this format is to allow a simple XML format 
     that is mostly human readable. The generation and parsing of the 
     various data types are done through the TypeConverter classes 
     associated with the data types.
-
+    
     Example:
-
+    
     ... ado.net/XML headers & schema ...
     <resheader name="resmimetype">text/microsoft-resx</resheader>
     <resheader name="version">2.0</resheader>
@@ -26,31 +26,31 @@
         <value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
         <comment>This is a comment</comment>
     </data>
-
-    There are any number of "resheader" rows that contain simple
+                
+    There are any number of "resheader" rows that contain simple 
     name/value pairs.
-
-    Each data row contains a name, and value. The row also contains a
-    type or mimetype. Type corresponds to a .NET class that support
-    text/value conversion through the TypeConverter architecture.
-    Classes that don't support this are serialized and stored with the
+    
+    Each data row contains a name, and value. The row also contains a 
+    type or mimetype. Type corresponds to a .NET class that support 
+    text/value conversion through the TypeConverter architecture. 
+    Classes that don't support this are serialized and stored with the 
     mimetype set.
-
-    The mimetype is used for serialized objects, and tells the
-    ResXResourceReader how to depersist the object. This is currently not
+    
+    The mimetype is used for serialized objects, and tells the 
+    ResXResourceReader how to depersist the object. This is currently not 
     extensible. For a given mimetype the value must be set accordingly:
-
-    Note - application/x-microsoft.net.object.binary.base64 is the format
-    that the ResXResourceWriter will generate, however the reader can
+    
+    Note - application/x-microsoft.net.object.binary.base64 is the format 
+    that the ResXResourceWriter will generate, however the reader can 
     read any of the formats listed below.
-
+    
     mimetype: application/x-microsoft.net.object.binary.base64
-    value   : The object must be serialized with
+    value   : The object must be serialized with 
             : System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
             : and then encoded with base64 encoding.
-
+    
     mimetype: application/x-microsoft.net.object.soap.base64
-    value   : The object must be serialized with
+    value   : The object must be serialized with 
             : System.Runtime.Serialization.Formatters.Soap.SoapFormatter
             : and then encoded with base64 encoding.
 
@@ -117,6 +117,11 @@
   <resheader name="writer">
     <value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
   </resheader>
+  <data name="BackgroundServiceExceptionStoppedHost" xml:space="preserve">
+    <value>The HostOptions.BackgroundServiceExceptionBehavior is configured to StopHost. A BackgroundService has thrown an unhandled exception, and the IHost instance is stopping. To avoid this behavior, configure this to Ignore; however the BackgroundService will not be restarted.
+
+Error: {0}</value>
+  </data>
   <data name="BuildCalled" xml:space="preserve">
     <value>Build can only be called once.</value>
   </data>

src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/HostTests.cs

@@ -145,7 +145,7 @@ public void CreateDefaultBuilder_EnablesValidateOnBuild()
         [ActiveIssue("https://github.com/dotnet/runtime/issues/34580", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
         public async Task CreateDefaultBuilder_ConfigJsonDoesNotReload()
         {
-            var reloadFlagConfig = new Dictionary<string, string>() {{ "hostbuilder:reloadConfigOnChange", "false" }};
+            var reloadFlagConfig = new Dictionary<string, string>() { { "hostbuilder:reloadConfigOnChange", "false" } };
             var appSettingsPath = Path.Combine(Path.GetTempPath(), "appsettings.json");
 
             string SaveRandomConfig()

src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/Internal/HostTests.cs

@@ -8,6 +8,7 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.DotNet.RemoteExecutor;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting.Fakes;
@@ -657,6 +658,91 @@ public async Task WebHostStopAsyncUsesDefaultTimeoutIfNoTokenProvided()
             }
         }
 
+        [Fact]
+        public void HostPropagatesExceptionsThrownWithBackgroundServiceExceptionBehaviorOfStopHost()
+        {
+            RemoteExecutor.Invoke(async () =>
+            {
+                using IHost host = CreateBuilder()
+                .ConfigureServices(
+                    services =>
+                    {
+                        services.AddHostedService<AsyncThrowingService>();
+                        services.Configure<HostOptions>(
+                          options =>
+                            options.BackgroundServiceExceptionBehavior =
+                                BackgroundServiceExceptionBehavior.StopHost);
+                    })
+                .Build();
+
+                await host.StartAsync();
+
+                // The following line should not be called, if it is - the test should fail.
+                Assert.True(false, $"The {nameof(AsyncThrowingService)} should have thrown, stopping the Host.");
+            }).Dispose();
+        }
+
+        [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))]
+        public void HostPropagatesExceptionsThrownWithBackgroundServiceExceptionBehaviorOfStopHostAfterSometime()
+        {
+            RemoteExecutor.Invoke(async () =>
+            {
+                var backgroundDelayTaskSource = new TaskCompletionSource<bool>();
+
+                using IHost host = CreateBuilder()
+                    .ConfigureServices(
+                        services =>
+                        {
+                            services.AddHostedService(
+                                _ => new AsyncThrowingService(backgroundDelayTaskSource.Task));
+
+                            services.Configure<HostOptions>(
+                              options =>
+                                options.BackgroundServiceExceptionBehavior =
+                                    BackgroundServiceExceptionBehavior.StopHost);
+                        })
+                    .Build();
+
+                await Task.WhenAll(host.StartAsync(), DelayThenSignalContinueAsync());
+
+                async Task DelayThenSignalContinueAsync()
+                {
+                    // Emulate the background service doing some work successfully
+                    // for 5 seconds before throwing exception.
+                    await Task.Delay(5000);
+
+                    backgroundDelayTaskSource.SetResult(true);
+                };
+
+                // The following line should not be called, if it is - the test should fail.
+                Assert.True(false, $"The {nameof(AsyncThrowingService)} should have thrown, stopping the Host.");
+            }).Dispose();
+        }
+
+        [Fact]
+        public void HostHandlesExceptionsThrownWithBackgroundServiceExceptionBehaviorOfIgnore()
+        {
+            var backgroundDelayTaskSource = new TaskCompletionSource<bool>();
+
+            using IHost host = CreateBuilder()
+                .ConfigureServices(
+                    services =>
+                    {
+                        services.AddHostedService(
+                            _ => new AsyncThrowingService(backgroundDelayTaskSource.Task));
+
+                        services.PostConfigure<HostOptions>(
+                          options =>
+                            options.BackgroundServiceExceptionBehavior =
+                                BackgroundServiceExceptionBehavior.Ignore);
+                    })
+                .Build();
+
+            host.Start();
+
+            backgroundDelayTaskSource.SetResult(true);
+        }
+
         [Fact]
         public void HostApplicationLifetimeEventsOrderedCorrectlyDuringShutdown()
         {
@@ -1373,7 +1459,9 @@ public ValueTask DisposeAsync()
 
         private class AsyncThrowingService : BackgroundService
         {
-            private readonly Task _executeDelayTask;
+            private readonly Task? _executeDelayTask;
+
+            public AsyncThrowingService() { }
 
             public AsyncThrowingService(Task executeDelayTask)
             {
@@ -1382,7 +1470,8 @@ public AsyncThrowingService(Task executeDelayTask)
 
             protected override async Task ExecuteAsync(CancellationToken stoppingToken)
             {
-                await _executeDelayTask;
+                if (_executeDelayTask is { })
+                    await _executeDelayTask;
 
                 throw new Exception("Background Exception");
             }

src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/Microsoft.Extensions.Hosting.Unit.Tests.csproj

@@ -3,6 +3,7 @@
   <PropertyGroup>
     <TargetFrameworks>$(NetCoreAppCurrent)-windows;$(NetCoreAppCurrent);net461</TargetFrameworks>
     <EnableDefaultItems>true</EnableDefaultItems>
+    <IncludeRemoteExecutor>true</IncludeRemoteExecutor>
   </PropertyGroup>
 
   <ItemGroup>