Migrations: Run unattended upgrades in background, add liveness/readiness health probes (closes #21987)

src/Umbraco.Cms.Api.Delivery/Controllers/DeliveryApiControllerBase.cs

@@ -7,13 +7,15 @@
 using Umbraco.Cms.Core;
 using Umbraco.Cms.Core.Features;
 using Umbraco.Cms.Web.Common.Authorization;
+using Umbraco.Cms.Web.Common.Controllers;
 
 namespace Umbraco.Cms.Api.Delivery.Controllers;
 
 [ApiController]
 [JsonOptionsName(Constants.JsonOptionsNames.DeliveryApi)]
 [MapToApi(DeliveryApiConfiguration.ApiName)]
 [Authorize(Policy = AuthorizationPolicies.UmbracoFeatureEnabled)]
+[MaintenanceModeActionFilter]
 public abstract class DeliveryApiControllerBase : Controller, IUmbracoFeature
 {
     protected string DecodePath(string path)

src/Umbraco.Cms.Api.Management/Controllers/ManagementApiControllerBase.cs

@@ -13,6 +13,7 @@
 using Umbraco.Cms.Core.Models.Membership;
 using Umbraco.Cms.Core.Security;
 using Umbraco.Cms.Web.Common.Authorization;
+using Umbraco.Cms.Web.Common.Controllers;
 using Umbraco.Cms.Web.Common.Filters;
 
 namespace Umbraco.Cms.Api.Management.Controllers;
@@ -25,6 +26,7 @@ namespace Umbraco.Cms.Api.Management.Controllers;
 [AppendEventMessages]
 [DisableBrowserCache]
 [Produces("application/json")]
+[MaintenanceModeActionFilter]
 public abstract class ManagementApiControllerBase : Controller, IUmbracoFeature
 {
     protected IActionResult CreatedAtId<T>(Expression<Func<T, string>> action, Guid id)

src/Umbraco.Cms.Api.Management/Controllers/Server/ConfigurationServerController.cs

@@ -9,9 +9,16 @@
 using Umbraco.Cms.Core.Configuration.Models;
 using Umbraco.Cms.Core.DependencyInjection;
 using Umbraco.Cms.Core.Hosting;
+using Umbraco.Cms.Web.Common.Controllers;
+
 namespace Umbraco.Cms.Api.Management.Controllers.Server;
 
 [ApiVersion("1.0")]
+
+// The backoffice shell reads /server/configuration during its initial connection to determine
+// settings such as whether local login is allowed. This endpoint must be reachable during an
+// unattended upgrade, so the maintenance filter is explicitly bypassed.
+[SkipMaintenanceModeFilter]
 public class ConfigurationServerController : ServerControllerBase
 {
     private readonly SecuritySettings _securitySettings;

src/Umbraco.Cms.Api.Management/Controllers/Server/StatusServerController.cs

@@ -4,10 +4,16 @@
 using Microsoft.AspNetCore.Mvc;
 using Umbraco.Cms.Core.Services;
 using Umbraco.Cms.Api.Management.ViewModels.Server;
+using Umbraco.Cms.Web.Common.Controllers;
 
 namespace Umbraco.Cms.Api.Management.Controllers.Server;
 
 [ApiVersion("1.0")]
+
+// The backoffice shell reads /server/status to detect RuntimeLevel.Upgrading and show the
+// "automatic upgrade in progress" modal. This endpoint must be reachable during an unattended
+// upgrade, so the maintenance filter is explicitly bypassed.
+[SkipMaintenanceModeFilter]
 public class StatusServerController : ServerControllerBase
 {
     private readonly IRuntimeState _runtimeState;

src/Umbraco.Cms.Api.Management/Routing/BackOfficeAreaRoutes.cs

@@ -26,7 +26,7 @@ public BackOfficeAreaRoutes(IRuntimeState runtimeState)
     /// <inheritdoc />
     public void CreateRoutes(IEndpointRouteBuilder endpoints)
     {
-        if (_runtimeState.Level is RuntimeLevel.Install or RuntimeLevel.Upgrade or RuntimeLevel.Run)
+        if (_runtimeState.Level is RuntimeLevel.Install or RuntimeLevel.Upgrade or RuntimeLevel.Upgrading or RuntimeLevel.Run)
         {
             MapMinimalBackOffice(endpoints);
 

src/Umbraco.Cms.Api.Management/Routing/PreviewRoutes.cs

@@ -19,7 +19,7 @@ public PreviewRoutes(IRuntimeState runtimeState)
 
     public void CreateRoutes(IEndpointRouteBuilder endpoints)
     {
-        if (_runtimeState.Level is RuntimeLevel.Install or RuntimeLevel.Upgrade or RuntimeLevel.Run)
+        if (_runtimeState.Level is RuntimeLevel.Install or RuntimeLevel.Upgrade or RuntimeLevel.Upgrading or RuntimeLevel.Run)
         {
             endpoints.MapHub<PreviewHub>(GetPreviewHubRoute());
         }

src/Umbraco.Cms.StaticAssets/umbraco/UmbracoWebsite/Upgrading.cshtml

@@ -0,0 +1,48 @@
+@using Umbraco.Cms.Core.Hosting
+@using Umbraco.Cms.Core.Routing
+@inject IHostingEnvironment HostingEnvironment
+@{
+    var backOfficePath = HostingEnvironment.GetBackOfficePath();
+}
+<!doctype html>
+<html class="no-js" lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
+
+    <title>Site is Being Upgraded</title>
+
+    <link rel="stylesheet" href="@WebPath.Combine(backOfficePath , "website", "/nonodes.css")" />
+    <style type="text/css">
+        body {
+            color: initial;
+        }
+
+        section {
+            background: none;
+        }
+
+        h1 {
+            margin-bottom: 0.5em;
+        }
+
+        h2 {
+            margin-bottom: 0.2em;
+        }
+    </style>
+</head>
+<body>
+
+    <section>
+        <article>
+            <div>
+                <h1>Website is Under Maintenance</h1>
+                <h2>Automatic upgrade in progress</h2>
+                <p>The site is currently running an automatic upgrade. It will be available again shortly.</p>
+            </div>
+        </article>
+    </section>
+
+</body>
+</html>

src/Umbraco.Core/Configuration/Models/GlobalSettings.cs

@@ -102,6 +102,11 @@ public class GlobalSettings
     private const bool StaticForceCombineUrlPathLeftToRight = true;
     private const bool StaticShowMaintenancePageWhenInUpgradeState = true;
 
+    /// <summary>
+    ///     The default value for the <see cref="UpgradingViewPath" /> setting.
+    /// </summary>
+    internal const string StaticUpgradingViewPath = "~/umbraco/UmbracoWebsite/Upgrading.cshtml";
+
     /// <summary>
     ///     Gets or sets a value for the reserved URLs (must end with a comma).
     /// </summary>
@@ -317,4 +322,10 @@ public class GlobalSettings
     /// </summary>
     [DefaultValue(StaticShowMaintenancePageWhenInUpgradeState)]
     public bool ShowMaintenancePageWhenInUpgradeState { get; set; } = StaticShowMaintenancePageWhenInUpgradeState;
+
+    /// <summary>
+    ///     Gets or sets the view path shown during an unattended background upgrade (<see cref="RuntimeLevel.Upgrading"/>).
+    /// </summary>
+    [DefaultValue(StaticUpgradingViewPath)]
+    public string UpgradingViewPath { get; set; } = StaticUpgradingViewPath;
 }

src/Umbraco.Core/Extensions/RuntimeStateExtensions.cs

@@ -31,7 +31,6 @@ public static bool EnableInstaller(this IRuntimeState state)
     /// <param name="state"></param>
     /// <returns></returns>
     public static bool RunUnattendedBootLogic(this IRuntimeState state)
-        => (state.Reason == RuntimeLevelReason.UpgradeMigrations ||
-            state.Reason == RuntimeLevelReason.UpgradePackageMigrations)
-           && state.Level == RuntimeLevel.Run;
+        => (state.Reason == RuntimeLevelReason.UpgradeMigrations || state.Reason == RuntimeLevelReason.UpgradePackageMigrations)
+           && (state.Level == RuntimeLevel.Run || state.Level == RuntimeLevel.Upgrading);
 }

src/Umbraco.Core/RuntimeLevel.cs

@@ -32,6 +32,12 @@ public enum RuntimeLevel
     /// </summary>
     Upgrade = 3,
 
+    /// <summary>
+    ///     The runtime is running an unattended upgrade as a background task.
+    ///     The HTTP server is up and responding to health probes and maintenance-page requests.
+    /// </summary>
+    Upgrading = 4,
+
     /// <summary>
     ///     The runtime has detected an up-to-date Umbraco install and is running.
     /// </summary>

src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.CoreServices.cs

@@ -92,6 +92,7 @@
         builder.AddNotificationAsyncHandler<RuntimeUnattendedInstallNotification, UnattendedInstaller>();
         builder.AddNotificationAsyncHandler<RuntimeUnattendedUpgradeNotification, UnattendedUpgrader>();
         builder.AddNotificationAsyncHandler<RuntimePremigrationsUpgradeNotification, PremigrationUpgrader>();
+        builder.Services.AddHostedService<UnattendedUpgradeBackgroundService>();
 
         // Database availability check.
         builder.Services.AddUnique<IDatabaseAvailabilityCheck, DefaultDatabaseAvailabilityCheck>();

src/Umbraco.Infrastructure/Install/UnattendedUpgradeBackgroundService.cs

@@ -0,0 +1,181 @@
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Umbraco.Cms.Core;
+using Umbraco.Cms.Core.Events;
+using Umbraco.Cms.Core.Exceptions;
+using Umbraco.Cms.Core.Notifications;
+using Umbraco.Cms.Core.Services;
+using ComponentCollection = Umbraco.Cms.Core.Composing.ComponentCollection;
+
+namespace Umbraco.Cms.Infrastructure.Install;
+
+/// <summary>
+/// Runs unattended database migrations as a background task so the HTTP server can start
+/// immediately and respond to health probes during the upgrade.
+/// </summary>
+/// <remarks>
+/// This service is only active when <see cref="RuntimeLevel.Upgrading"/> is set, which
+/// happens when <c>UpgradeUnattended</c> or <c>PackageMigrationsUnattended</c> is true
+/// and database migrations are pending.
+/// </remarks>
+internal sealed class UnattendedUpgradeBackgroundService : BackgroundService
+{
+    private readonly IRuntimeState _runtimeState;
+    private readonly IEventAggregator _eventAggregator;
+    private readonly ComponentCollection _components;
+    private readonly IHostApplicationLifetime _hostApplicationLifetime;
+    private readonly ILogger<UnattendedUpgradeBackgroundService> _logger;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="UnattendedUpgradeBackgroundService"/> class.
+    /// </summary>
+    /// <param name="runtimeState">The Umbraco runtime state.</param>
+    /// <param name="eventAggregator">The event aggregator used to publish upgrade notifications.</param>
+    /// <param name="components">The component collection to initialize after migration completes.</param>
+    /// <param name="hostApplicationLifetime">The host application lifetime for registering started/stopped callbacks.</param>
+    /// <param name="logger">The logger.</param>
+    public UnattendedUpgradeBackgroundService(
+        IRuntimeState runtimeState,
+        IEventAggregator eventAggregator,
+        ComponentCollection components,
+        IHostApplicationLifetime hostApplicationLifetime,
+        ILogger<UnattendedUpgradeBackgroundService> logger)
+    {
+        _runtimeState = runtimeState;
+        _eventAggregator = eventAggregator;
+        _components = components;
+        _hostApplicationLifetime = hostApplicationLifetime;
+        _logger = logger;
+    }
+
+    /// <inheritdoc/>
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        // Only run when an unattended upgrade is pending.
+        if (_runtimeState.Level != RuntimeLevel.Upgrading)
+        {
+            return;
+        }
+
+        _logger.LogInformation("Unattended upgrade background service started.");
+
+        try
+        {
+            await RunMigrationsAsync(stoppingToken);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Unattended upgrade failed.");
+            _runtimeState.Configure(RuntimeLevel.BootFailed, RuntimeLevelReason.BootFailedOnException, ex);
+            return;
+        }
+
+        // Re-evaluate runtime level after migrations complete. This handles all result cases:
+        // - CoreUpgradeComplete / PackageMigrationComplete: confirms the new Run level.
+        // - NotRequired: another instance may have already run migrations; re-check to get Run level.
+        // - HasErrors: BootFailedException is set, so DetermineRuntimeLevel() returns early (no-op).
+        DetermineRuntimeLevel();
+
+        // RunMigrationsAsync may have set BootFailed via a non-throwing error path (HasErrors result).
+        if (_runtimeState.Level == RuntimeLevel.BootFailed)
+        {
+            return;
+        }
+
+        // Migrations complete: initialize components and fire application lifecycle events.
+        try
+        {
+            await _components.InitializeAsync(false, stoppingToken);
+            await _eventAggregator.PublishAsync(new UmbracoApplicationStartingNotification(_runtimeState.Level, false), stoppingToken);
+
+            _hostApplicationLifetime.ApplicationStarted.Register(
+                () => _eventAggregator.Publish(new UmbracoApplicationStartedNotification(false)));
+            _hostApplicationLifetime.ApplicationStopped.Register(
+                () => _eventAggregator.Publish(new UmbracoApplicationStoppedNotification(false)));
+
+            _logger.LogInformation("Unattended upgrade completed successfully.");
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Unattended upgrade post-migration initialization failed.");
+            _runtimeState.Configure(RuntimeLevel.BootFailed, RuntimeLevelReason.BootFailedOnException, ex);
+        }
+    }
+
+    private async Task RunMigrationsAsync(CancellationToken stoppingToken)
+    {
+        // Step 1: Premigrations upgrade.
+        var premigrationNotification = new RuntimePremigrationsUpgradeNotification();
+        await _eventAggregator.PublishAsync(premigrationNotification, stoppingToken);
+
+        switch (premigrationNotification.UpgradeResult)
+        {
+            case RuntimePremigrationsUpgradeNotification.PremigrationUpgradeResult.HasErrors:
+                if (_runtimeState.BootFailedException is null)
+                {
+                    throw new InvalidOperationException(
+                        $"Premigration upgrade result was {RuntimePremigrationsUpgradeNotification.PremigrationUpgradeResult.HasErrors} but no {nameof(BootFailedException)} was registered.");
+                }
+
+                _runtimeState.Configure(RuntimeLevel.BootFailed, RuntimeLevelReason.BootFailedOnException);
+                _logger.LogError(_runtimeState.BootFailedException, "Premigration upgrade failed.");
+                return;
+
+            case RuntimePremigrationsUpgradeNotification.PremigrationUpgradeResult.CoreUpgradeComplete:
+            case RuntimePremigrationsUpgradeNotification.PremigrationUpgradeResult.NotRequired:
+                break;
+        }
+
+        if (_runtimeState.Level == RuntimeLevel.BootFailed)
+        {
+            return;
+        }
+
+        // Step 2: Post-premigrations (navigation and publish-status initialization).
+        await _eventAggregator.PublishAsync(new PostRuntimePremigrationsUpgradeNotification(), stoppingToken);
+
+        // Step 3: Unattended upgrade (main migrations and package migrations).
+        var upgradeNotification = new RuntimeUnattendedUpgradeNotification();
+        await _eventAggregator.PublishAsync(upgradeNotification, stoppingToken);
+
+        switch (upgradeNotification.UnattendedUpgradeResult)
+        {
+            case RuntimeUnattendedUpgradeNotification.UpgradeResult.HasErrors:
+                if (_runtimeState.BootFailedException is null)
+                {
+                    throw new InvalidOperationException(
+                        $"Unattended upgrade result was {RuntimeUnattendedUpgradeNotification.UpgradeResult.HasErrors} but no {nameof(BootFailedException)} was registered.");
+                }
+
+                _runtimeState.Configure(RuntimeLevel.BootFailed, RuntimeLevelReason.BootFailedOnException);
+                _logger.LogError(_runtimeState.BootFailedException, "Unattended upgrade failed.");
+                return;
+
+            case RuntimeUnattendedUpgradeNotification.UpgradeResult.CoreUpgradeComplete:
+            case RuntimeUnattendedUpgradeNotification.UpgradeResult.PackageMigrationComplete:
+            case RuntimeUnattendedUpgradeNotification.UpgradeResult.NotRequired:
+                break;
+        }
+    }
+
+    private void DetermineRuntimeLevel()
+    {
+        // If a boot failure was already registered (e.g. by a premigration handler), there is
+        // nothing more to determine — preserve the existing BootFailed state and return early,
+        // matching the equivalent guard in CoreRuntime.DetermineRuntimeLevel().
+        if (_runtimeState.BootFailedException is not null)
+        {
+            return;
+        }
+
+        try
+        {
+            _runtimeState.DetermineRuntimeLevel();
+        }
+        catch (Exception ex)
+        {
+            _runtimeState.Configure(RuntimeLevel.BootFailed, RuntimeLevelReason.BootFailedOnException, ex);
+            _logger.LogError(ex, "Failed to determine runtime level during unattended upgrade.");
+        }
+    }
+}

src/Umbraco.Infrastructure/Persistence/Repositories/Implement/AuditEntryRepository.cs

@@ -130,7 +130,7 @@ protected override void PersistNewItem(IAuditEntry entity)
         {
             Database.Insert(dto);
         }
-        catch (DbException) when (_runtimeState.Level == RuntimeLevel.Upgrade)
+        catch (DbException) when (_runtimeState.Level is RuntimeLevel.Upgrade or RuntimeLevel.Upgrading)
         {
             // This can happen when in upgrade state, before the migration to add user keys runs.
             // In this case, we will try to insert the audit entry without the user keys.