Migrations: Add auto upgrade coordination for load-balanced setups

src/Umbraco.Core/Configuration/Models/UnattendedSettings.cs

@@ -16,6 +16,7 @@ public class UnattendedSettings
     private const bool StaticInstallUnattended = false;
     private const bool StaticUpgradeUnattended = false;
     private const TelemetryLevel StaticTelemetryLevel = TelemetryLevel.Detailed;
+    private const string StaticMigrationClaimTimeout = "02:00:00";
 
     /// <summary>
     ///     Gets or sets a value indicating whether unattended installs are enabled.
@@ -45,6 +46,17 @@ public class UnattendedSettings
     /// </remarks>
     public bool PackageMigrationsUnattended { get; set; } = true;
 
+    /// <summary>
+    ///     Gets or sets the maximum time a migration leadership claim is considered valid before
+    ///     another server may take over. Protects against a leader crashing mid-migration.
+    /// </summary>
+    /// <remarks>
+    ///     Only relevant in load-balanced deployments with <see cref="UpgradeUnattended"/> enabled.
+    ///     Default is 2 hours, which should exceed the longest reasonable migration run time.
+    /// </remarks>
+    [DefaultValue(StaticMigrationClaimTimeout)]
+    public TimeSpan MigrationClaimTimeout { get; set; } = TimeSpan.Parse(StaticMigrationClaimTimeout);
+
     /// <summary>
     ///     Gets or sets a value to use for creating a user with a name for Unattended Installs
     /// </summary>

src/Umbraco.Core/Constants-Conventions.cs

@@ -36,6 +36,14 @@ public static class Migrations
             ///     The key used to store the Umbraco pre-migrations upgrade plan state.
             /// </summary>
             public const string UmbracoUpgradePlanPremigrationsKey = KeyValuePrefix + UmbracoUpgradePlanPremigrationsName;
+
+            /// <summary>
+            ///     The key used to coordinate migration leadership across servers in a load-balanced
+            ///     environment. The value is either empty (no active leader) or
+            ///     <c>"{machineIdentifier}|{claimedAtUtc:O}"</c> when a server holds the claim,
+            ///     where <c>machineIdentifier</c> is the value returned by <see cref="Umbraco.Cms.Core.Factories.IMachineInfoFactory.GetMachineIdentifier"/>.
+            /// </summary>
+            public const string UpgradeLockKey = "Umbraco.Core.Upgrader.Lock";
         }
 
         /// <summary>

src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.CoreServices.cs

@@ -94,6 +94,7 @@
         builder.AddNotificationAsyncHandler<RuntimeUnattendedInstallNotification, UnattendedInstaller>();
         builder.AddNotificationAsyncHandler<RuntimeUnattendedUpgradeNotification, UnattendedUpgrader>();
         builder.AddNotificationAsyncHandler<RuntimePremigrationsUpgradeNotification, PremigrationUpgrader>();
+        builder.Services.AddSingleton<IMigrationCoordinator, MigrationCoordinator>();
         builder.Services.AddHostedService<UnattendedUpgradeBackgroundService>();
 
         // Database availability check.

src/Umbraco.Infrastructure/Install/IMigrationCoordinator.cs

@@ -0,0 +1,24 @@
+namespace Umbraco.Cms.Infrastructure.Install;
+
+/// <summary>
+/// Coordinates migration leadership across servers in a load-balanced environment.
+/// </summary>
+internal interface IMigrationCoordinator
+{
+    /// <summary>
+    /// Attempts to become the migration leader, blocking until either this server wins the claim
+    /// or another server completes all migrations.
+    /// </summary>
+    /// <param name="cancellationToken">A token that cancels the leadership wait loop.</param>
+    /// <returns>
+    /// <c>true</c> if this server is the migration leader and must call <see cref="ReleaseLeadership"/> after
+    /// running migrations; <c>false</c> if another server completed migrations and this server should skip them.
+    /// </returns>
+    Task<bool> TryBecomeLeaderAsync(CancellationToken cancellationToken);
+
+    /// <summary>
+    /// Releases the migration leadership claim if it is still held by this instance.
+    /// Must be called in a <c>finally</c> block to ensure release even on failure.
+    /// </summary>
+    void ReleaseLeadership();
+}

src/Umbraco.Infrastructure/Install/MigrationCoordinator.cs

@@ -0,0 +1,155 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Umbraco.Cms.Core;
+using Umbraco.Cms.Core.Configuration.Models;
+using Umbraco.Cms.Core.Factories;
+using Umbraco.Cms.Core.Scoping;
+using Umbraco.Cms.Core.Services;
+
+namespace Umbraco.Cms.Infrastructure.Install;
+
+/// <summary>
+/// Coordinates migration leadership across servers in a load-balanced environment.
+/// Exactly one server claims leadership, runs all migrations, then releases the claim.
+/// All other servers wait until the leader finishes, then proceed with per-server initialization.
+/// </summary>
+internal sealed class MigrationCoordinator : IMigrationCoordinator
+{
+    private readonly ICoreScopeProvider _scopeProvider;
+    private readonly IKeyValueService _keyValueService;
+    private readonly IRuntimeState _runtimeState;
+    private readonly IMachineInfoFactory _machineInfoFactory;
+    private readonly IOptions<UnattendedSettings> _unattendedSettings;
+    private readonly ILogger<MigrationCoordinator> _logger;
+    private string? _leaderClaim;
+
+    public MigrationCoordinator(
+        ICoreScopeProvider scopeProvider,
+        IKeyValueService keyValueService,
+        IRuntimeState runtimeState,
+        IMachineInfoFactory machineInfoFactory,
+        IOptions<UnattendedSettings> unattendedSettings,
+        ILogger<MigrationCoordinator> logger)
+    {
+        _scopeProvider = scopeProvider;
+        _keyValueService = keyValueService;
+        _runtimeState = runtimeState;
+        _machineInfoFactory = machineInfoFactory;
+        _unattendedSettings = unattendedSettings;
+        _logger = logger;
+    }
+
+    /// <inheritdoc/>
+    public async Task<bool> TryBecomeLeaderAsync(CancellationToken cancellationToken)
+    {
+        var machineIdentifier = _machineInfoFactory.GetMachineIdentifier();
+
+        while (cancellationToken.IsCancellationRequested is false)
+        {
+            if (TryClaimLeadership(machineIdentifier))
+            {
+                _logger.LogInformation("This server claimed migration leadership.");
+                return true;
+            }
+
+            try
+            {
+                _runtimeState.DetermineRuntimeLevel();
+            }
+            catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+            {
+                return false;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(ex, "Could not determine runtime level during migration wait; will retry.");
+            }
+
+            switch (_runtimeState.Level)
+            {
+                case RuntimeLevel.Run:
+                    _logger.LogInformation("Migrations completed by another server; proceeding as follower.");
+                    return false;
+                case RuntimeLevel.BootFailed:
+                    _logger.LogError("Runtime entered BootFailed state while waiting for migrations.");
+                    return false;
+                default:
+                    _logger.LogDebug("Waiting for migration leader to finish...");
+                    try
+                    {
+                        await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
+                    }
+                    catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+                    {
+                        return false;
+                    }
+
+                    break;
+            }
+        }
+
+        return false;
+    }
+
+    /// <inheritdoc/>
+    public void ReleaseLeadership()
+    {
+        if (_leaderClaim is null)
+        {
+            return;
+        }
+
+        try
+        {
+            using ICoreScope scope = _scopeProvider.CreateCoreScope();
+            scope.WriteLock(Constants.Locks.KeyValues);
+
+            string? current = _keyValueService.GetValue(Constants.Conventions.Migrations.UpgradeLockKey);
+            if (current == _leaderClaim)
+            {
+                _keyValueService.SetValue(Constants.Conventions.Migrations.UpgradeLockKey, string.Empty);
+            }
+
+            scope.Complete();
+            _leaderClaim = null;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Failed to release migration leadership; continuing shutdown because leadership release is best-effort.");
+        }
+    }
+
+    private static bool IsStale(string claim, TimeSpan timeout)
+    {
+        var separatorIndex = claim.IndexOf('|');
+        return separatorIndex < 0
+               || !DateTimeOffset.TryParse(claim.AsSpan(separatorIndex + 1), out DateTimeOffset timestamp)
+               || DateTimeOffset.UtcNow - timestamp > timeout;
+    }
+
+    // Acquires WriteLock(KeyValues) so the read-then-write is serialized across all servers.
+    // Inner GetValue and SetValue calls create nested scopes that join the outer transaction;
+    // their internal WriteLock requests are no-ops because the lock is already held.
+    private bool TryClaimLeadership(string machineIdentifier)
+    {
+        TimeSpan timeout = _unattendedSettings.Value.MigrationClaimTimeout;
+
+        using ICoreScope scope = _scopeProvider.CreateCoreScope();
+        scope.WriteLock(Constants.Locks.KeyValues);
+
+        string? current = _keyValueService.GetValue(Constants.Conventions.Migrations.UpgradeLockKey);
+
+        bool canClaim = string.IsNullOrEmpty(current)
+            || IsStale(current, timeout)
+            || current.StartsWith(machineIdentifier + "|", StringComparison.Ordinal);
+
+        if (canClaim)
+        {
+            _leaderClaim = $"{machineIdentifier}|{DateTimeOffset.UtcNow:O}";
+            _keyValueService.SetValue(Constants.Conventions.Migrations.UpgradeLockKey, _leaderClaim);
+        }
+
+        scope.Complete();
+        return canClaim;
+    }
+}

src/Umbraco.Infrastructure/Install/UnattendedUpgradeBackgroundService.cs

@@ -24,6 +24,7 @@
     private readonly IEventAggregator _eventAggregator;
     private readonly ComponentCollection _components;
     private readonly IHostApplicationLifetime _hostApplicationLifetime;
+    private readonly IMigrationCoordinator _coordinator;
     private readonly ILogger<UnattendedUpgradeBackgroundService> _logger;
 
     /// <summary>
@@ -33,18 +34,21 @@
     /// <param name="eventAggregator">The event aggregator used to publish upgrade notifications.</param>
     /// <param name="components">The component collection to initialize after migration completes.</param>
     /// <param name="hostApplicationLifetime">The host application lifetime for registering started/stopped callbacks.</param>
+    /// <param name="coordinator">Coordinates migration leadership across servers in a load-balanced environment.</param>
     /// <param name="logger">The logger.</param>
     public UnattendedUpgradeBackgroundService(
         IRuntimeState runtimeState,
         IEventAggregator eventAggregator,
         ComponentCollection components,
         IHostApplicationLifetime hostApplicationLifetime,
+        IMigrationCoordinator coordinator,
         ILogger<UnattendedUpgradeBackgroundService> logger)
     {
         _runtimeState = runtimeState;
         _eventAggregator = eventAggregator;
         _components = components;
         _hostApplicationLifetime = hostApplicationLifetime;
+        _coordinator = coordinator;
         _logger = logger;
     }
 
@@ -59,24 +63,51 @@
 
         _logger.LogInformation("Unattended upgrade background service started.");
 
+        bool isLeader = false;
+
         try
         {
-            await RunMigrationsAsync(stoppingToken);
+            isLeader = await _coordinator.TryBecomeLeaderAsync(stoppingToken);
+
+            if (_runtimeState.Level == RuntimeLevel.BootFailed)
+            {
+                return;
+            }
+
+            if (isLeader)
+            {
+                // Belt-and-suspenders for graceful shutdowns (e.g. Azure SIGTERM): release the claim
+                // as soon as the host begins stopping, even if a migration step is still blocking.
+                _hostApplicationLifetime.ApplicationStopping.Register(() => _coordinator.ReleaseLeadership());
+                await RunMigrationsAsync(stoppingToken);
+            }
+            else
+            {
+                // Follower: rebuild per-server in-memory navigation and publish status
+                // from the fully-migrated database.
+                await _eventAggregator.PublishAsync(new PostRuntimePremigrationsUpgradeNotification(), stoppingToken);
+            }
         }
         catch (Exception ex)
         {
             _logger.LogError(ex, "Unattended upgrade failed.");
             _runtimeState.Configure(RuntimeLevel.BootFailed, RuntimeLevelReason.BootFailedOnException, ex);
             return;
         }
+        finally
+        {
+            // Always release the claim — even on leader failure — so other servers
+            // can detect completion or take over.
+            if (isLeader)
+            {
+                _coordinator.ReleaseLeadership();
+            }
+        }
 
-        // Re-evaluate runtime level after migrations complete. This handles all result cases:
-        // - CoreUpgradeComplete / PackageMigrationComplete: confirms the new Run level.
-        // - NotRequired: another instance may have already run migrations; re-check to get Run level.
-        // - HasErrors: BootFailedException is set, so DetermineRuntimeLevel() returns early (no-op).
+        // For the leader: confirms migrations succeeded and level transitions to Run.
+        // For followers: level is already Run (set during TryBecomeLeaderAsync polling).
         DetermineRuntimeLevel();
 
-        // RunMigrationsAsync may have set BootFailed via a non-throwing error path (HasErrors result).
         if (_runtimeState.Level == RuntimeLevel.BootFailed)
         {
             return;