diff --git a/src/Umbraco.Core/Configuration/Models/UnattendedSettings.cs b/src/Umbraco.Core/Configuration/Models/UnattendedSettings.cs index 30b3bb3d5e47..a9207730cfbc 100644 --- a/src/Umbraco.Core/Configuration/Models/UnattendedSettings.cs +++ b/src/Umbraco.Core/Configuration/Models/UnattendedSettings.cs @@ -16,6 +16,7 @@ public class UnattendedSettings private const bool StaticInstallUnattended = false; private const bool StaticUpgradeUnattended = false; private const TelemetryLevel StaticTelemetryLevel = TelemetryLevel.Detailed; + private const string StaticMigrationClaimTimeout = "02:00:00"; /// /// Gets or sets a value indicating whether unattended installs are enabled. @@ -45,6 +46,17 @@ public class UnattendedSettings /// public bool PackageMigrationsUnattended { get; set; } = true; + /// + /// Gets or sets the maximum time a migration leadership claim is considered valid before + /// another server may take over. Protects against a leader crashing mid-migration. + /// + /// + /// Only relevant in load-balanced deployments with enabled. + /// Default is 2 hours, which should exceed the longest reasonable migration run time. + /// + [DefaultValue(StaticMigrationClaimTimeout)] + public TimeSpan MigrationClaimTimeout { get; set; } = TimeSpan.Parse(StaticMigrationClaimTimeout); + /// /// Gets or sets a value to use for creating a user with a name for Unattended Installs /// diff --git a/src/Umbraco.Core/Constants-Conventions.cs b/src/Umbraco.Core/Constants-Conventions.cs index a590904121f7..74c3e9f28783 100644 --- a/src/Umbraco.Core/Constants-Conventions.cs +++ b/src/Umbraco.Core/Constants-Conventions.cs @@ -36,6 +36,14 @@ public static class Migrations /// The key used to store the Umbraco pre-migrations upgrade plan state. /// public const string UmbracoUpgradePlanPremigrationsKey = KeyValuePrefix + UmbracoUpgradePlanPremigrationsName; + + /// + /// The key used to coordinate migration leadership across servers in a load-balanced + /// environment. The value is either empty (no active leader) or + /// "{machineIdentifier}|{claimedAtUtc:O}" when a server holds the claim, + /// where machineIdentifier is the value returned by . + /// + public const string UpgradeLockKey = "Umbraco.Core.Upgrader.Lock"; } /// diff --git a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.CoreServices.cs b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.CoreServices.cs index 38d5ba25244c..b76bb282a8ca 100644 --- a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.CoreServices.cs +++ b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.CoreServices.cs @@ -94,6 +94,7 @@ public static IUmbracoBuilder AddCoreInitialServices(this IUmbracoBuilder builde builder.AddNotificationAsyncHandler(); builder.AddNotificationAsyncHandler(); builder.AddNotificationAsyncHandler(); + builder.Services.AddSingleton(); builder.Services.AddHostedService(); // Database availability check. diff --git a/src/Umbraco.Infrastructure/Install/IMigrationCoordinator.cs b/src/Umbraco.Infrastructure/Install/IMigrationCoordinator.cs new file mode 100644 index 000000000000..0f47423b4d1a --- /dev/null +++ b/src/Umbraco.Infrastructure/Install/IMigrationCoordinator.cs @@ -0,0 +1,24 @@ +namespace Umbraco.Cms.Infrastructure.Install; + +/// +/// Coordinates migration leadership across servers in a load-balanced environment. +/// +internal interface IMigrationCoordinator +{ + /// + /// Attempts to become the migration leader, blocking until either this server wins the claim + /// or another server completes all migrations. + /// + /// A token that cancels the leadership wait loop. + /// + /// true if this server is the migration leader and must call after + /// running migrations; false if another server completed migrations and this server should skip them. + /// + Task TryBecomeLeaderAsync(CancellationToken cancellationToken); + + /// + /// Releases the migration leadership claim if it is still held by this instance. + /// Must be called in a finally block to ensure release even on failure. + /// + void ReleaseLeadership(); +} diff --git a/src/Umbraco.Infrastructure/Install/MigrationCoordinator.cs b/src/Umbraco.Infrastructure/Install/MigrationCoordinator.cs new file mode 100644 index 000000000000..caf3cde43bda --- /dev/null +++ b/src/Umbraco.Infrastructure/Install/MigrationCoordinator.cs @@ -0,0 +1,177 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Configuration.Models; +using Umbraco.Cms.Core.Factories; +using Umbraco.Cms.Core.Scoping; +using Umbraco.Cms.Core.Services; + +namespace Umbraco.Cms.Infrastructure.Install; + +/// +/// Coordinates migration leadership across servers in a load-balanced environment. +/// Exactly one server claims leadership, runs all migrations, then releases the claim. +/// All other servers wait until the leader finishes, then proceed with per-server initialization. +/// +internal sealed class MigrationCoordinator : IMigrationCoordinator +{ + private readonly ICoreScopeProvider _scopeProvider; + private readonly IKeyValueService _keyValueService; + private readonly IRuntimeState _runtimeState; + private readonly IMachineInfoFactory _machineInfoFactory; + private readonly IOptions _unattendedSettings; + private readonly ILogger _logger; + private string? _leaderClaim; + + public MigrationCoordinator( + ICoreScopeProvider scopeProvider, + IKeyValueService keyValueService, + IRuntimeState runtimeState, + IMachineInfoFactory machineInfoFactory, + IOptions unattendedSettings, + ILogger logger) + { + _scopeProvider = scopeProvider; + _keyValueService = keyValueService; + _runtimeState = runtimeState; + _machineInfoFactory = machineInfoFactory; + _unattendedSettings = unattendedSettings; + _logger = logger; + } + + /// + public async Task TryBecomeLeaderAsync(CancellationToken cancellationToken) + { + var machineIdentifier = _machineInfoFactory.GetMachineIdentifier(); + + while (cancellationToken.IsCancellationRequested is false) + { + if (TryClaimLeadership(machineIdentifier)) + { + // Re-check after claiming: the previous leader may have finished between our last + // DetermineRuntimeLevel call and our successful claim of the now-empty lock. + try + { + _runtimeState.DetermineRuntimeLevel(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return false; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Could not re-determine runtime level after claiming leadership; proceeding as leader."); + } + + if (_runtimeState.Level == RuntimeLevel.Run) + { + ReleaseLeadership(); + _logger.LogInformation("Migrations completed by another server; proceeding as follower."); + return false; + } + + _logger.LogInformation("This server claimed migration leadership."); + return true; + } + + try + { + _runtimeState.DetermineRuntimeLevel(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return false; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Could not determine runtime level during migration wait; will retry."); + } + + switch (_runtimeState.Level) + { + case RuntimeLevel.Run: + _logger.LogInformation("Migrations completed by another server; proceeding as follower."); + return false; + case RuntimeLevel.BootFailed: + _logger.LogError("Runtime entered BootFailed state while waiting for migrations."); + return false; + default: + _logger.LogDebug("Waiting for migration leader to finish..."); + try + { + await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return false; + } + + break; + } + } + + return false; + } + + /// + public void ReleaseLeadership() + { + if (_leaderClaim is null) + { + return; + } + + try + { + using ICoreScope scope = _scopeProvider.CreateCoreScope(); + scope.WriteLock(Constants.Locks.KeyValues); + + string? current = _keyValueService.GetValue(Constants.Conventions.Migrations.UpgradeLockKey); + if (current == _leaderClaim) + { + _keyValueService.SetValue(Constants.Conventions.Migrations.UpgradeLockKey, string.Empty); + } + + scope.Complete(); + _leaderClaim = null; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to release migration leadership; continuing shutdown because leadership release is best-effort."); + } + } + + private static bool IsStale(string claim, TimeSpan timeout) + { + var separatorIndex = claim.IndexOf('|'); + return separatorIndex < 0 + || !DateTimeOffset.TryParse(claim.AsSpan(separatorIndex + 1), out DateTimeOffset timestamp) + || DateTimeOffset.UtcNow - timestamp > timeout; + } + + // Acquires WriteLock(KeyValues) so the read-then-write is serialized across all servers. + // Inner GetValue and SetValue calls create nested scopes that join the outer transaction; + // their internal WriteLock requests are no-ops because the lock is already held. + private bool TryClaimLeadership(string machineIdentifier) + { + TimeSpan timeout = _unattendedSettings.Value.MigrationClaimTimeout; + + using ICoreScope scope = _scopeProvider.CreateCoreScope(); + scope.WriteLock(Constants.Locks.KeyValues); + + string? current = _keyValueService.GetValue(Constants.Conventions.Migrations.UpgradeLockKey); + + bool canClaim = string.IsNullOrEmpty(current) + || IsStale(current, timeout) + || current.StartsWith(machineIdentifier + "|", StringComparison.Ordinal); + + if (canClaim) + { + _leaderClaim = $"{machineIdentifier}|{DateTimeOffset.UtcNow:O}"; + _keyValueService.SetValue(Constants.Conventions.Migrations.UpgradeLockKey, _leaderClaim); + } + + scope.Complete(); + return canClaim; + } +} diff --git a/src/Umbraco.Infrastructure/Install/UnattendedUpgradeBackgroundService.cs b/src/Umbraco.Infrastructure/Install/UnattendedUpgradeBackgroundService.cs index ecbfed4329ca..bc9eb961417b 100644 --- a/src/Umbraco.Infrastructure/Install/UnattendedUpgradeBackgroundService.cs +++ b/src/Umbraco.Infrastructure/Install/UnattendedUpgradeBackgroundService.cs @@ -24,6 +24,7 @@ internal sealed class UnattendedUpgradeBackgroundService : BackgroundService private readonly IEventAggregator _eventAggregator; private readonly ComponentCollection _components; private readonly IHostApplicationLifetime _hostApplicationLifetime; + private readonly IMigrationCoordinator _coordinator; private readonly ILogger _logger; /// @@ -33,18 +34,21 @@ internal sealed class UnattendedUpgradeBackgroundService : BackgroundService /// The event aggregator used to publish upgrade notifications. /// The component collection to initialize after migration completes. /// The host application lifetime for registering started/stopped callbacks. + /// Coordinates migration leadership across servers in a load-balanced environment. /// The logger. public UnattendedUpgradeBackgroundService( IRuntimeState runtimeState, IEventAggregator eventAggregator, ComponentCollection components, IHostApplicationLifetime hostApplicationLifetime, + IMigrationCoordinator coordinator, ILogger logger) { _runtimeState = runtimeState; _eventAggregator = eventAggregator; _components = components; _hostApplicationLifetime = hostApplicationLifetime; + _coordinator = coordinator; _logger = logger; } @@ -59,9 +63,30 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) _logger.LogInformation("Unattended upgrade background service started."); + bool isLeader = false; + try { - await RunMigrationsAsync(stoppingToken); + isLeader = await _coordinator.TryBecomeLeaderAsync(stoppingToken); + + if (_runtimeState.Level == RuntimeLevel.BootFailed) + { + return; + } + + if (isLeader) + { + // Belt-and-suspenders for graceful shutdowns (e.g. Azure SIGTERM): release the claim + // as soon as the host begins stopping, even if a migration step is still blocking. + _hostApplicationLifetime.ApplicationStopping.Register(() => _coordinator.ReleaseLeadership()); + await RunMigrationsAsync(stoppingToken); + } + else + { + // Follower: rebuild per-server in-memory navigation and publish status + // from the fully-migrated database. + await _eventAggregator.PublishAsync(new PostRuntimePremigrationsUpgradeNotification(), stoppingToken); + } } catch (Exception ex) { @@ -69,14 +94,20 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) _runtimeState.Configure(RuntimeLevel.BootFailed, RuntimeLevelReason.BootFailedOnException, ex); return; } + finally + { + // Always release the claim — even on leader failure — so other servers + // can detect completion or take over. + if (isLeader) + { + _coordinator.ReleaseLeadership(); + } + } - // Re-evaluate runtime level after migrations complete. This handles all result cases: - // - CoreUpgradeComplete / PackageMigrationComplete: confirms the new Run level. - // - NotRequired: another instance may have already run migrations; re-check to get Run level. - // - HasErrors: BootFailedException is set, so DetermineRuntimeLevel() returns early (no-op). + // For the leader: confirms migrations succeeded and level transitions to Run. + // For followers: level is already Run (set during TryBecomeLeaderAsync polling). DetermineRuntimeLevel(); - // RunMigrationsAsync may have set BootFailed via a non-throwing error path (HasErrors result). if (_runtimeState.Level == RuntimeLevel.BootFailed) { return; diff --git a/tests/Umbraco.Tests.Integration/Umbraco.Infrastructure/Install/MigrationCoordinatorTests.cs b/tests/Umbraco.Tests.Integration/Umbraco.Infrastructure/Install/MigrationCoordinatorTests.cs new file mode 100644 index 000000000000..62b8dc753f54 --- /dev/null +++ b/tests/Umbraco.Tests.Integration/Umbraco.Infrastructure/Install/MigrationCoordinatorTests.cs @@ -0,0 +1,187 @@ +// Copyright (c) Umbraco. +// See LICENSE for more details. + +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Moq; +using NUnit.Framework; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Configuration.Models; +using Umbraco.Cms.Core.Factories; +using Umbraco.Cms.Core.Scoping; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Infrastructure.Install; +using Umbraco.Cms.Tests.Common.Testing; +using Umbraco.Cms.Tests.Integration.Testing; + +namespace Umbraco.Cms.Tests.Integration.Umbraco.Infrastructure.Install; + +[TestFixture] +[Timeout(60000)] +[UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)] +internal sealed class MigrationCoordinatorTests : UmbracoIntegrationTest +{ + private IKeyValueService KeyValueService => GetRequiredService(); + + // Persistence + + [Test] + public async Task TryBecomeLeaderAsync_WhenNoClaim_WritesClaimToDatabase() + { + var coordinator = CreateCoordinator("machine-a", CreateUpgradingRuntimeState().Object); + + await coordinator.TryBecomeLeaderAsync(CancellationToken.None); + + var claim = KeyValueService.GetValue(Constants.Conventions.Migrations.UpgradeLockKey); + Assert.That(claim, Does.StartWith("machine-a|")); + } + + [Test] + public async Task ReleaseLeadership_AfterClaimingLeadership_ClearsKeyInDatabase() + { + var coordinator = CreateCoordinator("machine-a", CreateUpgradingRuntimeState().Object); + await coordinator.TryBecomeLeaderAsync(CancellationToken.None); + + coordinator.ReleaseLeadership(); + + var claim = KeyValueService.GetValue(Constants.Conventions.Migrations.UpgradeLockKey); + Assert.That(claim, Is.Null.Or.Empty); + } + + // Stale claim takeover + + [Test] + public async Task TryBecomeLeaderAsync_WhenExistingClaimIsStale_TakesOverLeadership() + { + var staleTimestamp = DateTimeOffset.UtcNow.AddHours(-3).ToString("O"); + KeyValueService.SetValue( + Constants.Conventions.Migrations.UpgradeLockKey, + $"crashed-server|{staleTimestamp}"); + + var coordinator = CreateCoordinator("machine-a", CreateUpgradingRuntimeState().Object, claimTimeout: TimeSpan.FromHours(2)); + var result = await coordinator.TryBecomeLeaderAsync(CancellationToken.None); + + Assert.IsTrue(result); + var claim = KeyValueService.GetValue(Constants.Conventions.Migrations.UpgradeLockKey); + Assert.That(claim, Does.StartWith("machine-a|")); + } + + // Concurrent race — the core guarantee + + [Test] + public void TryBecomeLeaderAsync_WhenTwoCoordinatorsRaceConcurrently_ExactlyOneWins() + { + // Both mocks start Upgrading. The winner calls DetermineRuntimeLevel once (post-claim check) + // and sees Upgrading — migrations haven't run yet, so it continues as leader and returns true. + // The loser calls DetermineRuntimeLevel twice: the first poll keeps Upgrading (5 s sleep), + // the second transitions to Run, and the loser returns false. + var runtimeState1 = CreateTransitioningRuntimeState(); + var runtimeState2 = CreateTransitioningRuntimeState(); + + var coordinator1 = CreateCoordinator("machine-a", runtimeState1.Object); + var coordinator2 = CreateCoordinator("machine-b", runtimeState2.Object); + + bool? result1 = null; + bool? result2 = null; + Exception? ex1 = null; + Exception? ex2 = null; + + var gate = new ManualResetEventSlim(false); + + var t1 = new Thread(() => + { + try + { + gate.Wait(); + result1 = coordinator1.TryBecomeLeaderAsync(CancellationToken.None).GetAwaiter().GetResult(); + } + catch (Exception ex) + { + ex1 = ex; + } + }); + + var t2 = new Thread(() => + { + try + { + gate.Wait(); + result2 = coordinator2.TryBecomeLeaderAsync(CancellationToken.None).GetAwaiter().GetResult(); + } + catch (Exception ex) + { + ex2 = ex; + } + }); + + // Suppress ambient scope from leaking into worker threads (matches LocksTests pattern). + using (ExecutionContext.SuppressFlow()) + { + t1.Start(); + t2.Start(); + } + + gate.Set(); + t1.Join(); + t2.Join(); + + Assert.IsNull(ex1, $"Coordinator 1 threw: {ex1}"); + Assert.IsNull(ex2, $"Coordinator 2 threw: {ex2}"); + Assert.IsNotNull(result1); + Assert.IsNotNull(result2); + Assert.AreNotEqual(result1, result2, "Exactly one coordinator should win leadership"); + + var winnerId = result1 == true ? "machine-a" : "machine-b"; + var claim = KeyValueService.GetValue(Constants.Conventions.Migrations.UpgradeLockKey); + Assert.That(claim, Does.StartWith(winnerId + "|")); + } + + private MigrationCoordinator CreateCoordinator( + string machineId, + IRuntimeState runtimeState, + TimeSpan? claimTimeout = null) + { + var machineInfoFactory = Mock.Of( + f => f.GetMachineIdentifier() == machineId); + + var settings = Options.Create(new UnattendedSettings + { + MigrationClaimTimeout = claimTimeout ?? TimeSpan.FromHours(2), + }); + + return new MigrationCoordinator( + GetRequiredService(), + GetRequiredService(), + runtimeState, + machineInfoFactory, + settings, + NullLogger.Instance); + } + + private static Mock CreateUpgradingRuntimeState() + { + var mock = new Mock(); + mock.SetupGet(x => x.Level).Returns(RuntimeLevel.Upgrading); + return mock; + } + + private static Mock CreateTransitioningRuntimeState() + { + var mock = new Mock(); + mock.SetupGet(x => x.Level).Returns(RuntimeLevel.Upgrading); + + // The winner calls DetermineRuntimeLevel() once from the post-claim check and should + // still see Upgrading (migrations haven't run yet). The loser calls it from the poll + // loop — first call keeps Upgrading, so it sleeps once; second call transitions to Run. + int callCount = 0; + mock.Setup(x => x.DetermineRuntimeLevel()) + .Callback(() => + { + if (Interlocked.Increment(ref callCount) >= 2) + { + mock.SetupGet(x => x.Level).Returns(RuntimeLevel.Run); + } + }); + return mock; + } +} diff --git a/tests/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/Install/MigrationCoordinatorTests.cs b/tests/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/Install/MigrationCoordinatorTests.cs new file mode 100644 index 000000000000..d26bcabd6461 --- /dev/null +++ b/tests/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/Install/MigrationCoordinatorTests.cs @@ -0,0 +1,330 @@ +// Copyright (c) Umbraco. +// See LICENSE for more details. + +using System.Data; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Moq; +using NUnit.Framework; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Configuration.Models; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.Factories; +using Umbraco.Cms.Core.Scoping; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Infrastructure.Install; + +namespace Umbraco.Cms.Tests.UnitTests.Umbraco.Infrastructure.Install; + +[TestFixture] +public class MigrationCoordinatorTests +{ + private const string TestMachineIdentifier = "test-machine"; + private const string OtherMachineIdentifier = "other-machine"; + + private Mock _scopeProviderMock = null!; + private Mock _keyValueServiceMock = null!; + private Mock _runtimeStateMock = null!; + + [SetUp] + public void SetUp() + { + _scopeProviderMock = new Mock(); + _keyValueServiceMock = new Mock(); + _runtimeStateMock = new Mock(); + + SetupScopeProviderMock(); + } + + [Test] + public async Task TryBecomeLeaderAsync_WhenClaimKeyIsEmpty_ClaimsLeadershipAndReturnsTrue() + { + _keyValueServiceMock + .Setup(x => x.GetValue(Constants.Conventions.Migrations.UpgradeLockKey)) + .Returns((string?)null); + + var sut = CreateSut(); + var result = await sut.TryBecomeLeaderAsync(CancellationToken.None); + + Assert.IsTrue(result); + _keyValueServiceMock.Verify( + x => x.SetValue( + Constants.Conventions.Migrations.UpgradeLockKey, + It.Is(v => v.StartsWith(TestMachineIdentifier + "|"))), + Times.Once); + } + + [Test] + public async Task TryBecomeLeaderAsync_WhenClaimIsStale_ClaimsLeadershipAndReturnsTrue() + { + var staleTimestamp = DateTimeOffset.UtcNow.AddHours(-3).ToString("O"); + _keyValueServiceMock + .Setup(x => x.GetValue(Constants.Conventions.Migrations.UpgradeLockKey)) + .Returns($"{OtherMachineIdentifier}|{staleTimestamp}"); + + var sut = CreateSut(claimTimeout: TimeSpan.FromHours(2)); + var result = await sut.TryBecomeLeaderAsync(CancellationToken.None); + + Assert.IsTrue(result); + _keyValueServiceMock.Verify( + x => x.SetValue( + Constants.Conventions.Migrations.UpgradeLockKey, + It.Is(v => v.StartsWith(TestMachineIdentifier + "|"))), + Times.Once); + } + + [Test] + public async Task TryBecomeLeaderAsync_WhenClaimBelongsToSameMachine_ReclaimsAndReturnsTrue() + { + var recentTimestamp = DateTimeOffset.UtcNow.AddMinutes(-1).ToString("O"); + _keyValueServiceMock + .Setup(x => x.GetValue(Constants.Conventions.Migrations.UpgradeLockKey)) + .Returns($"{TestMachineIdentifier}|{recentTimestamp}"); + + var sut = CreateSut(); + var result = await sut.TryBecomeLeaderAsync(CancellationToken.None); + + Assert.IsTrue(result); + _keyValueServiceMock.Verify( + x => x.SetValue( + Constants.Conventions.Migrations.UpgradeLockKey, + It.Is(v => v.StartsWith(TestMachineIdentifier + "|"))), + Times.Once); + } + + [Test] + public async Task TryBecomeLeaderAsync_WhenLeaderFinishedBetweenPollAndClaim_ReleasesClaimAndReturnsFalse() + { + // Simulate TOCTOU: lock is empty (leader released between our last DetermineRuntimeLevel and our claim). + _keyValueServiceMock + .Setup(x => x.GetValue(Constants.Conventions.Migrations.UpgradeLockKey)) + .Returns((string?)null); + + // DetermineRuntimeLevel transitions level to Run (leader already finished). + _runtimeStateMock.SetupGet(x => x.Level).Returns(RuntimeLevel.Upgrading); + _runtimeStateMock + .Setup(x => x.DetermineRuntimeLevel()) + .Callback(() => _runtimeStateMock.SetupGet(x => x.Level).Returns(RuntimeLevel.Run)); + + string? capturedClaim = null; + _keyValueServiceMock + .Setup(x => x.SetValue(Constants.Conventions.Migrations.UpgradeLockKey, It.IsAny())) + .Callback((_, v) => capturedClaim = v); + _keyValueServiceMock + .Setup(x => x.GetValue(Constants.Conventions.Migrations.UpgradeLockKey)) + .Returns(() => capturedClaim); + + var sut = CreateSut(); + var result = await sut.TryBecomeLeaderAsync(CancellationToken.None); + + Assert.IsFalse(result); + // The claim was written then cleared by ReleaseLeadership. + _keyValueServiceMock.Verify( + x => x.SetValue(Constants.Conventions.Migrations.UpgradeLockKey, string.Empty), + Times.Once); + } + + [Test] + public async Task TryBecomeLeaderAsync_WhenOtherMachineHoldsClaim_PollsUntilRunLevelAndReturnsFalse() + { + var recentTimestamp = DateTimeOffset.UtcNow.AddMinutes(-1).ToString("O"); + _keyValueServiceMock + .Setup(x => x.GetValue(Constants.Conventions.Migrations.UpgradeLockKey)) + .Returns($"{OtherMachineIdentifier}|{recentTimestamp}"); + + _runtimeStateMock.SetupGet(x => x.Level).Returns(RuntimeLevel.Upgrading); + _runtimeStateMock + .Setup(x => x.DetermineRuntimeLevel()) + .Callback(() => _runtimeStateMock.SetupGet(x => x.Level).Returns(RuntimeLevel.Run)); + + var sut = CreateSut(); + var result = await sut.TryBecomeLeaderAsync(CancellationToken.None); + + Assert.IsFalse(result); + } + + [Test] + public async Task TryBecomeLeaderAsync_WhenOtherMachineHoldsClaim_PollsUntilBootFailedAndReturnsFalse() + { + var recentTimestamp = DateTimeOffset.UtcNow.AddMinutes(-1).ToString("O"); + _keyValueServiceMock + .Setup(x => x.GetValue(Constants.Conventions.Migrations.UpgradeLockKey)) + .Returns($"{OtherMachineIdentifier}|{recentTimestamp}"); + + _runtimeStateMock.SetupGet(x => x.Level).Returns(RuntimeLevel.Upgrading); + _runtimeStateMock + .Setup(x => x.DetermineRuntimeLevel()) + .Callback(() => _runtimeStateMock.SetupGet(x => x.Level).Returns(RuntimeLevel.BootFailed)); + + var sut = CreateSut(); + var result = await sut.TryBecomeLeaderAsync(CancellationToken.None); + + Assert.IsFalse(result); + } + + [Test] + public async Task TryBecomeLeaderAsync_WhenDetermineRuntimeLevelThrows_LogsWarningAndReturnsFalseWhenRunDetected() + { + var recentTimestamp = DateTimeOffset.UtcNow.AddMinutes(-1).ToString("O"); + _keyValueServiceMock + .Setup(x => x.GetValue(Constants.Conventions.Migrations.UpgradeLockKey)) + .Returns($"{OtherMachineIdentifier}|{recentTimestamp}"); + + // Level returns Run so the switch exits without sleeping; DetermineRuntimeLevel still throws. + _runtimeStateMock.SetupGet(x => x.Level).Returns(RuntimeLevel.Run); + _runtimeStateMock + .Setup(x => x.DetermineRuntimeLevel()) + .Throws(new InvalidOperationException("db gone")); + + var sut = CreateSut(); + var result = await sut.TryBecomeLeaderAsync(CancellationToken.None); + + Assert.IsFalse(result); + } + + [Test] + public async Task TryBecomeLeaderAsync_WhenCancelledBeforeFirstIteration_ReturnsFalse() + { + var cts = new CancellationTokenSource(); + cts.Cancel(); + + var sut = CreateSut(); + var result = await sut.TryBecomeLeaderAsync(cts.Token); + + Assert.IsFalse(result); + _scopeProviderMock.Verify( + x => x.CreateCoreScope( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Test] + public async Task ReleaseLeadership_WhenClaimMatchesDatabaseValue_ClearsKey() + { + string? capturedClaim = null; + _keyValueServiceMock + .Setup(x => x.GetValue(Constants.Conventions.Migrations.UpgradeLockKey)) + .Returns(() => capturedClaim); + _keyValueServiceMock + .Setup(x => x.SetValue(Constants.Conventions.Migrations.UpgradeLockKey, It.IsAny())) + .Callback((_, v) => capturedClaim = v); + + var sut = CreateSut(); + await sut.TryBecomeLeaderAsync(CancellationToken.None); + + sut.ReleaseLeadership(); + + _keyValueServiceMock.Verify( + x => x.SetValue(Constants.Conventions.Migrations.UpgradeLockKey, string.Empty), + Times.Once); + } + + [Test] + public async Task ReleaseLeadership_WhenDatabaseValueDiffersFromLeaderClaim_DoesNotClearKey() + { + _keyValueServiceMock + .Setup(x => x.GetValue(Constants.Conventions.Migrations.UpgradeLockKey)) + .Returns((string?)null); + + var sut = CreateSut(); + await sut.TryBecomeLeaderAsync(CancellationToken.None); + + // Another server has since taken over the claim. + _keyValueServiceMock + .Setup(x => x.GetValue(Constants.Conventions.Migrations.UpgradeLockKey)) + .Returns($"{OtherMachineIdentifier}|{DateTimeOffset.UtcNow:O}"); + + sut.ReleaseLeadership(); + + _keyValueServiceMock.Verify( + x => x.SetValue(Constants.Conventions.Migrations.UpgradeLockKey, string.Empty), + Times.Never); + } + + [Test] + public void ReleaseLeadership_WhenCalledBeforeTryBecomeLeaderAsync_DoesNothing() + { + var sut = CreateSut(); + sut.ReleaseLeadership(); + + _scopeProviderMock.Verify( + x => x.CreateCoreScope( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Test] + public async Task ReleaseLeadership_WhenCalledTwice_IsIdempotent() + { + string? capturedClaim = null; + _keyValueServiceMock + .Setup(x => x.GetValue(Constants.Conventions.Migrations.UpgradeLockKey)) + .Returns(() => capturedClaim); + _keyValueServiceMock + .Setup(x => x.SetValue(Constants.Conventions.Migrations.UpgradeLockKey, It.IsAny())) + .Callback((_, v) => capturedClaim = v); + + var sut = CreateSut(); + await sut.TryBecomeLeaderAsync(CancellationToken.None); + + sut.ReleaseLeadership(); // Clears claim, sets _leaderClaim = null. + sut.ReleaseLeadership(); // _leaderClaim is null — returns immediately, no scope created. + + // TryBecomeLeaderAsync: 1 scope, first ReleaseLeadership: 1 scope, second: 0 scopes. + _scopeProviderMock.Verify( + x => x.CreateCoreScope( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Exactly(2)); + } + + private MigrationCoordinator CreateSut( + string machineIdentifier = TestMachineIdentifier, + TimeSpan? claimTimeout = null) + { + var machineInfoFactory = Mock.Of( + f => f.GetMachineIdentifier() == machineIdentifier); + + var settings = Options.Create(new UnattendedSettings + { + MigrationClaimTimeout = claimTimeout ?? TimeSpan.FromMinutes(5), + }); + + return new MigrationCoordinator( + _scopeProviderMock.Object, + _keyValueServiceMock.Object, + _runtimeStateMock.Object, + machineInfoFactory, + settings, + NullLogger.Instance); + } + + private void SetupScopeProviderMock() => + _scopeProviderMock + .Setup(x => x.CreateCoreScope( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(Mock.Of()); +} diff --git a/tests/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/Install/UnattendedUpgradeBackgroundServiceTests.cs b/tests/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/Install/UnattendedUpgradeBackgroundServiceTests.cs index af82d268ba60..fd6998b79671 100644 --- a/tests/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/Install/UnattendedUpgradeBackgroundServiceTests.cs +++ b/tests/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/Install/UnattendedUpgradeBackgroundServiceTests.cs @@ -232,21 +232,34 @@ public async Task ExecuteAsync_WhenDetermineRuntimeLevelThrows_SetsBootFailed() private static UnattendedUpgradeBackgroundService CreateSut( IRuntimeState runtimeState, IEventAggregator eventAggregator, - IHostApplicationLifetime? hostApplicationLifetime = null) + IHostApplicationLifetime? hostApplicationLifetime = null, + IMigrationCoordinator? coordinator = null) { var components = new ComponentCollection( () => Enumerable.Empty(), Mock.Of(), NullLogger.Instance); + // Default coordinator acts as the migration leader so existing tests cover the leader path. + coordinator ??= CreateLeaderCoordinator(); + return new UnattendedUpgradeBackgroundService( runtimeState, eventAggregator, components, hostApplicationLifetime ?? CreateMockLifetime().Object, + coordinator, NullLogger.Instance); } + private static IMigrationCoordinator CreateLeaderCoordinator() + { + var mock = new Mock(); + mock.Setup(x => x.TryBecomeLeaderAsync(It.IsAny())) + .ReturnsAsync(true); + return mock.Object; + } + private static Mock CreateMockRuntimeState( RuntimeLevel initialLevel = RuntimeLevel.Upgrading, BootFailedException? initialBootFailedException = null)