From e3bc2566278ee3fb79b5fd962fcf7b9e18cf732e Mon Sep 17 00:00:00 2001 From: Karol Zadora-Przylecki Date: Thu, 3 Apr 2025 18:23:42 -0700 Subject: [PATCH] Retry resource stopping operation --- src/Aspire.Hosting/Dcp/DcpExecutor.cs | 64 ++++++++++++++----- .../Dcp/TestKubernetesService.cs | 49 ++++++++++++++ 2 files changed, 97 insertions(+), 16 deletions(-) diff --git a/src/Aspire.Hosting/Dcp/DcpExecutor.cs b/src/Aspire.Hosting/Dcp/DcpExecutor.cs index 56acf4050c4..5f7177e85fb 100644 --- a/src/Aspire.Hosting/Dcp/DcpExecutor.cs +++ b/src/Aspire.Hosting/Dcp/DcpExecutor.cs @@ -906,7 +906,7 @@ async Task CreateResourceExecutablesAsyncCore(IResource resource, IEnumerable _snapshotBuilder.ToSnapshot((Executable) er.DcpResource, s))).ConfigureAwait(false); + await _executorEvents.PublishAsync(new OnResourceChangedContext(_shutdownCancellation.Token, resourceType, resource, er.DcpResourceName, new ResourceStatus(null, null, null), s => _snapshotBuilder.ToSnapshot((Executable)er.DcpResource, s))).ConfigureAwait(false); } await _executorEvents.PublishAsync(new OnResourceStartingContext(cancellationToken, resourceType, resource, DcpResourceName: null)).ConfigureAwait(false); @@ -1176,7 +1176,7 @@ async Task CreateContainerAsyncCore(AppResource cr, CancellationToken cancellati { // Publish snapshot built from DCP resource. Do this now to populate more values from DCP (URLs, source) to ensure they're // available if the resource isn't immediately started because it's waiting or is configured for explicit start. - await _executorEvents.PublishAsync(new OnResourceChangedContext(_shutdownCancellation.Token, KnownResourceTypes.Container, cr.ModelResource, cr.DcpResourceName, new ResourceStatus(null, null, null), s => _snapshotBuilder.ToSnapshot((Container) cr.DcpResource, s))).ConfigureAwait(false); + await _executorEvents.PublishAsync(new OnResourceChangedContext(_shutdownCancellation.Token, KnownResourceTypes.Container, cr.ModelResource, cr.DcpResourceName, new ResourceStatus(null, null, null), s => _snapshotBuilder.ToSnapshot((Container)cr.DcpResource, s))).ConfigureAwait(false); if (cr.ModelResource.TryGetLastAnnotation(out _)) { @@ -1430,21 +1430,53 @@ private static V1Patch CreatePatch(T obj, Action change) where T : CustomR public async Task StopResourceAsync(IResourceReference resourceReference, CancellationToken cancellationToken) { - var appResource = (AppResource)resourceReference; + _logger.LogDebug("Stopping resource '{ResourceName}'...", resourceReference.DcpResourceName); - V1Patch patch; - switch (appResource.DcpResource) + var result = await DeleteResourceRetryPipeline.ExecuteAsync(async (resourceName, attemptCancellationToken) => { - case Container c: - patch = CreatePatch(c, obj => obj.Spec.Stop = true); - await _kubernetesService.PatchAsync(c, patch, cancellationToken).ConfigureAwait(false); - break; - case Executable e: - patch = CreatePatch(e, obj => obj.Spec.Stop = true); - await _kubernetesService.PatchAsync(e, patch, cancellationToken).ConfigureAwait(false); - break; - default: - throw new InvalidOperationException($"Unexpected resource type: {appResource.DcpResource.GetType().FullName}"); + var appResource = (AppResource)resourceReference; + + V1Patch patch; + switch (appResource.DcpResource) + { + case Container c: + patch = CreatePatch(c, obj => obj.Spec.Stop = true); + await _kubernetesService.PatchAsync(c, patch, attemptCancellationToken).ConfigureAwait(false); + var cu = await _kubernetesService.GetAsync(c.Metadata.Name, cancellationToken: attemptCancellationToken).ConfigureAwait(false); + if (cu.Status?.State == ContainerState.Exited) + { + _logger.LogDebug("Container '{ResourceName}' was stopped.", resourceReference.DcpResourceName); + return true; + } + else + { + _logger.LogDebug("Container '{ResourceName}' is still running; trying again to stop it...", resourceReference.DcpResourceName); + return false; + } + + case Executable e: + patch = CreatePatch(e, obj => obj.Spec.Stop = true); + await _kubernetesService.PatchAsync(e, patch, attemptCancellationToken).ConfigureAwait(false); + var eu = await _kubernetesService.GetAsync(e.Metadata.Name, cancellationToken: attemptCancellationToken).ConfigureAwait(false); + if (eu.Status?.State == ExecutableState.Finished || eu.Status?.State == ExecutableState.Terminated) + { + _logger.LogDebug("Executable '{ResourceName}' was stopped.", resourceReference.DcpResourceName); + return true; + } + else + { + _logger.LogDebug("Executable '{ResourceName}' is still running; trying again to stop it...", resourceReference.DcpResourceName); + return false; + } + + default: + throw new InvalidOperationException($"Unexpected resource type: {appResource.DcpResource.GetType().FullName}"); + } + }, resourceReference.DcpResourceName, cancellationToken).ConfigureAwait(false); + + if (!result) + { + throw new InvalidOperationException($"Failed to stop resource '{resourceReference.DcpResourceName}'."); } } @@ -1509,7 +1541,7 @@ async Task EnsureResourceDeletedAsync(string resourceName) where T : CustomRe { _logger.LogDebug("Ensuring '{ResourceName}' is deleted.", resourceName); - var result = await DeleteResourceRetryPipeline.ExecuteAsync(async (resourceName, attemptCancellationToken) => + var result = await DeleteResourceRetryPipeline.ExecuteAsync(async (resourceName, attemptCancellationToken) => { string? uid = null; diff --git a/tests/Aspire.Hosting.Tests/Dcp/TestKubernetesService.cs b/tests/Aspire.Hosting.Tests/Dcp/TestKubernetesService.cs index 1c80ae7e148..d97cc5a34b0 100644 --- a/tests/Aspire.Hosting.Tests/Dcp/TestKubernetesService.cs +++ b/tests/Aspire.Hosting.Tests/Dcp/TestKubernetesService.cs @@ -11,6 +11,7 @@ using System.Text; using k8s.Models; using k8s.Autorest; +using Json.Patch; namespace Aspire.Hosting.Tests.Dcp; @@ -168,6 +169,54 @@ public Task GetLogStreamAsync(T obj, string logStreamType, bool? foll public Task PatchAsync(T obj, V1Patch patch, CancellationToken cancellationToken = default) where T : CustomResource { + // Not a complete implementation, but Aspire is using patching only to stop resources, + // so this is good enough. + + if (patch.Type == V1Patch.PatchType.JsonPatch) + { + Json.Patch.JsonPatch jsonPatch = (Json.Patch.JsonPatch)patch.Content; + + var res = CreatedResources.OfType().FirstOrDefault(r => + r.Metadata.Name == obj.Metadata.Name && + string.Equals(r.Metadata.NamespaceProperty, obj.Metadata.NamespaceProperty) + ); + if (res == null) + { + throw new ArgumentException($"Resource '{obj.Metadata.NamespaceProperty}/{obj.Metadata.Name}' not found"); + } + + var result = jsonPatch.Apply(res); + + if (res is Executable exe && result is Executable eu) + { + if (eu.Spec.Stop == true) + { + exe.Spec.Stop = true; + if (exe.Status is null) + { + exe.Status = new ExecutableStatus(); + } + exe.Status.State = ExecutableState.Finished; + } + } + + if (res is Container ctr && result is Container cu) + { + if (cu.Spec.Stop == true) + { + ctr.Spec.Stop = true; + if (ctr.Status is null) + { + ctr.Status = new ContainerStatus(); + } + ctr.Status.State = ContainerState.Exited; + } + } + + return Task.FromResult(res); + } + + // Fall back to doing noting. return Task.FromResult(obj); }