-
Notifications
You must be signed in to change notification settings - Fork 976
Support AppHost startup timeout env var #16686
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
6f1aeef
aedb4cb
4d7c01a
96247d6
44a3be1
dcee00a
ec0437e
32af322
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
|
|
||
| using System.Globalization; | ||
| using Aspire.Cli.Interaction; | ||
| using Aspire.Cli.Resources; | ||
| using Microsoft.Extensions.Configuration; | ||
|
|
||
| namespace Aspire.Cli.Commands; | ||
|
|
||
| internal static class AppHostStartupTimeout | ||
| { | ||
| public static bool TryGetTimeoutSeconds(IConfiguration configuration, IInteractionService interactionService, out int timeoutSeconds) | ||
| { | ||
| timeoutSeconds = WaitCommand.DefaultTimeoutSeconds; | ||
|
|
||
| var configuredTimeout = configuration[CliConfigNames.AppHostStartupTimeout]; | ||
| if (string.IsNullOrWhiteSpace(configuredTimeout)) | ||
| { | ||
| return true; | ||
| } | ||
|
|
||
| if (int.TryParse(configuredTimeout, NumberStyles.None, CultureInfo.InvariantCulture, out var parsedTimeout) && | ||
| parsedTimeout > 0) | ||
| { | ||
| timeoutSeconds = parsedTimeout; | ||
| return true; | ||
| } | ||
|
|
||
| interactionService.DisplayError(string.Format( | ||
| CultureInfo.CurrentCulture, | ||
| RunCommandStrings.InvalidAppHostStartupTimeoutEnvironmentVariable, | ||
| CliConfigNames.AppHostStartupTimeout)); | ||
| return false; | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -72,9 +72,12 @@ internal sealed class RunCommand : BaseCommand | |
| private readonly FileLoggerProvider _fileLoggerProvider; | ||
| private readonly ICliHostEnvironment _hostEnvironment; | ||
| private readonly ProfilingTelemetry _profilingTelemetry; | ||
| private readonly TimeProvider _timeProvider; | ||
| private bool _isDetachMode; | ||
| private const int MaxDisplayedAppHostStartupOutputLines = 80; | ||
|
|
||
| private static readonly TimeSpan s_appHostStartupCancellationTimeout = TimeSpan.FromSeconds(5); | ||
|
|
||
| // Guest AppHosts can bring up the temporary server/backchannel and then fail immediately | ||
| // afterward when the guest startup process hits a syntax, pre-execute, or model validation | ||
| // error. Keep guest AppHost startup waits alive briefly so those failures are reported instead of hidden. | ||
|
|
@@ -107,7 +110,8 @@ public RunCommand( | |
| AppHostLauncher appHostLauncher, | ||
| FileLoggerProvider fileLoggerProvider, | ||
| ICliHostEnvironment hostEnvironment, | ||
| ProfilingTelemetry profilingTelemetry) | ||
| ProfilingTelemetry profilingTelemetry, | ||
| TimeProvider timeProvider) | ||
| : base("run", RunCommandStrings.Description, features, updateNotifier, executionContext, interactionService, telemetry) | ||
| { | ||
| _runner = runner; | ||
|
|
@@ -123,6 +127,7 @@ public RunCommand( | |
| _fileLoggerProvider = fileLoggerProvider; | ||
| _hostEnvironment = hostEnvironment; | ||
| _profilingTelemetry = profilingTelemetry; | ||
| _timeProvider = timeProvider; | ||
|
|
||
| Options.Add(s_detachOption); | ||
| Options.Add(s_noBuildOption); | ||
|
|
@@ -168,10 +173,15 @@ protected override async Task<CommandResult> ExecuteAsync(ParseResult parseResul | |
| return CommandResult.Failure(CliExitCodes.InvalidCommand, RunCommandStrings.NoBuildNotSupportedWithWatchMode); | ||
| } | ||
|
|
||
| if (!AppHostStartupTimeout.TryGetTimeoutSeconds(_configuration, InteractionService, out var timeoutSeconds)) | ||
| { | ||
| return CommandResult.Failure(CliExitCodes.InvalidCommand); | ||
| } | ||
|
|
||
| // Handle detached mode - spawn child process and exit | ||
| if (detach) | ||
| { | ||
| return await ExecuteDetachedAsync(parseResult, passedAppHostProjectFile, isExtensionHost, cancellationToken); | ||
| return await ExecuteDetachedAsync(parseResult, passedAppHostProjectFile, isExtensionHost, timeoutSeconds, cancellationToken); | ||
| } | ||
|
|
||
| // A user may run `aspire run` in an Aspire terminal in VS Code. In this case, intercept and prompt | ||
|
|
@@ -277,16 +287,29 @@ protected override async Task<CommandResult> ExecuteAsync(ParseResult parseResul | |
|
|
||
| // Start the project run as a pending task - we'll handle UX while it runs | ||
| Task<int> pendingRun; | ||
| var startupTimeout = TimeSpan.FromSeconds(timeoutSeconds); | ||
| var startupStartTimestamp = _timeProvider.GetTimestamp(); | ||
| using var runCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. In the timeout path, if |
||
| using (_profilingTelemetry.StartRunAppHostStartProject(project.LanguageId, noBuild, waitForDebugger)) | ||
| { | ||
| pendingRun = project.RunAsync(context, cancellationToken); | ||
| pendingRun = project.RunAsync(context, runCancellationTokenSource.Token); | ||
| } | ||
|
|
||
| // Wait for the build to complete first (project handles its own build status spinners) | ||
| bool buildSuccess; | ||
| using (var waitForBuildActivity = _profilingTelemetry.StartRunAppHostWaitForBuild()) | ||
| { | ||
| buildSuccess = await buildCompletionSource.Task.WaitAsync(cancellationToken); | ||
| try | ||
| { | ||
| buildSuccess = await buildCompletionSource.Task.WaitAsync(GetRemainingStartupTimeout(startupStartTimestamp, startupTimeout), _timeProvider, cancellationToken); | ||
| } | ||
| catch (TimeoutException) | ||
| { | ||
| runActivity?.SetTag(TelemetryConstants.Tags.ErrorType, "startup_timeout"); | ||
| await CancelAppHostStartupAsync(runCancellationTokenSource, pendingRun).ConfigureAwait(false); | ||
| return CreateStartupTimeoutResult(timeoutSeconds); | ||
| } | ||
|
|
||
| waitForBuildActivity.SetAppHostBuildSuccess(buildSuccess); | ||
| } | ||
| if (!buildSuccess) | ||
|
|
@@ -312,13 +335,25 @@ protected override async Task<CommandResult> ExecuteAsync(ParseResult parseResul | |
|
|
||
| try | ||
| { | ||
| var startup = await WaitForAppHostStartupAsync( | ||
| pendingRun, | ||
| backchannelCompletionSource, | ||
| logCaptureCancellationSource, | ||
| context.OutputCollector, | ||
| appHostStartupOutputStartIndex, | ||
| cancellationToken).ConfigureAwait(false); | ||
| AppHostStartupResult startup; | ||
| try | ||
| { | ||
| startup = await WaitForAppHostStartupAsync( | ||
| pendingRun, | ||
| backchannelCompletionSource, | ||
| logCaptureCancellationSource, | ||
| context.OutputCollector, | ||
| appHostStartupOutputStartIndex, | ||
| startupStartTimestamp, | ||
| startupTimeout, | ||
| cancellationToken).ConfigureAwait(false); | ||
| } | ||
| catch (TimeoutException) | ||
| { | ||
| runActivity?.SetTag(TelemetryConstants.Tags.ErrorType, "startup_timeout"); | ||
| await CancelAppHostStartupAsync(runCancellationTokenSource, pendingRun).ConfigureAwait(false); | ||
| return CreateStartupTimeoutResult(timeoutSeconds); | ||
| } | ||
|
|
||
| var backchannel = startup.Backchannel; | ||
| var dashboardUrls = startup.DashboardUrls; | ||
|
|
@@ -471,6 +506,15 @@ await InteractionService.DisplayLiveAsync(BuildLiveRenderable(), async updateTar | |
| : CommandResult.FromExitCode(exitCode); | ||
| } | ||
| } | ||
| catch (OperationCanceledException ex) when (ex.CancellationToken == runCancellationTokenSource.Token && cancellationToken.IsCancellationRequested) | ||
| { | ||
| runActivity?.SetTag(TelemetryConstants.Tags.ErrorType, "canceled"); | ||
|
|
||
| // The user cancelled (e.g. Ctrl+C); the linked CTS we passed to project.RunAsync | ||
| // propagated the cancellation and the OCE bubbled out with the linked token. | ||
| // Treat as successful exit since the user intentionally stopped the AppHost. | ||
| return CommandResult.Cancelled(CliExitCodes.Success); | ||
| } | ||
| finally | ||
| { | ||
| logCaptureCancellationSource.Cancel(); | ||
|
|
@@ -643,10 +687,12 @@ private async Task<AppHostStartupResult> WaitForAppHostStartupAsync( | |
| CancellationTokenSource logCaptureCancellationSource, | ||
| OutputCollector? outputCollector, | ||
| int appHostStartupOutputStartIndex, | ||
| long startupStartTimestamp, | ||
| TimeSpan startupTimeout, | ||
| CancellationToken cancellationToken) | ||
| { | ||
| using var startupCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); | ||
| var happyPathTask = RunStartupHappyPathAsync(backchannelCompletionSource, logCaptureCancellationSource, pendingRun, startupCts.Token); | ||
| var happyPathTask = RunStartupHappyPathAsync(backchannelCompletionSource, logCaptureCancellationSource, pendingRun, startupStartTimestamp, startupTimeout, startupCts.Token); | ||
|
|
||
| // Race the startup readiness signal against the AppHost system task. The AppHost | ||
| // system is owned by the project and tears itself down (via an internal escalation | ||
|
|
@@ -690,6 +736,13 @@ private async Task<AppHostStartupResult> WaitForAppHostStartupAsync( | |
| DisplayRecentAppHostStartupOutput(InteractionService, outputCollector, appHostStartupOutputStartIndex); | ||
| throw; | ||
| } | ||
| catch (TimeoutException) | ||
| { | ||
| // Bubble startup-timeout signal up to ExecuteAsync so it can cancel the run | ||
| // and emit the localized timeout guidance. Must not be wrapped by the generic | ||
| // catch below. | ||
| throw; | ||
| } | ||
| catch (Exception ex) when (ex is not OperationCanceledException) | ||
| { | ||
| var failureMessage = string.Format(CultureInfo.CurrentCulture, InteractionServiceStrings.UnexpectedErrorOccurred, ex.Message); | ||
|
|
@@ -720,14 +773,16 @@ private async Task<AppHostStartupResult> RunStartupHappyPathAsync( | |
| TaskCompletionSource<IAppHostCliBackchannel> backchannelCompletionSource, | ||
| CancellationTokenSource logCaptureCancellationSource, | ||
| Task<int> pendingRun, | ||
| long startupStartTimestamp, | ||
| TimeSpan startupTimeout, | ||
| CancellationToken cancellationToken) | ||
| { | ||
| IAppHostCliBackchannel backchannel; | ||
| using (var waitForBackchannelActivity = _profilingTelemetry.StartRunAppHostWaitForBackchannel()) | ||
| { | ||
| backchannel = await InteractionService.ShowStatusAsync( | ||
| RunCommandStrings.ConnectingToAppHost, | ||
| async () => await backchannelCompletionSource.Task.WaitAsync(cancellationToken).ConfigureAwait(false)); | ||
| async () => await backchannelCompletionSource.Task.WaitAsync(GetRemainingStartupTimeout(startupStartTimestamp, startupTimeout), _timeProvider, cancellationToken).ConfigureAwait(false)); | ||
| waitForBackchannelActivity.SetAppHostBackchannelConnected(true); | ||
| } | ||
|
|
||
|
|
@@ -1027,12 +1082,12 @@ public void ProcessResourceState(RpcResourceState resourceState, Action<string, | |
| /// with the poll delay. Shows exit code and log file path. | ||
| /// Returns <see cref="CliExitCodes.FailedToDotnetRunAppHost"/>.</item> | ||
| /// <item><b>Timeout waiting for backchannel</b>: The auxiliary backchannel socket doesn't appear | ||
| /// within 120 seconds. The child process is killed. Shows timeout message and log file path. | ||
| /// within the configured startup timeout. The child process is killed. Shows timeout message and log file path. | ||
| /// Returns <see cref="CliExitCodes.FailedToDotnetRunAppHost"/>.</item> | ||
| /// </list> | ||
| /// <para>On any failure, the log file path is displayed so the user can investigate.</para> | ||
| /// </remarks> | ||
| private Task<CommandResult> ExecuteDetachedAsync(ParseResult parseResult, FileInfo? passedAppHostProjectFile, bool isExtensionHost, CancellationToken cancellationToken) | ||
| private Task<CommandResult> ExecuteDetachedAsync(ParseResult parseResult, FileInfo? passedAppHostProjectFile, bool isExtensionHost, int timeoutSeconds, CancellationToken cancellationToken) | ||
| { | ||
| var format = parseResult.GetValue(AppHostLauncher.s_formatOption); | ||
| var isolated = parseResult.GetValue(AppHostLauncher.s_isolatedOption); | ||
|
|
@@ -1056,10 +1111,58 @@ private Task<CommandResult> ExecuteDetachedAsync(ParseResult parseResult, FileIn | |
| isolated, | ||
| isExtensionHost, | ||
| waitForDebugger, | ||
| timeoutSeconds, | ||
| globalArgs, | ||
| additionalArgs, | ||
| stopAfterLaunchDelay, | ||
| cancellationToken); | ||
| } | ||
|
|
||
| private TimeSpan GetRemainingStartupTimeout(long startupStartTimestamp, TimeSpan startupTimeout) | ||
| { | ||
| var elapsed = _timeProvider.GetElapsedTime(startupStartTimestamp); | ||
| return elapsed >= startupTimeout ? TimeSpan.Zero : startupTimeout - elapsed; | ||
| } | ||
|
|
||
| private async Task CancelAppHostStartupAsync(CancellationTokenSource runCancellationTokenSource, Task<int> pendingRun) | ||
| { | ||
| runCancellationTokenSource.Cancel(); | ||
|
|
||
| try | ||
| { | ||
| await pendingRun.WaitAsync(s_appHostStartupCancellationTimeout, _timeProvider).ConfigureAwait(false); | ||
| } | ||
| catch (OperationCanceledException) when (runCancellationTokenSource.IsCancellationRequested) | ||
| { | ||
| } | ||
| catch (TimeoutException ex) | ||
| { | ||
| _logger.LogDebug(ex, "Timed out waiting for AppHost startup cancellation to complete."); | ||
| _ = ObserveAppHostRunFailureAsync(pendingRun); | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| _logger.LogDebug(ex, "AppHost run failed after startup cancellation."); | ||
| } | ||
| } | ||
|
|
||
| private async Task ObserveAppHostRunFailureAsync(Task<int> pendingRun) | ||
| { | ||
| try | ||
| { | ||
| await pendingRun.ConfigureAwait(false); | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| _logger.LogDebug(ex, "AppHost run failed after startup cancellation timeout."); | ||
| } | ||
| } | ||
|
|
||
| private static CommandResult CreateStartupTimeoutResult(int timeoutSeconds) | ||
| { | ||
| return CommandResult.Failure( | ||
| CliExitCodes.FailedToDotnetRunAppHost, | ||
| string.Format(CultureInfo.CurrentCulture, RunCommandStrings.TimeoutWaitingForAppHost, timeoutSeconds, CliConfigNames.AppHostStartupTimeout)); | ||
| } | ||
|
|
||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The default for AppHost startup is coupled to
WaitCommand.DefaultTimeoutSeconds, but these are semantically distinct timeouts:WaitCommand.DefaultTimeoutSecondsis the default foraspire wait --timeout(waiting for a resource to reach a state), whereas this one controls how longaspire run/aspire startwill wait for the AppHost to build + connect the backchannel. A future change to the wait command's default would silently shift AppHost startup behavior across the CLI. Consider a dedicatedAppHostStartupTimeout.DefaultTimeoutSecondsso the two defaults can evolve independently.