Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
319 changes: 175 additions & 144 deletions .github/skills/cli-e2e-testing/SKILL.md

Large diffs are not rendered by default.

33 changes: 26 additions & 7 deletions .github/skills/deployment-e2e-testing/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,9 @@ Key differences from CLI E2E tests:

### Core Classes

- **`DeploymentE2ETestHelpers`** (`Helpers/DeploymentE2ETestHelpers.cs`): Terminal automation helpers
- **`DeploymentE2ETestHelpers`** (`Helpers/DeploymentE2ETestHelpers.cs`): Terminal factory and environment helpers
- **`DeploymentE2EAutomatorHelpers`** (`Helpers/DeploymentE2EAutomatorHelpers.cs`): Async extension methods on `Hex1bTerminalAutomator` for deployment scenarios
- **`Hex1bAutomatorTestHelpers`** (shared): Common async extension methods on `Hex1bTerminalAutomator` (`WaitForSuccessPromptAsync`, `AspireNewAsync`, etc.)
- **`AzureAuthenticationHelpers`** (`Helpers/AzureAuthenticationHelpers.cs`): Azure auth and resource naming
- **`DeploymentReporter`** (`Helpers/DeploymentReporter.cs`): GitHub step summary reporting
- **`SequenceCounter`** (`Helpers/SequenceCounter.cs`): Prompt tracking (same as CLI E2E)
Expand Down Expand Up @@ -92,14 +94,18 @@ public sealed class MyDeploymentTests(ITestOutputHelper output)
var pendingRun = terminal.RunAsync(TestContext.Current.CancellationToken);

var counter = new SequenceCounter();
var sequenceBuilder = new Hex1bTerminalInputSequenceBuilder();
var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500));

// Build sequence: prepare, create project, deploy, verify
sequenceBuilder.PrepareEnvironment(workspace, counter);
// ... add deployment steps ...
await auto.PrepareEnvironmentAsync(workspace, counter);
if (DeploymentE2ETestHelpers.IsRunningInCI)
{
await auto.SourceAspireCliEnvironmentAsync(counter);
}
await auto.AspireNewAsync("MyProject", counter, useRedisCache: false);
// ... deployment steps ...

var sequence = sequenceBuilder.Build();
await sequence.ApplyAsync(terminal, TestContext.Current.CancellationToken);
await auto.TypeAsync("exit");
await auto.EnterAsync();
await pendingRun;

// 4. Report success
Expand Down Expand Up @@ -127,6 +133,19 @@ public sealed class MyDeploymentTests(ITestOutputHelper output)
}
```

## Extension Methods

### DeploymentE2EAutomatorHelpers Extensions on Hex1bTerminalAutomator

| Method | Description |
|--------|-------------|
| `PrepareEnvironmentAsync(workspace, counter)` | Sets up the terminal environment with custom prompt and workspace directory |
| `InstallAspireCliFromPullRequestAsync(prNumber, counter)` | Downloads and installs the Aspire CLI from a PR build artifact |
| `InstallAspireCliReleaseAsync(counter)` | Installs the latest released Aspire CLI |
| `SourceAspireCliEnvironmentAsync(counter)` | Adds `~/.aspire/bin` to PATH so the `aspire` command is available |

These extend `Hex1bTerminalAutomator` and are used alongside the shared `Hex1bAutomatorTestHelpers` methods (`WaitForSuccessPromptAsync`, `AspireNewAsync`, etc.) documented in the [CLI E2E Testing Skill](../cli-e2e-testing/SKILL.md).

## Azure Authentication

### In CI (GitHub Actions)
Expand Down
270 changes: 92 additions & 178 deletions tests/Aspire.Cli.EndToEnd.Tests/AgentCommandTests.cs

Large diffs are not rendered by default.

166 changes: 58 additions & 108 deletions tests/Aspire.Cli.EndToEnd.Tests/BannerTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -25,48 +25,32 @@ public async Task Banner_DisplayedOnFirstRun()

var pendingRun = terminal.RunAsync(TestContext.Current.CancellationToken);

// Pattern to detect the ASPIRE banner text (the welcome message)
// The banner displays "Welcome to the" followed by ASCII art "ASPIRE"
var bannerPattern = new CellPatternSearcher()
.Find("Welcome to the");

// Pattern to detect the telemetry notice (shown on first run)
var telemetryNoticePattern = new CellPatternSearcher()
.Find("Telemetry");

var counter = new SequenceCounter();
var sequenceBuilder = new Hex1bTerminalInputSequenceBuilder();
var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500));

sequenceBuilder.PrepareDockerEnvironment(counter, workspace);
await auto.PrepareDockerEnvironmentAsync(counter, workspace);

sequenceBuilder.InstallAspireCliInDocker(installMode, counter);
await auto.InstallAspireCliInDockerAsync(installMode, counter);

// Delete the first-time use sentinel file to simulate first run
// The sentinel is stored at ~/.aspire/cli/cli.firstUseSentinel
// Using 'aspire cache clear' because it's not an informational
// command and so will show the banner.
sequenceBuilder
.ClearFirstRunSentinel(counter)
.VerifySentinelDeleted(counter)
.ClearScreen(counter)
.Type("aspire cache clear")
.Enter()
.WaitUntil(s =>
{
// Verify the banner appears
var hasBanner = bannerPattern.Search(s).Count > 0;
var hasTelemetryNotice = telemetryNoticePattern.Search(s).Count > 0;

// Both should appear on first run
return hasBanner && hasTelemetryNotice;
}, TimeSpan.FromSeconds(30))
.WaitForSuccessPrompt(counter)
.Type("exit")
.Enter();

var sequence = sequenceBuilder.Build();

await sequence.ApplyAsync(terminal, TestContext.Current.CancellationToken);
await auto.TypeAsync("rm -f ~/.aspire/cli/cli.firstUseSentinel");
await auto.EnterAsync();
await auto.WaitForSuccessPromptAsync(counter);
await auto.TypeAsync("test ! -f ~/.aspire/cli/cli.firstUseSentinel");
await auto.EnterAsync();
await auto.WaitForSuccessPromptAsync(counter);
await auto.ClearScreenAsync(counter);
await auto.TypeAsync("aspire cache clear");
await auto.EnterAsync();
await auto.WaitUntilAsync(
s => s.ContainsText("Welcome to the") && s.ContainsText("Telemetry"),
timeout: TimeSpan.FromSeconds(30), description: "waiting for banner and telemetry notice on first run");
await auto.WaitForSuccessPromptAsync(counter);
await auto.TypeAsync("exit");
await auto.EnterAsync();

await pendingRun;
}
Expand All @@ -82,43 +66,23 @@ public async Task Banner_DisplayedWithExplicitFlag()

var pendingRun = terminal.RunAsync(TestContext.Current.CancellationToken);

// Pattern to detect the ASPIRE banner welcome text
// The banner displays "Welcome to the" followed by ASCII art "ASPIRE"
var bannerPattern = new CellPatternSearcher()
.Find("Welcome to the");

// Pattern to detect version info in the banner
// The format is "CLI — version X.Y.Z"
var versionPattern = new CellPatternSearcher()
.Find("CLI");

var counter = new SequenceCounter();
var sequenceBuilder = new Hex1bTerminalInputSequenceBuilder();
var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500));

sequenceBuilder.PrepareDockerEnvironment(counter, workspace);
await auto.PrepareDockerEnvironmentAsync(counter, workspace);

sequenceBuilder.InstallAspireCliInDocker(installMode, counter);
await auto.InstallAspireCliInDockerAsync(installMode, counter);

// Clear screen to have a clean slate for pattern matching
sequenceBuilder
.ClearScreen(counter)
.Type("aspire --banner")
.Enter()
.WaitUntil(s =>
{
// Verify the banner appears with version info
var hasBanner = bannerPattern.Search(s).Count > 0;
var hasVersion = versionPattern.Search(s).Count > 0;

return hasBanner && hasVersion;
}, TimeSpan.FromSeconds(30))
.WaitForSuccessPrompt(counter)
.Type("exit")
.Enter();

var sequence = sequenceBuilder.Build();

await sequence.ApplyAsync(terminal, TestContext.Current.CancellationToken);
await auto.ClearScreenAsync(counter);
await auto.TypeAsync("aspire --banner");
await auto.EnterAsync();
await auto.WaitUntilAsync(
s => s.ContainsText("Welcome to the") && s.ContainsText("CLI"),
timeout: TimeSpan.FromSeconds(30), description: "waiting for banner with version info");
await auto.WaitForSuccessPromptAsync(counter);
await auto.TypeAsync("exit");
await auto.EnterAsync();

await pendingRun;
}
Expand All @@ -135,55 +99,41 @@ public async Task Banner_NotDisplayedWithNoLogoFlag()

var pendingRun = terminal.RunAsync(TestContext.Current.CancellationToken);

// Pattern to detect the ASPIRE banner - should NOT appear
// The banner displays "Welcome to the" followed by ASCII art "ASPIRE"
var bannerPattern = new CellPatternSearcher()
.Find("Welcome to the");

// Pattern to detect the help text (confirms command completed)
var helpPattern = new CellPatternSearcher()
.Find("Commands:");

var counter = new SequenceCounter();
var sequenceBuilder = new Hex1bTerminalInputSequenceBuilder();
var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500));

sequenceBuilder.PrepareDockerEnvironment(counter, workspace);
await auto.PrepareDockerEnvironmentAsync(counter, workspace);

sequenceBuilder.InstallAspireCliInDocker(installMode, counter);
await auto.InstallAspireCliInDockerAsync(installMode, counter);

// Delete the first-time use sentinel file to simulate first run,
// but use --nologo to suppress the banner
sequenceBuilder
.ClearFirstRunSentinel(counter)
.ClearScreen(counter)
.Type("aspire --nologo --help")
.Enter()
.WaitUntil(s =>
await auto.TypeAsync("rm -f ~/.aspire/cli/cli.firstUseSentinel");
await auto.EnterAsync();
await auto.WaitForSuccessPromptAsync(counter);
await auto.ClearScreenAsync(counter);
await auto.TypeAsync("aspire --nologo --help");
await auto.EnterAsync();
await auto.WaitUntilAsync(s =>
{
// Wait for help output to confirm command completed
if (!s.ContainsText("Commands:"))
{
return false;
}

// Verify the banner does NOT appear
if (s.ContainsText("Welcome to the"))
{
// Wait for help output to confirm command completed
var hasHelp = helpPattern.Search(s).Count > 0;
if (!hasHelp)
{
return false;
}

// Verify the banner does NOT appear
var hasBanner = bannerPattern.Search(s).Count > 0;
if (hasBanner)
{
throw new InvalidOperationException(
"Unexpected banner displayed when --nologo flag was used!");
}

return true;
}, TimeSpan.FromSeconds(30))
.WaitForSuccessPrompt(counter)
.Type("exit")
.Enter();

var sequence = sequenceBuilder.Build();

await sequence.ApplyAsync(terminal, TestContext.Current.CancellationToken);
throw new InvalidOperationException(
"Unexpected banner displayed when --nologo flag was used!");
}

return true;
}, timeout: TimeSpan.FromSeconds(30), description: "waiting for help output without banner");
await auto.WaitForSuccessPromptAsync(counter);
await auto.TypeAsync("exit");
await auto.EnterAsync();

await pendingRun;
}
Expand Down
58 changes: 26 additions & 32 deletions tests/Aspire.Cli.EndToEnd.Tests/BundleSmokeTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -27,44 +27,38 @@ public async Task CreateAndRunAspireStarterProjectWithBundle()

var pendingRun = terminal.RunAsync(TestContext.Current.CancellationToken);

// Verify the dashboard is actually reachable, not just that the URL was printed.
// When the dashboard path bug was present, the URL appeared on screen but curling
// it returned connection refused because the dashboard process failed to start.
var waitForDashboardCurlSuccess = new CellPatternSearcher()
.Find("dashboard-http-200");

var counter = new SequenceCounter();
var sequenceBuilder = new Hex1bTerminalInputSequenceBuilder();
var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500));

await auto.PrepareDockerEnvironmentAsync(counter, workspace);
await auto.InstallAspireCliInDockerAsync(installMode, counter);

await auto.AspireNewAsync("BundleStarterApp", counter);

sequenceBuilder.PrepareDockerEnvironment(counter, workspace);
// Start AppHost in detached mode and capture JSON output
await auto.TypeAsync("aspire start --format json | tee /tmp/aspire-detach.json");
await auto.EnterAsync();
await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(3));

sequenceBuilder.InstallAspireCliInDocker(installMode, counter);
// Verify the dashboard is reachable by extracting the URL from the detach output
// and curling it. Extract just the base URL (https://localhost:PORT) using sed, which is
// portable across macOS (BSD) and Linux (GNU) unlike grep -oP.
await auto.TypeAsync("DASHBOARD_URL=$(sed -n 's/.*\"dashboardUrl\"[[:space:]]*:[[:space:]]*\"\\(https:\\/\\/localhost:[0-9]*\\).*/\\1/p' /tmp/aspire-detach.json | head -1)");
await auto.EnterAsync();
await auto.WaitForSuccessPromptAsync(counter);

sequenceBuilder.AspireNew("BundleStarterApp", counter)
// Start AppHost in detached mode and capture JSON output
.Type("aspire start --format json | tee /tmp/aspire-detach.json")
.Enter()
.WaitForSuccessPrompt(counter, TimeSpan.FromMinutes(3))
// Verify the dashboard is reachable by extracting the URL from the detach output
// and curling it. Extract just the base URL (https://localhost:PORT) using sed, which is
// portable across macOS (BSD) and Linux (GNU) unlike grep -oP.
.Type("DASHBOARD_URL=$(sed -n 's/.*\"dashboardUrl\"[[:space:]]*:[[:space:]]*\"\\(https:\\/\\/localhost:[0-9]*\\).*/\\1/p' /tmp/aspire-detach.json | head -1)")
.Enter()
.WaitForSuccessPrompt(counter)
.Type("curl -ksSL -o /dev/null -w 'dashboard-http-%{http_code}' \"$DASHBOARD_URL\" || echo 'dashboard-http-failed'")
.Enter()
.WaitUntil(s => waitForDashboardCurlSuccess.Search(s).Count > 0, TimeSpan.FromSeconds(15))
.WaitForSuccessPrompt(counter)
// Clean up: use aspire stop to gracefully shut down the detached AppHost.
.Type("aspire stop")
.Enter()
.WaitForSuccessPrompt(counter)
.Type("exit")
.Enter();
await auto.TypeAsync("curl -ksSL -o /dev/null -w 'dashboard-http-%{http_code}' \"$DASHBOARD_URL\" || echo 'dashboard-http-failed'");
await auto.EnterAsync();
await auto.WaitUntilTextAsync("dashboard-http-200", timeout: TimeSpan.FromSeconds(15));
await auto.WaitForSuccessPromptAsync(counter);

var sequence = sequenceBuilder.Build();
// Clean up: use aspire stop to gracefully shut down the detached AppHost.
await auto.TypeAsync("aspire stop");
await auto.EnterAsync();
await auto.WaitForSuccessPromptAsync(counter);

await sequence.ApplyAsync(terminal, TestContext.Current.CancellationToken);
await auto.TypeAsync("exit");
await auto.EnterAsync();

await pendingRun;
}
Expand Down
Loading
Loading