Add E2E deployment test for React + ASP.NET Core to Azure App Service

.github/workflows/deployment-cleanup.yml

@@ -1,9 +1,13 @@
 # Cleanup workflow for deployment test resources
 #
 # Uses a mark-and-sweep approach:
-# 1. Mark: Tag untagged rg-aspire-* RGs with 'deployment-test-first-seen' timestamp
+# 1. Mark: Tag untagged deployment test RGs with 'deployment-test-first-seen' timestamp
 # 2. Sweep: Delete RGs where 'deployment-test-first-seen' is older than threshold
 #
+# Targets resource groups with prefixes:
+# - rg-aspire-* (legacy naming)
+# - e2e-* (current naming)
+#
 # This ensures RGs are only deleted after they've been seen for the full retention period.
 #
 name: Deployment Environment Cleanup
@@ -82,11 +86,13 @@ jobs:
           NOW_ISO=$(date -u +%Y-%m-%dT%H:%M:%SZ)
           MAX_AGE_SECONDS=$((MAX_AGE_HOURS * 3600))
 
-          # List all resource groups matching rg-aspire-* prefix
-          RG_LIST=$(az group list --query "[?starts_with(name, 'rg-aspire-')].name" -o tsv || echo "")
+          # List all resource groups matching deployment test prefixes
+          # Current naming: e2e-* (e.g., e2e-starter-12345678-1)
+          # Legacy naming: rg-aspire-* (may still exist from older tests)
+          RG_LIST=$(az group list --query "[?starts_with(name, 'e2e-') || starts_with(name, 'rg-aspire-')].name" -o tsv || echo "")
 
           if [ -z "$RG_LIST" ]; then
-            echo "No resource groups found matching 'rg-aspire-*' prefix."
+            echo "No resource groups found matching deployment test prefixes (e2e-* or rg-aspire-*)."
             echo "✅ No resource groups found." >> $GITHUB_STEP_SUMMARY
             echo "marked_count=0" >> $GITHUB_OUTPUT
             echo "deleted_count=0" >> $GITHUB_OUTPUT

.github/workflows/deployment-test-command.yml

@@ -80,20 +80,6 @@ jobs:
             core.setOutput('head_sha', pr.head.sha);
             core.setOutput('head_ref', pr.head.ref);
 
-      - name: Acknowledge command
-        if: steps.check_membership.outputs.is_member == 'true'
-        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
-        with:
-          script: |
-            const prNumber = ${{ steps.pr.outputs.number }};
-
-            await github.rest.issues.createComment({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: context.issue.number,
-              body: `🚀 Starting deployment tests on PR #${prNumber}...\n\nThis will deploy to real Azure infrastructure. Results will be posted here when complete.\n\n[View workflow run](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/workflows/deployment-tests.yml)`
-            });
-
       - name: Trigger deployment tests
         if: steps.check_membership.outputs.is_member == 'true'
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
@@ -102,6 +88,7 @@ jobs:
             // Dispatch from the PR's head ref to test the PR's code changes.
             // Security: Org membership check is the security boundary - only trusted
             // dotnet org members can trigger this workflow.
+            // Note: The triggered workflow posts its own "starting" comment with the run URL.
             await github.rest.actions.createWorkflowDispatch({
               owner: context.repo.owner,
               repo: context.repo.repo,

.github/workflows/deployment-tests.yml

@@ -31,6 +31,28 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  # Post "starting" comment to PR when triggered via /deployment-test command
+  notify-start:
+    name: Notify PR
+    runs-on: ubuntu-latest
+    if: ${{ github.repository_owner == 'dotnet' && inputs.pr_number != '' }}
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Post starting comment
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          PR_NUMBER="${{ inputs.pr_number }}"
+          RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
+          gh pr comment "${PR_NUMBER}" --repo "${{ github.repository }}" --body \
+            "🚀 **Deployment tests starting** on PR #${PR_NUMBER}...
+
+          This will deploy to real Azure infrastructure. Results will be posted here when complete.
+
+          [View workflow run](${RUN_URL})"
+
   # Enumerate test classes to build the matrix
   enumerate:
     name: Enumerate Tests
@@ -214,7 +236,7 @@ jobs:
           AZURE_TENANT_ID: ${{ secrets.AZURE_DEPLOYMENT_TEST_TENANT_ID }}
           AZURE_CLIENT_ID: ${{ secrets.AZURE_DEPLOYMENT_TEST_CLIENT_ID }}
           Azure__SubscriptionId: ${{ secrets.AZURE_DEPLOYMENT_TEST_SUBSCRIPTION_ID }}
-          Azure__Location: eastus
+          Azure__Location: westus3
           GH_TOKEN: ${{ github.token }}
         run: |
           ./dotnet.sh test tests/Aspire.Deployment.EndToEnd.Tests/Aspire.Deployment.EndToEnd.Tests.csproj \
@@ -332,13 +354,58 @@ jobs:
       pull-requests: write
       actions: read
     steps:
-      - name: Download recording artifacts
+      - name: Get job results and download recording artifacts
+        id: get_results
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
           script: |
             const fs = require('fs');
             const path = require('path');
 
+            // Get all jobs for this workflow run to determine per-test results
+            const jobs = await github.paginate(
+              github.rest.actions.listJobsForWorkflowRun,
+              {
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: context.runId,
+                per_page: 100
+              }
+            );
+
+            console.log(`Total jobs found: ${jobs.length}`);
+
+            // Filter for deploy-test matrix jobs (format: "Deploy (TestClassName)")
+            const deployJobs = jobs.filter(job => job.name.startsWith('Deploy ('));
+
+            const passedTests = [];
+            const failedTests = [];
+            const cancelledTests = [];
+
+            for (const job of deployJobs) {
+              // Extract test name from job name "Deploy (TestClassName)"
+              const match = job.name.match(/^Deploy \((.+)\)$/);
+              const testName = match ? match[1] : job.name;
+
+              console.log(`Job "${job.name}" - conclusion: ${job.conclusion}, status: ${job.status}`);
+
+              if (job.conclusion === 'success') {
+                passedTests.push(testName);
+              } else if (job.conclusion === 'failure') {
+                failedTests.push(testName);
+              } else if (job.conclusion === 'cancelled') {
+                cancelledTests.push(testName);
+              }
+            }
+
+            console.log(`Passed: ${passedTests.length}, Failed: ${failedTests.length}, Cancelled: ${cancelledTests.length}`);
+
+            // Output results for later steps
+            core.setOutput('passed_tests', JSON.stringify(passedTests));
+            core.setOutput('failed_tests', JSON.stringify(failedTests));
+            core.setOutput('cancelled_tests', JSON.stringify(cancelledTests));
+            core.setOutput('total_tests', passedTests.length + failedTests.length + cancelledTests.length);
+
             // List all artifacts for the current workflow run
             const allArtifacts = await github.paginate(
               github.rest.actions.listWorkflowRunArtifacts,
@@ -402,60 +469,82 @@ jobs:
       - name: Upload recordings to asciinema and post comment
         env:
           GH_TOKEN: ${{ github.token }}
+          PASSED_TESTS: ${{ steps.get_results.outputs.passed_tests }}
+          FAILED_TESTS: ${{ steps.get_results.outputs.failed_tests }}
+          CANCELLED_TESTS: ${{ steps.get_results.outputs.cancelled_tests }}
+          TOTAL_TESTS: ${{ steps.get_results.outputs.total_tests }}
         shell: bash
         run: |
           PR_NUMBER="${{ inputs.pr_number }}"
           RUN_ID="${{ github.run_id }}"
           RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${RUN_ID}"
           TEST_RESULT="${{ needs.deploy-test.result }}"
 
-          # Determine status emoji and message
-          case "$TEST_RESULT" in
-            success)
-              EMOJI="✅"
-              STATUS="passed"
-              DETAILS="All deployment tests completed successfully."
-              ;;
-            failure)
-              EMOJI="❌"
-              STATUS="failed"
-              DETAILS="One or more deployment tests failed. Check the workflow run for details."
-              ;;
-            cancelled)
-              EMOJI="⚠️"
-              STATUS="cancelled"
-              DETAILS="The deployment tests were cancelled."
-              ;;
-            skipped)
-              EMOJI="⏭️"
-              STATUS="skipped"
-              DETAILS="The deployment tests were skipped (no tests to run or prerequisites not met)."
-              ;;
-            *)
-              EMOJI="❓"
-              STATUS="${TEST_RESULT:-unknown}"
-              DETAILS="The deployment test result could not be determined."
-              ;;
-          esac
+          # Parse the test results from JSON
+          PASSED_COUNT=$(echo "$PASSED_TESTS" | jq 'length')
+          FAILED_COUNT=$(echo "$FAILED_TESTS" | jq 'length')
+          CANCELLED_COUNT=$(echo "$CANCELLED_TESTS" | jq 'length')
 
-          # Build the comment body
+          # Determine overall status
+          if [ "$FAILED_COUNT" -gt 0 ]; then
+            EMOJI="❌"
+            STATUS="failed"
+          elif [ "$CANCELLED_COUNT" -gt 0 ] && [ "$PASSED_COUNT" -eq 0 ]; then
+            EMOJI="⚠️"
+            STATUS="cancelled"
+          elif [ "$PASSED_COUNT" -gt 0 ]; then
+            EMOJI="✅"
+            STATUS="passed"
+          else
+            EMOJI="❓"
+            STATUS="unknown"
+          fi
+
+          # Build the comment header
           COMMENT_BODY="${EMOJI} **Deployment E2E Tests ${STATUS}**
-
-          ${DETAILS}
-
+          
+          **Summary:** ${PASSED_COUNT} passed, ${FAILED_COUNT} failed, ${CANCELLED_COUNT} cancelled
+          
           [View workflow run](${RUN_URL})"
 
-          # Check for recordings and upload them
+          # Add passed tests section if any
+          if [ "$PASSED_COUNT" -gt 0 ]; then
+            PASSED_LIST=$(echo "$PASSED_TESTS" | jq -r '.[]' | while read test; do echo "- ✅ ${test}"; done)
+            COMMENT_BODY="${COMMENT_BODY}
+
+          ### Passed Tests
+          ${PASSED_LIST}"
+          fi
+
+          # Add failed tests section if any
+          if [ "$FAILED_COUNT" -gt 0 ]; then
+            FAILED_LIST=$(echo "$FAILED_TESTS" | jq -r '.[]' | while read test; do echo "- ❌ ${test}"; done)
+            COMMENT_BODY="${COMMENT_BODY}
+
+          ### Failed Tests
+          ${FAILED_LIST}"
+          fi
+
+          # Add cancelled tests section if any
+          if [ "$CANCELLED_COUNT" -gt 0 ]; then
+            CANCELLED_LIST=$(echo "$CANCELLED_TESTS" | jq -r '.[]' | while read test; do echo "- ⚠️ ${test}"; done)
+            COMMENT_BODY="${COMMENT_BODY}
+
+          ### Cancelled Tests
+          ${CANCELLED_LIST}"
+          fi
+
+          # Check for recordings and upload them (only for failed tests)
           RECORDINGS_DIR="cast_files"
 
           if [ -d "$RECORDINGS_DIR" ] && compgen -G "$RECORDINGS_DIR"/*.cast > /dev/null; then
             # Install asciinema
             pip install --quiet asciinema
 
             RECORDING_TABLE="
-
+          
           ### 🎬 Terminal Recordings
-
+          
           | Test | Recording |
           |------|-----------|"
 

tests/Aspire.Deployment.EndToEnd.Tests/AcaStarterDeploymentTests.cs

@@ -14,9 +14,9 @@ namespace Aspire.Deployment.EndToEnd.Tests;
 /// </summary>
 public sealed class AcaStarterDeploymentTests(ITestOutputHelper output)
 {
-    // Timeout set to 15 minutes to allow for Azure provisioning.
-    // Full deployments can take 10-20+ minutes. Increase if needed.
-    private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(15);
+    // Timeout set to 40 minutes to allow for Azure provisioning.
+    // Full deployments can take up to 30 minutes if Azure infrastructure is backed up.
+    private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(40);
 
     [Fact]
     public async Task DeployStarterTemplateToAzureContainerApps()
@@ -211,10 +211,11 @@ private async Task DeployStarterTemplateToAzureContainerAppsCore(CancellationTok
                 .Type("aspire deploy --clear-cache")
                 .Enter()
                 // Wait for pipeline to complete successfully
-                .WaitUntil(s => waitingForPipelineSucceeded.Search(s).Count > 0, TimeSpan.FromMinutes(10))
+                .WaitUntil(s => waitingForPipelineSucceeded.Search(s).Count > 0, TimeSpan.FromMinutes(30))
                 .WaitForSuccessPrompt(counter, TimeSpan.FromMinutes(2));
 
-            // Step 10: Extract deployment URLs and verify endpoints
+            // Step 10: Extract deployment URLs and verify endpoints with retry
+            // Retry each endpoint for up to 3 minutes (18 attempts * 10 seconds)
             output.WriteLine("Step 8: Verifying deployed endpoints...");
             sequenceBuilder
                 .Type($"RG_NAME=\"{resourceGroupName}\" && " +
@@ -225,13 +226,18 @@ private async Task DeployStarterTemplateToAzureContainerAppsCore(CancellationTok
                       "if [ -z \"$urls\" ]; then echo \"❌ No external container app endpoints found\"; exit 1; fi && " +
                       "failed=0 && " +
                       "for url in $urls; do " +
-                      "echo -n \"Checking https://$url... \"; " +
+                      "echo \"Checking https://$url...\"; " +
+                      "success=0; " +
+                      "for i in $(seq 1 18); do " +
                       "STATUS=$(curl -s -o /dev/null -w \"%{http_code}\" \"https://$url\" --max-time 10 2>/dev/null); " +
-                      "if [ \"$STATUS\" = \"200\" ] || [ \"$STATUS\" = \"302\" ]; then echo \"✅ $STATUS\"; else echo \"❌ $STATUS\"; failed=1; fi; " +
+                      "if [ \"$STATUS\" = \"200\" ] || [ \"$STATUS\" = \"302\" ]; then echo \"  ✅ $STATUS (attempt $i)\"; success=1; break; fi; " +
+                      "echo \"  Attempt $i: $STATUS, retrying in 10s...\"; sleep 10; " +
+                      "done; " +
+                      "if [ \"$success\" -eq 0 ]; then echo \"  ❌ Failed after 18 attempts\"; failed=1; fi; " +
                       "done && " +
                       "if [ \"$failed\" -ne 0 ]; then echo \"❌ One or more endpoint checks failed\"; exit 1; fi")
                 .Enter()
-                .WaitForSuccessPrompt(counter, TimeSpan.FromMinutes(2));
+                .WaitForSuccessPrompt(counter, TimeSpan.FromMinutes(5));
 
             // Step 11: Exit terminal
             sequenceBuilder