diff --git a/.github/actions/check-changed-files/action.yml b/.github/actions/check-changed-files/action.yml
index 4e44385e177..f012557ea88 100644
--- a/.github/actions/check-changed-files/action.yml
+++ b/.github/actions/check-changed-files/action.yml
@@ -1,12 +1,19 @@
name: 'Check Changed Files'
description: |
- Check if all changed files in a PR match provided regex patterns.
+ Check if all changed files in a PR match provided glob patterns.
- This action compares changed files in a pull request against one or more regex patterns
+ This action compares changed files in a pull request against one or more glob patterns
and determines if all changed files match at least one of the provided patterns.
+ It only supports pull_request events.
Inputs:
- - patterns: List of regex patterns (multiline string) to match against changed file paths
+ - patterns_file: Path to a file containing glob patterns (relative to repository root).
+ Lines starting with '#' and blank lines are ignored.
+
+ Pattern syntax:
+ - ** matches any path including directory separators (recursive)
+ - * matches any characters except a directory separator
+ - . is treated as a literal dot (no escaping needed)
Outputs:
- only_changed: Boolean indicating if all changed files matched the patterns
@@ -15,8 +22,8 @@ description: |
- matched_files: JSON array of files that matched at least one pattern
- unmatched_files: JSON array of files that didn't match any pattern
inputs:
- patterns:
- description: 'List of regex patterns to match against changed files'
+ patterns_file:
+ description: 'Path to a file containing glob patterns (relative to repository root)'
required: true
outputs:
@@ -57,12 +64,60 @@ runs:
exit 1
fi
- # Read patterns from input (multiline string)
- PATTERNS_INPUT="${{ inputs.patterns }}"
+ # Convert a glob pattern to an anchored ERE regex pattern.
+ # Glob syntax supported:
+ # ** matches any path including directory separators
+ # * matches any characters except a directory separator
+ # . is treated as a literal dot
+ # All other characters are treated as literals.
+ glob_to_regex() {
+ local glob="$1"
+ local result="$glob"
+ # Replace ** and * with placeholders before escaping
+ result="${result//\*\*/__DOUBLESTAR__}"
+ result="${result//\*/__STAR__}"
+ # Escape regex metacharacters that could appear in file paths.
+ # Note: { } ^ $ are not escaped because they are either not special
+ # in ERE mid-pattern or cannot appear in file paths.
+ result="${result//\\/\\\\}"
+ result="${result//./\\.}"
+ result="${result//+/\\+}"
+ result="${result//\?/\\?}"
+ result="${result//\[/\\[}"
+ result="${result//\]/\\]}"
+ result="${result//\(/\\(}"
+ result="${result//\)/\\)}"
+ result="${result//|/\\|}"
+ # Restore glob placeholders as regex
+ result="${result//__STAR__/[^/]*}"
+ result="${result//__DOUBLESTAR__/.*}"
+ # Anchor to full path
+ echo "^${result}$"
+ }
+
+ PATTERNS=()
+
+ PATTERNS_FILE="${{ inputs.patterns_file }}"
- # Validate patterns input
- if [ -z "$PATTERNS_INPUT" ]; then
- echo "Error: patterns input is required"
+ # Read glob patterns from file, skip comments and blank lines
+ FULL_PATH="${GITHUB_WORKSPACE}/${PATTERNS_FILE}"
+ if [ ! -f "$FULL_PATH" ]; then
+ echo "Error: patterns_file '$FULL_PATH' not found"
+ exit 1
+ fi
+ while IFS= read -r line; do
+ # Remove leading/trailing whitespace
+ line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
+ # Skip blank lines and comments
+ if [ -z "$line" ] || [[ "$line" == \#* ]]; then
+ continue
+ fi
+ PATTERNS+=("$(glob_to_regex "$line")")
+ done < "$FULL_PATH"
+
+ # Check if we have any valid patterns
+ if [ ${#PATTERNS[@]} -eq 0 ]; then
+ echo "Error: No valid patterns provided"
exit 1
fi
@@ -77,23 +132,6 @@ runs:
echo "Changed files:"
echo "$CHANGED_FILES"
- # Convert patterns to array and filter out empty lines
- PATTERNS=()
- while IFS= read -r pattern; do
- # Remove leading/trailing whitespace
- pattern=$(echo "$pattern" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
- # Skip empty patterns
- if [ -n "$pattern" ]; then
- PATTERNS+=("$pattern")
- fi
- done <<< "$PATTERNS_INPUT"
-
- # Check if we have any valid patterns
- if [ ${#PATTERNS[@]} -eq 0 ]; then
- echo "Error: No valid patterns provided"
- exit 1
- fi
-
# Initialize arrays
MATCHED_FILES=()
UNMATCHED_FILES=()
diff --git a/.github/actions/enumerate-tests/action.yml b/.github/actions/enumerate-tests/action.yml
index 45093b71df7..f13fb81b093 100644
--- a/.github/actions/enumerate-tests/action.yml
+++ b/.github/actions/enumerate-tests/action.yml
@@ -1,36 +1,32 @@
name: 'Enumerate test projects'
description: 'Enumerate list of test projects'
inputs:
- includeIntegrations:
+ buildArgs:
required: false
- type: boolean
- default: false
- includeTemplates:
- required: false
- type: boolean
- default: false
- includeCliE2E:
- required: false
- type: boolean
- default: false
- includeDeployment:
- required: false
- type: boolean
- default: false
-
+ type: string
+ default: ''
+ description: 'Additional MSBuild arguments passed to the test matrix generation step (e.g., /p:IncludeTemplateTests=true /p:OnlyDeploymentTests=true)'
+
+# Output format: JSON with structure {"include": [{...}, ...]}
+# Each entry contains:
+# - type: 'regular' | 'collection' | 'class'
+# - projectName: Full project name (e.g., 'Aspire.Hosting.Tests')
+# - name: Display name for the test run
+# - shortname: Short identifier
+# - testProjectPath: Relative path to the test project
+# - workitemprefix: Prefix for work item naming
+# - runs-on: GitHub Actions runner (e.g., 'ubuntu-latest', 'windows-latest')
+# - testSessionTimeout: Timeout for the test session (e.g., '20m')
+# - testHangTimeout: Timeout for hung tests (e.g., '10m')
+# - requiresNugets: Boolean indicating if NuGet packages are needed
+# - requiresTestSdk: Boolean indicating if test SDK is needed
+# - extraTestArgs: Additional test arguments (e.g., '--filter-trait "Partition=P1"')
+# - collection: (collection type only) Collection/partition name
+# - classname: (class type only) Fully qualified test class name
outputs:
- integrations_tests_matrix:
- description: Integration tests matrix
- value: ${{ steps.generate_integrations_matrix.outputs.integrations_tests_matrix }}
- templates_tests_matrix:
- description: Templates tests matrix
- value: ${{ steps.generate_templates_matrix.outputs.templates_tests_matrix }}
- cli_e2e_tests_matrix:
- description: Cli E2E tests matrix
- value: ${{ steps.generate_cli_e2e_matrix.outputs.cli_e2e_tests_matrix }}
- deployment_tests_matrix:
- description: Deployment E2E tests matrix
- value: ${{ steps.generate_deployment_matrix.outputs.deployment_tests_matrix }}
+ all_tests:
+ description: Combined matrix of all test entries
+ value: ${{ steps.expand_matrix.outputs.all_tests }}
runs:
using: "composite"
steps:
@@ -42,123 +38,38 @@ runs:
with:
global-json-file: ${{ github.workspace }}/global.json
- - name: Get list of integration tests
- if: ${{ inputs.includeIntegrations }}
- shell: pwsh
- run: >
- dotnet build ${{ github.workspace }}/tests/Shared/GetTestProjects.proj
- /bl:${{ github.workspace }}/artifacts/log/Debug/GetTestProjects.binlog
- /p:TestsListOutputPath=${{ github.workspace }}/artifacts/TestsForGithubActions.list
- /p:ContinuousIntegrationBuild=true
-
- - name: Generate list of template tests
- if: ${{ inputs.includeTemplates }}
- shell: pwsh
- run: >
- dotnet build ${{ github.workspace }}/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj
- "/t:Build;ExtractTestClassNames"
- /bl:${{ github.workspace }}/artifacts/log/Debug/BuildTemplatesTests.binlog
- -p:ExtractTestClassNamesForHelix=true
- -p:PrepareForHelix=true
- -p:ExtractTestClassNamesPrefix=Aspire.Templates.Tests
- -p:InstallBrowsersForPlaywright=false
+ - name: Restore
+ shell: bash
+ run: ./restore.sh
- - name: Generate list of CLI E2E tests
- if: ${{ inputs.includeCliE2E }}
- shell: pwsh
- run: >
- dotnet build ${{ github.workspace }}/tests/Aspire.Cli.EndToEnd.Tests/Aspire.Cli.EndToEnd.Tests.csproj
- "/t:Build;ExtractTestClassNames"
- /bl:${{ github.workspace }}/artifacts/log/Debug/BuildCliEndToEndTests.binlog
- -p:ExtractTestClassNamesForHelix=true
- -p:PrepareForHelix=true
- -p:ExtractTestClassNamesPrefix=Aspire.Cli.EndToEnd.Tests
- -p:InstallBrowsersForPlaywright=false
+ - name: Build ExtractTestPartitions tool
+ shell: bash
+ run: dotnet build tools/ExtractTestPartitions/ExtractTestPartitions.csproj -c Release --nologo -v quiet
- - name: Generate list of Deployment E2E tests
- if: ${{ inputs.includeDeployment }}
- shell: pwsh
+ - name: Generate canonical test matrix
+ shell: bash
run: >
- dotnet build ${{ github.workspace }}/tests/Aspire.Deployment.EndToEnd.Tests/Aspire.Deployment.EndToEnd.Tests.csproj
- "/t:Build;ExtractTestClassNames"
- /bl:${{ github.workspace }}/artifacts/log/Debug/BuildDeploymentEndToEndTests.binlog
- -p:ExtractTestClassNamesForHelix=true
- -p:PrepareForHelix=true
- -p:ExtractTestClassNamesPrefix=Aspire.Deployment.EndToEnd.Tests
-
- - name: Generate tests matrix
- id: generate_integrations_matrix
- if: ${{ inputs.includeIntegrations }}
- shell: pwsh
- run: |
- $filePath = "${{ github.workspace }}/artifacts/TestsForGithubActions.list"
- $lines = Get-Content $filePath
- $jsonObject = @{
- "shortname" = $lines | Sort-Object
- }
- $jsonString = ConvertTo-Json $jsonObject -Compress
- "integrations_tests_matrix=$jsonString"
- "integrations_tests_matrix=$jsonString" | Out-File -FilePath $env:GITHUB_OUTPUT
-
- - name: Generate templates matrix
- id: generate_templates_matrix
- if: ${{ inputs.includeTemplates }}
- shell: pwsh
- run: |
- $inputFilePath = "${{ github.workspace }}/artifacts/helix/templates-tests/Aspire.Templates.Tests.tests.list"
- $lines = Get-Content $inputFilePath
-
- $prefix = "Aspire.Templates.Tests."
- $lines = Get-Content $inputFilePath | ForEach-Object {
- $_ -replace "^$prefix", ""
- }
-
- $jsonObject = @{
- "shortname" = $lines | Sort-Object
- }
- $jsonString = ConvertTo-Json $jsonObject -Compress
- "templates_tests_matrix=$jsonString"
- "templates_tests_matrix=$jsonString" | Out-File -FilePath $env:GITHUB_OUTPUT
-
- - name: Generate cli e2e matrix
- id: generate_cli_e2e_matrix
- if: ${{ inputs.includeCliE2E }}
+ ./build.sh -test
+ /p:TestRunnerName=TestEnumerationRunsheetBuilder
+ /p:TestMatrixOutputPath=artifacts/canonical-test-matrix.json
+ /p:GenerateCIPartitions=true
+ /bl
+ ${{ inputs.buildArgs }}
+
+ - name: Expand matrix for GitHub Actions
+ id: expand_matrix
shell: pwsh
run: |
- $inputFilePath = "${{ github.workspace }}/artifacts/helix/cli-e2e-tests/Aspire.Cli.EndToEnd.Tests.tests.list"
- $lines = Get-Content $inputFilePath
-
- $prefix = "Aspire.Cli.EndToEnd.Tests."
- $lines = @(Get-Content $inputFilePath | ForEach-Object {
- $_ -replace "^$prefix", ""
- })
-
- $jsonObject = @{
- "shortname" = @($lines | Sort-Object)
- }
- $jsonString = ConvertTo-Json $jsonObject -Compress
- "cli_e2e_tests_matrix=$jsonString"
- "cli_e2e_tests_matrix=$jsonString" | Out-File -FilePath $env:GITHUB_OUTPUT
-
- - name: Generate deployment matrix
- id: generate_deployment_matrix
- if: ${{ inputs.includeDeployment }}
- shell: pwsh
- run: |
- $inputFilePath = "${{ github.workspace }}/artifacts/helix/deployment-e2e-tests/Aspire.Deployment.EndToEnd.Tests.tests.list"
- $lines = Get-Content $inputFilePath
-
- $prefix = "Aspire.Deployment.EndToEnd.Tests."
- $lines = @(Get-Content $inputFilePath | ForEach-Object {
- $_ -replace "^$prefix", ""
- })
-
- $jsonObject = @{
- "shortname" = @($lines | Sort-Object)
+ $canonicalMatrixPath = "${{ github.workspace }}/artifacts/canonical-test-matrix.json"
+ $expandScript = "${{ github.workspace }}/eng/scripts/expand-test-matrix-github.ps1"
+
+ if (Test-Path $canonicalMatrixPath) {
+ & $expandScript -CanonicalMatrixFile $canonicalMatrixPath -OutputToGitHubEnv
+ } else {
+ $emptyMatrix = '{"include":[]}'
+ "all_tests=$emptyMatrix" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
+ Write-Host "No canonical test matrix found, using empty matrix"
}
- $jsonString = ConvertTo-Json $jsonObject -Compress
- "deployment_tests_matrix=$jsonString"
- "deployment_tests_matrix=$jsonString" | Out-File -FilePath $env:GITHUB_OUTPUT
- name: Upload logs
if: always()
@@ -167,4 +78,6 @@ runs:
name: logs-enumerate-tests-${{ runner.os }}
path: |
artifacts/log/**/*.binlog
- artifacts/**/*.list
+ artifacts/**/*tests-partitions.json
+ artifacts/**/*tests-metadata.json
+ artifacts/canonical-test-matrix.json
diff --git a/.github/skills/cli-e2e-testing/SKILL.md b/.github/skills/cli-e2e-testing/SKILL.md
index 8234bee8e3f..6217aa7bf28 100644
--- a/.github/skills/cli-e2e-testing/SKILL.md
+++ b/.github/skills/cli-e2e-testing/SKILL.md
@@ -375,7 +375,7 @@ Environment variables set in CI:
- `GH_TOKEN`: GitHub token for API access
- `GITHUB_WORKSPACE`: Workspace root for artifact paths
-Each test class runs as a separate CI job via `CliEndToEndTestRunsheetBuilder` for parallel execution.
+Each test class runs as a separate CI job via the unified `TestEnumerationRunsheetBuilder` infrastructure (using `SplitTestsOnCI=true`) for parallel execution.
## CI Troubleshooting
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6b739188136..bc39f93bf69 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -37,28 +37,7 @@ jobs:
if: ${{ github.event_name == 'pull_request' }}
uses: ./.github/actions/check-changed-files
with:
- # Patterns that do NOT require CI to run
- patterns: |
- \.md$
- eng/pipelines/.*
- eng/test-configuration.json
- \.github/instructions/.*
- \.github/skills/.*
- \.github/workflows/apply-test-attributes.yml
- \.github/workflows/backmerge-release.yml
- \.github/workflows/backport.yml
- \.github/workflows/dogfood-comment.yml
- \.github/workflows/generate-api-diffs.yml
- \.github/workflows/generate-ats-diffs.yml
- \.github/workflows/labeler-*.yml
- \.github/workflows/markdownlint*.yml
- \.github/workflows/refresh-manifests.yml
- \.github/workflows/reproduce-flaky-tests.yml
- \.github/workflows/pr-review-needed.yml
- \.github/workflows/specialized-test-runner.yml
- \.github/workflows/tests-outerloop.yml
- \.github/workflows/tests-quarantine.yml
- \.github/workflows/update-*.yml
+ patterns_file: eng/testing/github-ci-trigger-patterns.txt
- id: compute_version_suffix
name: Compute version suffix for PRs
diff --git a/.github/workflows/deployment-tests.yml b/.github/workflows/deployment-tests.yml
index 5643a0d8a05..41128c915e9 100644
--- a/.github/workflows/deployment-tests.yml
+++ b/.github/workflows/deployment-tests.yml
@@ -45,7 +45,7 @@ jobs:
run: |
PR_NUMBER="${{ inputs.pr_number }}"
RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-
+
gh pr comment "${PR_NUMBER}" --repo "${{ github.repository }}" --body \
"🚀 **Deployment tests starting** on PR #${PR_NUMBER}...
@@ -61,19 +61,19 @@ jobs:
permissions:
contents: read
outputs:
- matrix: ${{ steps.enumerate.outputs.deployment_tests_matrix }}
+ matrix: ${{ steps.enumerate.outputs.all_tests }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: ./.github/actions/enumerate-tests
id: enumerate
with:
- includeDeployment: true
+ buildArgs: '/p:OnlyDeploymentTests=true'
- name: Display test matrix
run: |
echo "Deployment test matrix:"
- echo '${{ steps.enumerate.outputs.deployment_tests_matrix }}' | jq .
+ echo '${{ steps.enumerate.outputs.all_tests }}' | jq .
# Build solution and CLI once, share via artifacts
build:
@@ -107,16 +107,16 @@ jobs:
ARTIFACT_DIR="${{ github.workspace }}/cli-artifacts"
mkdir -p "$ARTIFACT_DIR/bin"
mkdir -p "$ARTIFACT_DIR/packages"
-
+
# Copy CLI binary and dependencies
cp -r "${{ github.workspace }}/artifacts/bin/Aspire.Cli/Release/net10.0/"* "$ARTIFACT_DIR/bin/"
-
+
# Copy NuGet packages
PACKAGES_DIR="${{ github.workspace }}/artifacts/packages/Release/Shipping"
if [ -d "$PACKAGES_DIR" ]; then
find "$PACKAGES_DIR" -name "*.nupkg" -exec cp {} "$ARTIFACT_DIR/packages/" \;
fi
-
+
echo "CLI artifacts prepared:"
ls -la "$ARTIFACT_DIR/bin/"
echo "Package count: $(find "$ARTIFACT_DIR/packages" -name "*.nupkg" | wc -l)"
@@ -132,7 +132,7 @@ jobs:
deploy-test:
name: Deploy (${{ matrix.shortname }})
needs: [enumerate, build]
- if: ${{ needs.enumerate.outputs.matrix != '{"shortname":[]}' && needs.enumerate.outputs.matrix != '' }}
+ if: ${{ needs.enumerate.outputs.matrix != '{"include":[]}' && needs.enumerate.outputs.matrix != '' }}
runs-on: 8-core-ubuntu-latest
environment: deployment-testing
permissions:
@@ -170,22 +170,22 @@ jobs:
run: |
ASPIRE_HOME="$HOME/.aspire"
mkdir -p "$ASPIRE_HOME/bin"
-
+
# Copy CLI binary and dependencies
cp -r "${{ github.workspace }}/cli-artifacts/bin/"* "$ASPIRE_HOME/bin/"
chmod +x "$ASPIRE_HOME/bin/aspire"
-
+
# Add to PATH for this job
echo "$ASPIRE_HOME/bin" >> $GITHUB_PATH
-
+
# Set up NuGet hive for local packages
HIVE_DIR="$ASPIRE_HOME/hives/local/packages"
mkdir -p "$HIVE_DIR"
cp "${{ github.workspace }}/cli-artifacts/packages/"*.nupkg "$HIVE_DIR/" 2>/dev/null || true
-
+
# Configure CLI to use local channel
"$ASPIRE_HOME/bin/aspire" config set channel local --global || true
-
+
echo "✅ Aspire CLI installed:"
"$ASPIRE_HOME/bin/aspire" --version
@@ -199,7 +199,7 @@ jobs:
script: |
const token = await core.getIDToken('api://AzureADTokenExchange');
core.setSecret(token);
-
+
// Login directly - token never leaves this step
await exec.exec('az', [
'login', '--service-principal',
@@ -208,7 +208,7 @@ jobs:
'--federated-token', token,
'--allow-no-subscriptions'
]);
-
+
await exec.exec('az', [
'account', 'set',
'--subscription', process.env.AZURE_SUBSCRIPTION_ID
@@ -245,7 +245,7 @@ jobs:
--results-directory ${{ github.workspace }}/testresults \
-- \
--filter-not-trait "quarantined=true" \
- --filter-class "Aspire.Deployment.EndToEnd.Tests.${{ matrix.shortname }}" \
+ ${{ matrix.extraTestArgs }} \
|| echo "test_failed=true" >> $GITHUB_OUTPUT
- name: Upload test results
@@ -361,7 +361,7 @@ jobs:
script: |
const fs = require('fs');
const path = require('path');
-
+
// Get all jobs for this workflow run to determine per-test results
const jobs = await github.paginate(
github.rest.actions.listJobsForWorkflowRun,
@@ -372,23 +372,23 @@ jobs:
per_page: 100
}
);
-
+
console.log(`Total jobs found: ${jobs.length}`);
-
+
// Filter for deploy-test matrix jobs (format: "Deploy (TestClassName)")
const deployJobs = jobs.filter(job => job.name.startsWith('Deploy ('));
-
+
const passedTests = [];
const failedTests = [];
const cancelledTests = [];
-
+
for (const job of deployJobs) {
// Extract test name from job name "Deploy (TestClassName)"
const match = job.name.match(/^Deploy \((.+)\)$/);
const testName = match ? match[1] : job.name;
-
+
console.log(`Job "${job.name}" - conclusion: ${job.conclusion}, status: ${job.status}`);
-
+
if (job.conclusion === 'success') {
passedTests.push(testName);
} else if (job.conclusion === 'failure') {
@@ -397,15 +397,15 @@ jobs:
cancelledTests.push(testName);
}
}
-
+
console.log(`Passed: ${passedTests.length}, Failed: ${failedTests.length}, Cancelled: ${cancelledTests.length}`);
-
+
// Output results for later steps
core.setOutput('passed_tests', JSON.stringify(passedTests));
core.setOutput('failed_tests', JSON.stringify(failedTests));
core.setOutput('cancelled_tests', JSON.stringify(cancelledTests));
core.setOutput('total_tests', passedTests.length + failedTests.length + cancelledTests.length);
-
+
// List all artifacts for the current workflow run
const allArtifacts = await github.paginate(
github.rest.actions.listWorkflowRunArtifacts,
@@ -416,53 +416,53 @@ jobs:
per_page: 100
}
);
-
+
console.log(`Total artifacts found: ${allArtifacts.length}`);
-
+
// Filter for deployment test recording artifacts
- const recordingArtifacts = allArtifacts.filter(a =>
+ const recordingArtifacts = allArtifacts.filter(a =>
a.name.startsWith('deployment-test-recordings-')
);
-
+
console.log(`Found ${recordingArtifacts.length} recording artifacts`);
-
+
// Create recordings directory
const recordingsDir = 'recordings';
fs.mkdirSync(recordingsDir, { recursive: true });
-
+
// Download each artifact
for (const artifact of recordingArtifacts) {
console.log(`Downloading ${artifact.name}...`);
-
+
const download = await github.rest.actions.downloadArtifact({
owner: context.repo.owner,
repo: context.repo.repo,
artifact_id: artifact.id,
archive_format: 'zip'
});
-
+
const artifactPath = path.join(recordingsDir, `${artifact.name}.zip`);
fs.writeFileSync(artifactPath, Buffer.from(download.data));
console.log(`Saved to ${artifactPath}`);
}
-
+
core.setOutput('artifact_count', recordingArtifacts.length);
- name: Extract recordings from artifacts
shell: bash
run: |
mkdir -p cast_files
-
+
for zipfile in recordings/*.zip; do
if [ -f "$zipfile" ]; then
echo "Extracting $zipfile..."
unzip -o "$zipfile" -d "recordings/extracted_$(basename "$zipfile" .zip)" || true
fi
done
-
+
# Find and copy all .cast files
find recordings -name "*.cast" -exec cp {} cast_files/ \; 2>/dev/null || true
-
+
echo "Found recordings:"
ls -la cast_files/ || echo "No .cast files found"
@@ -479,12 +479,12 @@ jobs:
RUN_ID="${{ github.run_id }}"
RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${RUN_ID}"
TEST_RESULT="${{ needs.deploy-test.result }}"
-
+
# Parse the test results from JSON
PASSED_COUNT=$(echo "$PASSED_TESTS" | jq 'length')
FAILED_COUNT=$(echo "$FAILED_TESTS" | jq 'length')
CANCELLED_COUNT=$(echo "$CANCELLED_TESTS" | jq 'length')
-
+
# Determine overall status
if [ "$FAILED_COUNT" -gt 0 ]; then
EMOJI="❌"
@@ -499,66 +499,66 @@ jobs:
EMOJI="❓"
STATUS="unknown"
fi
-
+
# Build the comment header
COMMENT_BODY="${EMOJI} **Deployment E2E Tests ${STATUS}**
-
+
**Summary:** ${PASSED_COUNT} passed, ${FAILED_COUNT} failed, ${CANCELLED_COUNT} cancelled
-
+
[View workflow run](${RUN_URL})"
-
+
# Add passed tests section if any
if [ "$PASSED_COUNT" -gt 0 ]; then
PASSED_LIST=$(echo "$PASSED_TESTS" | jq -r '.[]' | while read test; do echo "- ✅ ${test}"; done)
COMMENT_BODY="${COMMENT_BODY}
-
+
### Passed Tests
${PASSED_LIST}"
fi
-
+
# Add failed tests section if any
if [ "$FAILED_COUNT" -gt 0 ]; then
FAILED_LIST=$(echo "$FAILED_TESTS" | jq -r '.[]' | while read test; do echo "- ❌ ${test}"; done)
COMMENT_BODY="${COMMENT_BODY}
-
+
### Failed Tests
${FAILED_LIST}"
fi
-
+
# Add cancelled tests section if any
if [ "$CANCELLED_COUNT" -gt 0 ]; then
CANCELLED_LIST=$(echo "$CANCELLED_TESTS" | jq -r '.[]' | while read test; do echo "- ⚠️ ${test}"; done)
COMMENT_BODY="${COMMENT_BODY}
-
+
### Cancelled Tests
${CANCELLED_LIST}"
fi
-
+
# Check for recordings and upload them
RECORDINGS_DIR="cast_files"
-
+
if [ -d "$RECORDINGS_DIR" ] && compgen -G "$RECORDINGS_DIR"/*.cast > /dev/null; then
# Install asciinema
pip install --quiet asciinema
-
+
RECORDING_TABLE="
-
+
### 🎬 Terminal Recordings
-
+
| Test | Recording |
|------|-----------|"
-
+
UPLOAD_COUNT=0
-
+
for castfile in "$RECORDINGS_DIR"/*.cast; do
if [ -f "$castfile" ]; then
filename=$(basename "$castfile" .cast)
echo "Uploading $castfile..."
-
+
# Upload to asciinema and capture URL
UPLOAD_OUTPUT=$(asciinema upload "$castfile" 2>&1) || true
ASCIINEMA_URL=$(echo "$UPLOAD_OUTPUT" | grep -oP 'https://asciinema\.org/a/[a-zA-Z0-9_-]+' | head -1) || true
-
+
if [ -n "$ASCIINEMA_URL" ]; then
RECORDING_TABLE="${RECORDING_TABLE}
| ${filename} | [▶️ View Recording](${ASCIINEMA_URL}) |"
@@ -571,16 +571,16 @@ jobs:
fi
fi
done
-
+
if [ $UPLOAD_COUNT -gt 0 ]; then
COMMENT_BODY="${COMMENT_BODY}${RECORDING_TABLE}"
fi
-
+
echo "Uploaded $UPLOAD_COUNT recordings"
else
echo "No recordings found in $RECORDINGS_DIR"
fi
-
+
# Post the comment
gh pr comment "${PR_NUMBER}" --repo "${{ github.repository }}" --body "$COMMENT_BODY"
echo "Posted comment to PR #${PR_NUMBER}"
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 19627b4e58a..2c2b9b8f419 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -39,6 +39,11 @@ on:
required: false
type: boolean
default: false
+ # Controls whether to set CLI E2E environment variables (GH_TOKEN, GITHUB_PR_NUMBER, GITHUB_PR_HEAD_SHA)
+ requiresCliArchive:
+ required: false
+ type: boolean
+ default: false
# Controls whether to install Playwright browsers during project build
enablePlaywrightInstall:
required: false
@@ -234,7 +239,8 @@ jobs:
run: >
${{ env.BUILD_SCRIPT }} -restore -ci -build -projects ${{ env.TEST_PROJECT_PATH }}
/p:PrepareForHelix=true
- /bl:${{ github.workspace }}/artifacts/log/Debug/PrepareForHelix.binlog
+ /p:ArchiveTests=true
+ /bl:${{ github.workspace }}/artifacts/log/Debug/BuildAndArchive.binlog
${{ !inputs.enablePlaywrightInstall && '/p:InstallBrowsersForPlaywright=false' || '' }}
${{ inputs.versionOverrideArg }}
@@ -296,6 +302,10 @@ jobs:
TEST_LOG_PATH: ${{ github.workspace }}/artifacts/log/test-logs
TestsRunningOutsideOfRepo: true
TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX: 'netaspireci.azurecr.io'
+ # PR metadata and token for CLI E2E tests that download artifacts from a PR
+ GITHUB_PR_NUMBER: ${{ inputs.requiresCliArchive && github.event.pull_request.number || '' }}
+ GITHUB_PR_HEAD_SHA: ${{ inputs.requiresCliArchive && github.event.pull_request.head.sha || '' }}
+ GH_TOKEN: ${{ inputs.requiresCliArchive && github.token || '' }}
run: |
# Start heartbeat monitor in background
${{ github.workspace }}/${{ env.DOTNET_SCRIPT }} ${{ github.workspace }}/tools/scripts/Heartbeat.cs &
@@ -334,6 +344,10 @@ jobs:
TEST_LOG_PATH: ${{ github.workspace }}/artifacts/log/test-logs
TestsRunningOutsideOfRepo: true
TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX: 'netaspireci.azurecr.io'
+ # PR metadata and token for CLI E2E tests that download artifacts from a PR
+ GITHUB_PR_NUMBER: ${{ inputs.requiresCliArchive && github.event.pull_request.number || '' }}
+ GITHUB_PR_HEAD_SHA: ${{ inputs.requiresCliArchive && github.event.pull_request.head.sha || '' }}
+ GH_TOKEN: ${{ inputs.requiresCliArchive && github.token || '' }}
run: |
# Start heartbeat monitor in background (output goes to console directly)
$heartbeatProcess = Start-Process -FilePath "dotnet" `
@@ -374,12 +388,10 @@ jobs:
NUGET_PACKAGES: ${{ github.workspace }}/.packages
PLAYWRIGHT_INSTALLED: ${{ !inputs.enablePlaywrightInstall && 'false' || 'true' }}
TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX: 'netaspireci.azurecr.io'
- # PR number for CLI E2E tests that download artifacts from a PR
- GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
- # PR head SHA for version verification (not the merge commit SHA)
- GITHUB_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
- # GitHub token for CLI E2E tests to download artifacts
- GH_TOKEN: ${{ github.token }}
+ # PR metadata and token for CLI E2E tests that download artifacts from a PR
+ GITHUB_PR_NUMBER: ${{ inputs.requiresCliArchive && github.event.pull_request.number || '' }}
+ GITHUB_PR_HEAD_SHA: ${{ inputs.requiresCliArchive && github.event.pull_request.head.sha || '' }}
+ GH_TOKEN: ${{ inputs.requiresCliArchive && github.token || '' }}
run: |
# Start heartbeat monitor in background
${{ env.DOTNET_SCRIPT }} ${{ github.workspace }}/tools/scripts/Heartbeat.cs &
@@ -418,12 +430,10 @@ jobs:
NUGET_PACKAGES: ${{ github.workspace }}/.packages
PLAYWRIGHT_INSTALLED: ${{ !inputs.enablePlaywrightInstall && 'false' || 'true' }}
TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX: 'netaspireci.azurecr.io'
- # PR number for CLI E2E tests that download artifacts from a PR
- GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
- # PR head SHA for version verification (not the merge commit SHA)
- GITHUB_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
- # GitHub token for CLI E2E tests to download artifacts
- GH_TOKEN: ${{ github.token }}
+ # PR metadata and token for CLI E2E tests that download artifacts from a PR
+ GITHUB_PR_NUMBER: ${{ inputs.requiresCliArchive && github.event.pull_request.number || '' }}
+ GITHUB_PR_HEAD_SHA: ${{ inputs.requiresCliArchive && github.event.pull_request.head.sha || '' }}
+ GH_TOKEN: ${{ inputs.requiresCliArchive && github.token || '' }}
run: |
# Start heartbeat monitor in background (output goes to console directly)
$heartbeatProcess = Start-Process -FilePath "${{ env.DOTNET_SCRIPT }}" `
diff --git a/.github/workflows/specialized-test-runner.yml b/.github/workflows/specialized-test-runner.yml
index 434f8e5442f..2adf4776930 100644
--- a/.github/workflows/specialized-test-runner.yml
+++ b/.github/workflows/specialized-test-runner.yml
@@ -102,7 +102,7 @@ jobs:
uses: ./.github/workflows/build-packages.yml
run_tests:
- name: Test
+ name: ${{ matrix.tests.project }}
needs: [generate_tests_matrix, build_packages]
strategy:
fail-fast: false
diff --git a/.github/workflows/tests-runner.yml b/.github/workflows/tests-runner.yml
deleted file mode 100644
index c18a1106f96..00000000000
--- a/.github/workflows/tests-runner.yml
+++ /dev/null
@@ -1,206 +0,0 @@
-# Executes all the tests on all the platforms
-name: Tests
-
-on:
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-jobs:
- # Duplicated jobs so their dependencies are not blocked on both the
- # setup jobs
-
- # Generates a runsheet for all the tests in the solution that do not require
- # NuGet packages to be built.
- # The runsheet generation is expected to be fast.
- generate_tests_matrix:
- name: Generate test runsheet
- runs-on: windows-latest
- outputs:
- runsheet: ${{ steps.generate_tests_matrix.outputs.runsheet }}
- steps:
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-
- # In order to build the solution, we need to install the SDK and the toolsets first
- # as defined in global.json. For this, create a temporary project file and run the
- # build command with the -restore option. This will install the SDK and toolsets.
- #
- # We don't want to run 'build.cmd -restore' as it will also restore all the packages,
- # which takes a long time and is not needed for this job.
- - name: Install toolsets
- shell: pwsh
- run: |
- mkdir ./artifacts/tmp -force | Out-Null
- '' | Out-File -FilePath ./artifacts/tmp/install-toolset.proj -Encoding utf8
- ./build.cmd -restore -projects ./artifacts/tmp/install-toolset.proj
-
- - name: Generate test runsheet
- id: generate_tests_matrix
- shell: pwsh
- run: |
- ./build.cmd -test /p:TestRunnerName=TestRunsheetBuilder -bl -c Release
-
- - name: Upload logs, and test results
- if: ${{ always() }}
- uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
- with:
- name: runsheet-logs
- path: |
- ${{ github.workspace }}/artifacts/log/*/*.binlog
- ${{ github.workspace }}/artifacts/log/*/TestLogs/**
- ${{ github.workspace }}/artifacts/tmp/*/combined_runsheet.json
- retention-days: 3
-
- # Generates a runsheet for all the tests in the solution that DO require
- # NuGet packages to be built.
- # The runsheet generation is expected to be slow as we need to restore and build
- # the whole solution and publish all the packages.
- generate_e2e_matrix:
- name: Generate E2E test runsheet
- runs-on: windows-latest
- outputs:
- runsheet: ${{ steps.generate_e2e_matrix.outputs.runsheet }}
- steps:
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-
- - name: Build with packages
- run: |
- ./build.cmd -restore -build -pack -c Release -ci -bl /p:InstallBrowsersForPlaywright=false /p:SkipTestProjects=true
-
- - name: Generate test runsheet
- id: generate_e2e_matrix
- run: |
- ./build.cmd -test /p:TestRunnerName=TestRunsheetBuilder /p:FullE2e=true -bl -c Release
-
- - name: Upload built NuGets
- uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
- with:
- name: built-nugets
- path: artifacts/packages
- retention-days: 3
-
- - name: Upload logs, and test results
- if: ${{ always() }}
- uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
- with:
- name: runsheet-e2e-logs
- path: |
- ${{ github.workspace }}/artifacts/log/*/*.binlog
- ${{ github.workspace }}/artifacts/log/*/TestLogs/**
- ${{ github.workspace }}/artifacts/tmp/*/combined_runsheet.json
- retention-days: 3
-
- run_tests:
- name: Test
- needs: generate_tests_matrix
- strategy:
- fail-fast: false
- matrix:
- tests: ${{ fromJson(needs.generate_tests_matrix.outputs.runsheet) }}
-
- runs-on: ${{ matrix.tests.os }} # Use the OS from the matrix
-
- steps:
- - name: Trust HTTPS development certificate
- if: runner.os == 'Linux'
- run: dotnet dev-certs https --trust
-
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-
- - name: Verify Docker is running
- # nested docker containers not supported on windows
- if: runner.os == 'Linux'
- run: docker info
-
- - name: Install Azure Functions Core Tools
- if: runner.os == 'Linux'
- run: |
- npm i -g azure-functions-core-tools@4 --unsafe-perm true
-
- - name: Test ${{ matrix.tests.label }}
- run: |
- ${{ matrix.tests.command }}
-
- - name: Upload test results
- if: always()
- uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
- with:
- name: ${{ matrix.tests.project }}-${{ matrix.tests.os }}-logs
- path: |
- ${{ github.workspace }}/artifacts/TestResults/*/*.trx
- ${{ github.workspace }}/artifacts/log/*/TestLogs/**
- retention-days: 30
-
- - name: Upload logs
- if: failure()
- uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
- with:
- name: ${{ matrix.tests.project }}-${{ matrix.tests.os }}-binlogs
- path: |
- ${{ github.workspace }}/artifacts/log/*/*.binlog
- retention-days: 3
-
- run_e2e_tests:
- name: E2ETest
- needs: generate_e2e_matrix
- strategy:
- fail-fast: false
- matrix:
- tests: ${{ fromJson(needs.generate_e2e_matrix.outputs.runsheet) }}
-
- runs-on: ${{ matrix.tests.os }} # Use the OS from the matrix
-
- steps:
- - name: Trust HTTPS development certificate
- if: runner.os == 'Linux'
- run: dotnet dev-certs https --trust
-
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-
- - name: Download built NuGets
- uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9
- with:
- pattern: built-nugets
- path: ${{ github.workspace }}/artifacts/packages
-
- - name: Copy NuGets to the correct location
- shell: pwsh
- run:
- Move-Item -Path "${{ github.workspace }}/artifacts/packages/built-nugets/Release" -Destination "${{ github.workspace }}/artifacts/packages"
-
- - name: Verify Docker is running
- # nested docker containers not supported on windows
- if: runner.os == 'Linux'
- run: docker info
-
- - name: Install Azure Functions Core Tools
- if: runner.os == 'Linux'
- run: |
- npm i -g azure-functions-core-tools@4 --unsafe-perm true
-
- - name: Test ${{ matrix.tests.label }}
- env:
- BUILT_NUGETS_PATH: ${{ github.workspace }}/artifacts/packages/Release/Shipping
- run: |
- ${{ matrix.tests.command }}
-
- - name: Upload test results
- if: always()
- uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
- with:
- name: ${{ matrix.tests.project }}-${{ matrix.tests.os }}-logs
- path: |
- ${{ github.workspace }}/artifacts/TestResults/*/*.trx
- ${{ github.workspace }}/artifacts/log/*/TestLogs/**
- retention-days: 30
-
- - name: Upload logs
- if: failure()
- uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
- with:
- name: ${{ matrix.tests.project }}-${{ matrix.tests.os }}-binlogs
- path: |
- ${{ github.workspace }}/artifacts/log/*/*.binlog
- retention-days: 3
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index eab7145d481..4d8c2aa0311 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -9,55 +9,28 @@ on:
type: string
jobs:
- # Duplicated jobs so their dependencies are not blocked on both the
- # setup jobs
-
- setup_for_tests_lin:
- name: Setup for tests (Linux)
+ setup_for_tests:
+ name: Setup for tests
runs-on: ubuntu-latest
outputs:
- integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }}
- templates_tests_matrix: ${{ steps.generate_tests_matrix.outputs.templates_tests_matrix }}
- cli_e2e_tests_matrix: ${{ steps.generate_tests_matrix.outputs.cli_e2e_tests_matrix }}
+ tests_matrix_no_nugets: ${{ steps.split_matrix.outputs.tests_matrix_no_nugets }}
+ tests_matrix_no_nugets_overflow: ${{ steps.split_matrix.outputs.tests_matrix_no_nugets_overflow }}
+ tests_matrix_requires_nugets: ${{ steps.split_matrix.outputs.tests_matrix_requires_nugets }}
+ tests_matrix_requires_cli_archive: ${{ steps.split_matrix.outputs.tests_matrix_requires_cli_archive }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: ./.github/actions/enumerate-tests
id: generate_tests_matrix
with:
- includeIntegrations: true
- includeTemplates: true
- includeCliE2E: ${{ github.event_name == 'pull_request' }}
-
- setup_for_tests_macos:
- name: Setup for tests (macOS)
- runs-on: macos-latest
- outputs:
- integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }}
- templates_tests_matrix: ${{ steps.generate_tests_matrix.outputs.templates_tests_matrix }}
- steps:
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-
- - uses: ./.github/actions/enumerate-tests
- id: generate_tests_matrix
- with:
- includeIntegrations: true
- includeTemplates: true
-
- setup_for_tests_win:
- name: Setup for tests (Windows)
- runs-on: windows-latest
- outputs:
- integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }}
- templates_tests_matrix: ${{ steps.generate_tests_matrix.outputs.templates_tests_matrix }}
- steps:
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+ buildArgs: '/p:IncludeTemplateTests=true /p:IncludeCliE2ETests=${{ github.event_name == ''pull_request'' }}'
- - uses: ./.github/actions/enumerate-tests
- id: generate_tests_matrix
- with:
- includeIntegrations: true
- includeTemplates: true
+ - name: Split matrix by dependency type
+ id: split_matrix
+ shell: pwsh
+ run: |
+ $splitScript = "${{ github.workspace }}/eng/scripts/split-test-matrix-by-deps.ps1"
+ & $splitScript -AllTestsMatrix '${{ steps.generate_tests_matrix.outputs.all_tests }}' -OutputToGitHubEnv
build_packages:
name: Build packages
@@ -72,133 +45,82 @@ jobs:
with:
versionOverrideArg: ${{ inputs.versionOverrideArg }}
- integrations_test_lin:
- uses: ./.github/workflows/run-tests.yml
- name: Integrations Linux
- needs: setup_for_tests_lin
- strategy:
- fail-fast: false
- matrix:
- ${{ fromJson(needs.setup_for_tests_lin.outputs.integrations_tests_matrix) }}
- with:
- testShortName: ${{ matrix.shortname }}
- os: "ubuntu-latest"
- # Docker tests are run on linux, and Hosting tests take longer to finish
- testSessionTimeout: ${{ matrix.shortname == 'Hosting' && '25m' || '15m' }}
- extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\""
- versionOverrideArg: ${{ inputs.versionOverrideArg }}
-
- integrations_test_macos:
- uses: ./.github/workflows/run-tests.yml
- name: Integrations macos
- needs: setup_for_tests_macos
- strategy:
- fail-fast: false
- matrix:
- ${{ fromJson(needs.setup_for_tests_macos.outputs.integrations_tests_matrix) }}
- with:
- testShortName: ${{ matrix.shortname }}
- os: "macos-latest"
- extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\""
- versionOverrideArg: ${{ inputs.versionOverrideArg }}
-
- integrations_test_win:
- uses: ./.github/workflows/run-tests.yml
- name: Integrations Windows
- needs: setup_for_tests_win
- strategy:
- fail-fast: false
- matrix:
- ${{ fromJson(needs.setup_for_tests_win.outputs.integrations_tests_matrix) }}
- with:
- testShortName: ${{ matrix.shortname }}
- os: "windows-latest"
- extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\""
- versionOverrideArg: ${{ inputs.versionOverrideArg }}
-
- templates_test_lin:
- name: Templates Linux
+ tests_no_nugets:
+ name: ${{ matrix.shortname }}
uses: ./.github/workflows/run-tests.yml
- needs: [setup_for_tests_lin, build_packages]
+ needs: setup_for_tests
+ if: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets).include[0] != null }}
strategy:
fail-fast: false
- matrix: ${{ fromJson(needs.setup_for_tests_lin.outputs.templates_tests_matrix) }}
+ matrix: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets) }}
with:
testShortName: ${{ matrix.shortname }}
- os: "ubuntu-latest"
- testProjectPath: tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj
- testSessionTimeout: 20m
- testHangTimeout: "12m"
- extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true --filter-class Aspire.Templates.Tests.${{ matrix.shortname }}"
+ os: ${{ matrix.runs-on }}
+ testProjectPath: ${{ matrix.testProjectPath }}
+ testSessionTimeout: ${{ matrix.testSessionTimeout }}
+ testHangTimeout: ${{ matrix.testHangTimeout }}
+ extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}"
versionOverrideArg: ${{ inputs.versionOverrideArg }}
- requiresNugets: true
- requiresTestSdk: true
+ requiresNugets: ${{ matrix.requiresNugets }}
+ requiresTestSdk: ${{ matrix.requiresTestSdk }}
- templates_test_macos:
- name: Templates macos
+ tests_no_nugets_overflow:
+ name: ${{ matrix.shortname }}
uses: ./.github/workflows/run-tests.yml
- needs: [setup_for_tests_macos, build_packages]
+ needs: setup_for_tests
+ if: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets_overflow).include[0] != null }}
strategy:
fail-fast: false
- matrix: ${{ fromJson(needs.setup_for_tests_macos.outputs.templates_tests_matrix) }}
+ matrix: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets_overflow) }}
with:
testShortName: ${{ matrix.shortname }}
- os: "macos-latest"
- testProjectPath: tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj
- testSessionTimeout: 20m
- testHangTimeout: "12m"
- extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true --filter-class Aspire.Templates.Tests.${{ matrix.shortname }}"
+ os: ${{ matrix.runs-on }}
+ testProjectPath: ${{ matrix.testProjectPath }}
+ testSessionTimeout: ${{ matrix.testSessionTimeout }}
+ testHangTimeout: ${{ matrix.testHangTimeout }}
+ extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}"
versionOverrideArg: ${{ inputs.versionOverrideArg }}
- requiresNugets: true
- requiresTestSdk: true
+ requiresNugets: ${{ matrix.requiresNugets }}
+ requiresTestSdk: ${{ matrix.requiresTestSdk }}
- templates_test_win:
- name: Templates Windows
+ tests_requires_nugets:
+ name: ${{ matrix.shortname }}
uses: ./.github/workflows/run-tests.yml
- needs: [setup_for_tests_win, build_packages]
+ needs: [setup_for_tests, build_packages]
+ if: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets).include[0] != null }}
strategy:
fail-fast: false
- matrix: ${{ fromJson(needs.setup_for_tests_win.outputs.templates_tests_matrix) }}
+ matrix: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets) }}
with:
testShortName: ${{ matrix.shortname }}
- os: "windows-latest"
- testProjectPath: tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj
- testSessionTimeout: 20m
- testHangTimeout: "12m"
- extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true --filter-class Aspire.Templates.Tests.${{ matrix.shortname }}"
- versionOverrideArg: ${{ inputs.versionOverrideArg }}
- requiresNugets: true
- requiresTestSdk: true
-
- endtoend_tests:
- name: EndToEnd Linux
- uses: ./.github/workflows/run-tests.yml
- needs: build_packages
- with:
- testShortName: EndToEnd
- # EndToEnd is not run on Windows/macOS due to missing Docker support
- os: ubuntu-latest
- testProjectPath: tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj
- requiresNugets: true
+ os: ${{ matrix.runs-on }}
+ testProjectPath: ${{ matrix.testProjectPath }}
+ testSessionTimeout: ${{ matrix.testSessionTimeout }}
+ testHangTimeout: ${{ matrix.testHangTimeout }}
+ extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}"
versionOverrideArg: ${{ inputs.versionOverrideArg }}
+ requiresNugets: ${{ matrix.requiresNugets }}
+ requiresTestSdk: ${{ matrix.requiresTestSdk }}
- cli_e2e_tests:
- name: Cli E2E Linux
- # Only run CLI E2E tests during PR builds
- if: ${{ github.event_name == 'pull_request' }}
+ tests_requires_cli_archive:
+ name: ${{ matrix.shortname }}
uses: ./.github/workflows/run-tests.yml
- needs: [setup_for_tests_lin, build_packages, build_cli_archives]
+ needs: [setup_for_tests, build_packages, build_cli_archives]
+ if: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_cli_archive).include[0] != null }}
strategy:
fail-fast: false
- matrix: ${{ fromJson(needs.setup_for_tests_lin.outputs.cli_e2e_tests_matrix) }}
+ matrix: ${{ fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_cli_archive) }}
with:
testShortName: ${{ matrix.shortname }}
- os: "ubuntu-latest"
- testProjectPath: tests/Aspire.Cli.EndToEnd.Tests/Aspire.Cli.EndToEnd.Tests.csproj
- testSessionTimeout: 20m
- testHangTimeout: "12m"
- extraTestArgs: "--filter-not-trait quarantined=true --filter-not-trait outerloop=true --filter-class Aspire.Cli.EndToEnd.Tests.${{ matrix.shortname }}"
+ os: ${{ matrix.runs-on }}
+ testProjectPath: ${{ matrix.testProjectPath }}
+ testSessionTimeout: ${{ matrix.testSessionTimeout }}
+ testHangTimeout: ${{ matrix.testHangTimeout }}
+ extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}"
versionOverrideArg: ${{ inputs.versionOverrideArg }}
+ requiresNugets: ${{ matrix.requiresNugets }}
+ requiresTestSdk: ${{ matrix.requiresTestSdk }}
+ requiresCliArchive: true
polyglot_validation:
name: Polyglot SDK Validation
@@ -219,7 +141,7 @@ jobs:
- name: Setup Node.js environment
uses: actions/setup-node@v2
with:
- node-version: ${{ matrix.node-version }}
+ node-version: '20.x'
- name: Install dependencies
run: npm install
- name: Run tests
@@ -237,17 +159,12 @@ jobs:
runs-on: ubuntu-latest
name: Final Test Results
needs: [
- build_cli_archives,
- cli_e2e_tests,
- endtoend_tests,
extension_tests_win,
- integrations_test_lin,
- integrations_test_macos,
- integrations_test_win,
+ tests_no_nugets,
+ tests_no_nugets_overflow,
+ tests_requires_nugets,
+ tests_requires_cli_archive,
polyglot_validation,
- templates_test_lin,
- templates_test_macos,
- templates_test_win
]
steps:
- name: Checkout code
@@ -290,23 +207,24 @@ jobs:
- name: Fail if any dependency failed
# 'skipped' can be when a transitive dependency fails and the dependent job gets 'skipped'.
- # For example, one of setup_* jobs failing and the Integration test jobs getting 'skipped'
- # Note: cli_e2e_tests is intentionally skipped on non-PR builds, so we check it separately
+ # For example, one of setup_* jobs failing and the dependent test jobs getting 'skipped'.
+ # Overflow jobs are intentionally skipped when below the overflow threshold, so we
+ # check specific job names rather than using contains(needs.*.result, 'skipped').
if: >-
${{ always() &&
(contains(needs.*.result, 'failure') ||
contains(needs.*.result, 'cancelled') ||
- (github.event_name == 'pull_request' && contains(needs.*.result, 'skipped')) ||
+ (github.event_name == 'pull_request' &&
+ (needs.extension_tests_win.result == 'skipped' ||
+ needs.tests_no_nugets.result == 'skipped' ||
+ needs.tests_requires_nugets.result == 'skipped' ||
+ needs.tests_requires_cli_archive.result == 'skipped' ||
+ needs.polyglot_validation.result == 'skipped')) ||
(github.event_name != 'pull_request' &&
- (needs.endtoend_tests.result == 'skipped' ||
- needs.extension_tests_win.result == 'skipped' ||
- needs.integrations_test_lin.result == 'skipped' ||
- needs.integrations_test_macos.result == 'skipped' ||
- needs.integrations_test_win.result == 'skipped' ||
- needs.polyglot_validation.result == 'skipped' ||
- needs.templates_test_lin.result == 'skipped' ||
- needs.templates_test_macos.result == 'skipped' ||
- needs.templates_test_win.result == 'skipped'))) }}
+ (needs.extension_tests_win.result == 'skipped' ||
+ needs.tests_no_nugets.result == 'skipped' ||
+ needs.tests_requires_nugets.result == 'skipped' ||
+ needs.polyglot_validation.result == 'skipped'))) }}
run: |
echo "One or more dependent jobs failed."
exit 1
diff --git a/Directory.Packages.props b/Directory.Packages.props
index 60afe5b7ee4..118112abbc0 100644
--- a/Directory.Packages.props
+++ b/Directory.Packages.props
@@ -132,6 +132,8 @@
+
+
diff --git a/docs/ci/TestingOnCI.md b/docs/ci/TestingOnCI.md
new file mode 100644
index 00000000000..6713a9456ed
--- /dev/null
+++ b/docs/ci/TestingOnCI.md
@@ -0,0 +1,300 @@
+# Testing on CI
+
+This document describes the test infrastructure for CI pipelines in the Aspire repository. The infrastructure is designed to be platform-agnostic, with a canonical matrix format that can be consumed by GitHub Actions, Azure DevOps, or other CI systems.
+
+## Overview
+
+The CI test infrastructure uses a unified matrix generation system that:
+
+1. Enumerates all test projects and their metadata
+2. Optionally splits large test projects into parallel jobs
+3. Generates a canonical test matrix (platform-agnostic)
+4. Expands the matrix for specific CI platforms (GitHub Actions, Azure DevOps)
+5. Runs tests in parallel across multiple operating systems
+
+## Architecture
+
+```text
+┌─────────────────────────────────────────────────────────────────────┐
+│ MSBuild Phase │
+│ (TestEnumerationRunsheetBuilder.targets + build-test-matrix.ps1) │
+│ │
+│ ┌──────────────┐ ┌──────────────┐ ┌─────────────────────┐ │
+│ │ .tests- │ │ .tests- │ │ canonical-test- │ │
+│ │ metadata.json│ ─► │ partitions. │ ─► │ matrix.json │ │
+│ │ (per project)│ │ json (split) │ │ (canonical format) │ │
+│ └──────────────┘ └──────────────┘ └─────────────────────┘ │
+└─────────────────────────────────────────────────────────────────────┘
+ │
+ ▼
+┌─────────────────────────────────────────────────────────────────────┐
+│ Platform-Specific Expansion │
+│ │
+│ ┌─────────────────────────┐ ┌─────────────────────────┐ │
+│ │ expand-test-matrix- │ │ (future) │ │
+│ │ github.ps1 │ │ expand-test-matrix- │ │
+│ │ • OS → runner mapping │ │ azdo.ps1 │ │
+│ │ • { "include": [...] } │ │ • OS → vmImage mapping │ │
+│ └─────────────────────────┘ └─────────────────────────┘ │
+└─────────────────────────────────────────────────────────────────────┘
+```
+
+## Test Matrix Generation Flow
+
+### Phase 1: Test Enumeration
+
+The `enumerate-tests` GitHub Action (`.github/actions/enumerate-tests/action.yml`) triggers a special build:
+
+```bash
+./build.sh -test /p:TestRunnerName=TestEnumerationRunsheetBuilder
+```
+
+This invokes `eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets` for each test project, which:
+
+- Checks OS compatibility via `RunOnGithubActions{Linux/Windows/MacOS}` properties
+- Determines if the project uses test splitting (`SplitTestsOnCI` property)
+- Writes a `.tests-metadata.json` file to `artifacts/helix/` containing:
+ - `projectName`, `shortName`, `testProjectPath`
+ - `supportedOSes` array (e.g., `["windows", "linux", "macos"]`)
+ - `requiresNugets`, `requiresTestSdk`, `requiresCliArchive` flags
+ - `enablePlaywrightInstall` flag
+ - `testSessionTimeout`, `testHangTimeout` values
+ - `uncollectedTestsSessionTimeout`, `uncollectedTestsHangTimeout` values
+ - `splitTests` flag
+
+### Phase 2: Test Partition Discovery
+
+For projects with `SplitTestsOnCI=true`, the `GenerateTestPartitionsForCI` target runs `eng/scripts/split-test-projects-for-ci.ps1`, which:
+
+1. **Attempts partition extraction**: Uses `tools/ExtractTestPartitions` to scan the test assembly for `[Trait("Partition", "name")]` attributes on test classes
+2. **If partitions found**: Writes entries like `collection:PartitionName` plus `uncollected:*` (safety net for tests without partition traits). Note: the term "collection" here refers to partition groups, not xUnit `[Collection]` attributes which serve a different purpose (shared test fixtures).
+3. **If no partitions found**: Falls back to class-based splitting using `--list-tests` output, writing entries like `class:Namespace.ClassName`
+
+Output: `.tests-partitions.json` file alongside the metadata file.
+
+### Phase 3: Canonical Matrix Generation
+
+After all projects build, `eng/AfterSolutionBuild.targets` runs `eng/scripts/build-test-matrix.ps1`, which:
+
+1. Collects all `.tests-metadata.json` files
+2. For split test projects, reads the corresponding `.tests-partitions.json`
+3. Applies default values for missing properties
+4. Normalizes boolean values
+5. Creates matrix entries:
+ - **Regular tests**: One entry per project
+ - **Partition-based splits**: One entry per partition + one for `uncollected:*`
+ - **Class-based splits**: One entry per test class
+6. Outputs `artifacts/canonical-test-matrix.json` in canonical format (flat array with `requiresNugets`, `requiresCliArchive` booleans per entry)
+
+**Canonical format:**
+```json
+{
+ "tests": [
+ {
+ "name": "Templates-StarterTests",
+ "shortname": "Templates-StarterTests",
+ "testProjectPath": "tests/Aspire.Templates.Tests/...",
+ "supportedOSes": ["windows", "linux", "macos"],
+ "requiresNugets": true,
+ "requiresTestSdk": true,
+ "testSessionTimeout": "20m",
+ "testHangTimeout": "10m",
+ "extraTestArgs": "--filter-class \"...\""
+ },
+ {
+ "name": "Hosting-Docker",
+ "shortname": "Hosting-Docker",
+ "testProjectPath": "tests/Aspire.Hosting.Tests/...",
+ "supportedOSes": ["linux"],
+ "requiresNugets": false,
+ "testSessionTimeout": "30m",
+ "extraTestArgs": "--filter-trait \"Partition=Docker\""
+ }
+ ]
+}
+```
+
+### Phase 4: Platform-Specific Expansion
+
+Each CI platform has a thin script that transforms the canonical matrix:
+
+**GitHub Actions** (`eng/scripts/expand-test-matrix-github.ps1`):
+- Expands each entry for every OS in its `supportedOSes` array
+- Maps OS names to GitHub runners (`linux` → `ubuntu-latest`, etc.)
+- Splits entries into categories by dependency requirements:
+ - `no_nugets` — tests with no package dependencies
+ - `requires_nugets` — tests needing built NuGet packages
+ - `requires_cli_archive` — tests needing native CLI archives
+- Applies overflow splitting for the `no_nugets` category (threshold: 250 entries) to stay under the GitHub Actions 256-job-per-matrix limit
+- Outputs 4 GitHub Actions matrices: `no_nugets` (primary), `no_nugets_overflow`, `requires_nugets`, `requires_cli_archive`
+
+**Azure DevOps** (future):
+- Would map OS names to vmImage or pool names
+- Would output Azure DevOps matrix format: `{ ConfigName: { vars } }`
+
+This separation keeps 90% of the logic platform-agnostic while allowing each CI system to use its native matrix format.
+
+### Phase 5: Test Execution
+
+In `.github/workflows/tests.yml`, the workflow:
+
+1. Receives 4 pre-split matrices from the `enumerate-tests` action (split by `expand-test-matrix-github.ps1`)
+2. Runs 4 job groups using the split matrices:
+ - `tests_no_nugets`: Runs immediately after enumeration
+ - `tests_no_nugets_overflow`: Runs immediately (handles entries beyond the 250-entry threshold)
+ - `tests_requires_nugets`: Waits for `build_packages` job
+ - `tests_requires_cli_archive`: Waits for both `build_packages` and `build_cli_archives` jobs
+
+Each job invokes `.github/workflows/run-tests.yml` with matrix parameters including `extraTestArgs` for filtering (e.g., `--filter-trait "Partition=X"`).
+
+> **Note:** The workflow automatically prepends `--filter-not-trait "quarantined=true" --filter-not-trait "outerloop=true"` before any `extraTestArgs`, ensuring quarantined and outerloop tests are always excluded from the main test run.
+
+#### GitHub Actions 256-Job Limit
+
+GitHub Actions enforces a maximum of 256 jobs per `strategy.matrix`. To stay within this limit, the `no_nugets` category (typically the largest) is split into primary and overflow buckets at a threshold of 250 entries. If the total entry count is below 250, the overflow matrix is empty and the overflow job is skipped.
+
+## Enabling Test Splitting for a Project
+
+To split a test project into parallel CI jobs, add these properties to the `.csproj`:
+
+```xml
+
+
+ true
+ Aspire.YourProject.Tests
+
+
+ 30m
+ 15m
+ 15m
+ 10m
+
+```
+
+### Using Partition Traits (Recommended)
+
+For explicit control over test grouping, add `[Trait("Partition", "name")]` to test classes:
+
+```csharp
+[Trait("Partition", "Docker")]
+public class DockerResourceTests
+{
+ // Tests that require Docker
+}
+
+[Trait("Partition", "Publishing")]
+public class PublishingTests
+{
+ // Publishing-related tests
+}
+```
+
+Tests without a `Partition` trait run in a separate `uncollected` job, ensuring nothing is missed.
+
+### Class-Based Splitting (Fallback)
+
+If no `Partition` traits are found, the infrastructure automatically falls back to class-based splitting, creating one CI job per test class. This is less efficient but requires no code changes.
+
+## Controlling OS Compatibility
+
+By default, tests run on all three platforms. To restrict a project to specific OSes:
+
+```xml
+
+
+ false
+ true
+ false
+
+```
+
+## Requiring NuGet Packages
+
+For tests that need the built Aspire packages (e.g., template tests, end-to-end tests):
+
+```xml
+
+ true
+
+ true
+
+```
+
+These tests wait for the `build_packages` job before running.
+
+## Requiring CLI Native Archives
+
+For tests that need native CLI archives (e.g., CLI end-to-end tests):
+
+```xml
+
+ true
+ true
+
+```
+
+These tests wait for both the `build_packages` and `build_cli_archives` jobs before running. The workflow also sets `GH_TOKEN`, `GITHUB_PR_NUMBER`, and `GITHUB_PR_HEAD_SHA` environment variables for CLI E2E test scenarios.
+
+## Enabling Playwright
+
+For tests that require Playwright browser automation:
+
+```xml
+
+ true
+
+```
+
+This flag is tracked in the test metadata and controls whether Playwright browsers are installed during the test build step.
+
+## Deployment Tests
+
+Deployment end-to-end tests have a separate flow from the standard test matrix:
+
+1. The `enumerate-tests` action builds the deployment test project with `GenerateTestPartitionsForCI`
+2. A separate `generate_deployment_matrix` step reads the generated partitions
+3. The deployment matrix is output independently from the main test matrices
+4. Deployment tests run in a dedicated workflow (`tests-deployment.yml`) with Azure credentials
+
+## File Artifacts
+
+During enumeration, these files are generated in `artifacts/`:
+
+| File | Description |
+|------|-------------|
+| `helix/.tests-metadata.json` | Project metadata (OS support, timeouts, flags) |
+| `helix/.tests-partitions.json` | Partition/class list for split projects |
+| `canonical-test-matrix.json` | Canonical matrix (platform-agnostic) |
+
+## Scripts Reference
+
+| Script | Purpose |
+|--------|---------|
+| `eng/scripts/build-test-matrix.ps1` | Generates canonical matrix from metadata files |
+| `eng/scripts/expand-test-matrix-github.ps1` | Expands canonical matrix for GitHub Actions |
+| `eng/scripts/split-test-projects-for-ci.ps1` | Discovers test partitions/classes for splitting |
+
+## Debugging Test Enumeration
+
+To run enumeration locally and inspect the generated matrix:
+
+```bash
+./build.sh -test \
+ /p:TestRunnerName=TestEnumerationRunsheetBuilder \
+ /p:TestMatrixOutputPath=artifacts/canonical-test-matrix.json \
+ /p:IncludeTemplateTests=true \
+ /p:GenerateCIPartitions=true
+```
+
+Then inspect:
+- `artifacts/helix/*.tests-metadata.json` for per-project metadata
+- `artifacts/helix/*.tests-partitions.json` for split test entries
+- `artifacts/canonical-test-matrix.json` for the canonical matrix
+
+To test GitHub-specific expansion locally:
+
+```powershell
+pwsh eng/scripts/expand-test-matrix-github.ps1 `
+ -CanonicalMatrixFile artifacts/canonical-test-matrix.json `
+ -OutputMatrixFile artifacts/github-matrix.json
+```
diff --git a/docs/ci/ci-trigger-patterns.md b/docs/ci/ci-trigger-patterns.md
new file mode 100644
index 00000000000..ab7f4cbc67d
--- /dev/null
+++ b/docs/ci/ci-trigger-patterns.md
@@ -0,0 +1,71 @@
+# CI Trigger Patterns
+
+## Overview
+
+The file `eng/testing/github-ci-trigger-patterns.txt` lists glob patterns for files whose changes do **not** require the full CI to run.
+
+When a pull request is opened or updated, the CI workflow (`ci.yml`) checks whether **all** changed files match at least one pattern in the file. If they do, the workflow is skipped (no build or test jobs run). This keeps CI fast for changes that only affect documentation, pipeline configuration, or unrelated workflow files.
+
+> **Note:** This mechanism applies only to **pull requests**. Pushes to `main` or `release/*` branches always run the full CI pipeline. The `check-changed-files` action explicitly rejects non-`pull_request` events.
+
+## Why a Separate File?
+
+Previously the patterns were inlined in `.github/workflows/ci.yml`. Any change to that file (even just adding a new pattern to skip CI) would trigger CI on itself. Moving the patterns to `eng/testing/github-ci-trigger-patterns.txt` decouples pattern maintenance from the workflow definition.
+
+## Pattern Syntax
+
+Patterns use a simple **glob** style:
+
+| Syntax | Meaning |
+|--------|---------|
+| `**` | Matches any path including directory separators (recursive) |
+| `*` | Matches any characters except a directory separator |
+| `.` | Treated as a literal dot — no backslash escaping needed |
+
+All other characters (letters, digits, `-`, `_`, `/`, etc.) are treated as literals.
+
+Lines starting with `#` and blank lines are ignored.
+
+### Examples
+
+```text
+# All Markdown files anywhere in the repo
+**.md
+
+# All files under eng/pipelines/ recursively
+eng/pipelines/**
+
+# A specific file
+eng/test-configuration.json
+
+# Workflow files matching a glob (e.g. labeler-promote.yml, labeler-train.yml)
+.github/workflows/labeler-*.yml
+```
+
+## How to Add a New Pattern
+
+To add files whose changes should not trigger CI:
+
+1. Open `eng/testing/github-ci-trigger-patterns.txt`.
+2. Add one pattern per line, optionally preceded by a comment.
+3. Submit a PR — CI will not run for that PR if all changed files match the patterns.
+
+> **Tip:** Changing the patterns file itself is listed as a skippable change (`eng/testing/github-ci-trigger-patterns.txt`), so a PR that only updates this file will not trigger CI.
+
+## How It Works
+
+The `.github/actions/check-changed-files` composite action:
+
+1. Reads `eng/testing/github-ci-trigger-patterns.txt` from the checked-out repository.
+2. Converts each glob pattern to an anchored ERE (Extended Regular Expression) regex:
+ - `**` → `.*`
+ - `*` → `[^/]*`
+ - `.` and other regex metacharacters (`+`, `?`, `[`, `]`, `(`, `)`, `|`) → escaped with `\`
+3. For every file changed in the PR, checks whether the file path matches at least one of the converted regexes.
+4. Outputs `only_changed=true` when every changed file matched, allowing the calling workflow to skip further jobs.
+
+## Related Files
+
+- `eng/testing/github-ci-trigger-patterns.txt` — the patterns file described on this page
+- `.github/actions/check-changed-files/action.yml` — the composite action that reads and evaluates the patterns
+- `.github/workflows/ci.yml` — the CI workflow that calls the action
diff --git a/docs/specs/safe-npm-tool-install.md b/docs/specs/safe-npm-tool-install.md
new file mode 100644
index 00000000000..ab4367bb022
--- /dev/null
+++ b/docs/specs/safe-npm-tool-install.md
@@ -0,0 +1,183 @@
+# Safe npm Global Tool Installation
+
+## Overview
+
+The Aspire CLI installs the `@playwright/cli` npm package as a global tool during `aspire agent init`. Because this tool runs with the user's full privileges, we must verify its authenticity and provenance before installation. This document describes the verification process, the threat model, and the reasoning behind each step.
+
+## Threat Model
+
+### What we're protecting against
+
+1. **Registry compromise** — An attacker gains write access to the npm registry and publishes a malicious version of `@playwright/cli`
+2. **Publish token theft** — An attacker steals a maintainer's npm publish token and publishes a tampered package
+3. **Man-in-the-middle** — An attacker intercepts the network request and substitutes a different tarball
+4. **Dependency confusion** — A malicious package with a similar name is installed instead of the intended one
+
+### What we're NOT protecting against
+
+- Compromise of the legitimate source repository (`microsoft/playwright-cli`) itself
+- Compromise of the GitHub Actions build infrastructure (Sigstore OIDC provider)
+- Compromise of the Sigstore transparency log infrastructure
+- Malicious code introduced through legitimate dependencies of `@playwright/cli`
+
+### Trust anchors
+
+Our verification chain relies on these trust anchors:
+
+| Trust anchor | What it provides | How it's protected |
+|---|---|---|
+| **npm registry** | Package metadata, tarball hosting | HTTPS/TLS, npm's infrastructure security |
+| **Sigstore (Fulcio + Rekor)** | Cryptographic attestation signatures | Public CA with OIDC federation, append-only transparency log, verified in-process via Sigstore .NET library with TUF trust root |
+| **GitHub Actions OIDC** | Builder identity claims in Sigstore certificates | GitHub's infrastructure security |
+| **Hardcoded expected values** | Package name, version range, expected source repository | Code review, our own release process |
+
+## Verification Process
+
+### Step 1: Resolve package version and metadata
+
+**Action:** Run `npm view @playwright/cli@{versionRange} version` and `npm view @playwright/cli@{version} dist.integrity` to get the resolved version and the registry's SRI integrity hash. The default version range is `>=0.1.1`, which resolves to the latest published version at or above 0.1.1. This can be overridden to a specific version via the `playwrightCliVersion` configuration key.
+
+**What this establishes:** We know the exact version we intend to install and the hash the registry claims for its tarball.
+
+**Trust basis:** npm registry over HTTPS/TLS.
+
+**Limitations:** If the registry is compromised, both the version and hash could be attacker-controlled. This step alone is insufficient — it only establishes what the registry *claims*.
+
+### Step 2: Check if already installed at a suitable version
+
+**Action:** Run `playwright-cli --version` and compare against the resolved version.
+
+**What this establishes:** Whether installation can be skipped entirely (already up-to-date or newer).
+
+**Trust basis:** The previously-installed binary. If the user's system is compromised, this could be spoofed, but that's outside our threat model.
+
+### Step 3: Verify Sigstore attestation and provenance metadata
+
+**Action:**
+1. Fetch the attestation bundle from `https://registry.npmjs.org/-/npm/v1/attestations/@playwright/cli@{version}`
+2. Find the attestation with `predicateType: "https://slsa.dev/provenance/v1"` (SLSA Build L3 provenance)
+3. Extract the Sigstore bundle from the `bundle` field of the attestation
+4. Cryptographically verify the Sigstore bundle using the `SigstoreVerifier` from the [Sigstore .NET library](https://github.com/mitchdenny/sigstore-dotnet), with a `VerificationPolicy` configured for `CertificateIdentity.ForGitHubActions("microsoft", "playwright-cli")`
+5. Base64-decode the DSSE envelope payload to extract the in-toto statement
+6. Verify the following fields from the provenance predicate:
+
+| Field | Location in payload | Expected value | What it proves |
+|---|---|---|---|
+| **Source repository** | `predicate.buildDefinition.externalParameters.workflow.repository` | `https://github.com/microsoft/playwright-cli` | The package was built from the legitimate source code |
+| **Workflow path** | `predicate.buildDefinition.externalParameters.workflow.path` | `.github/workflows/publish.yml` | The build used the expected CI pipeline, not an ad-hoc or attacker-injected workflow |
+| **Build type** | `predicate.buildDefinition.buildType` | `https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1` | The build ran on GitHub Actions, which implicitly confirms the OIDC token issuer is `https://token.actions.githubusercontent.com` |
+| **Workflow ref** | `predicate.buildDefinition.externalParameters.workflow.ref` | Validated via caller-provided callback (for `@playwright/cli`: kind=`tags`, name=`v{version}`) | The build was triggered from a version tag matching the package version, not an arbitrary branch or commit. The tag format is package-specific — different packages may use different conventions (e.g., `v0.1.1`, `0.1.1`, `@scope/pkg@0.1.1`). The ref is parsed into structured components (`WorkflowRefInfo`) and the caller provides a validation callback. |
+
+**What this establishes:** That the Sigstore bundle is cryptographically authentic — the signing certificate was issued by Sigstore's Fulcio CA, the signature is recorded in the Rekor transparency log, and the OIDC identity in the certificate matches the `microsoft/playwright-cli` GitHub Actions workflow. Additionally, the provenance metadata confirms the package was built from the expected repository, workflow, CI system, and version tag.
+
+**Trust basis:** Sigstore's public key infrastructure via the `Sigstore` and `Tuf` .NET libraries. The TUF trust root is automatically downloaded and verified. Even if the npm registry is compromised, an attacker cannot forge valid Sigstore signatures — they would need to compromise Fulcio (the Sigstore CA) or obtain a valid OIDC token from GitHub Actions for the legitimate repository's workflow. Since the Sigstore verification and provenance field checking happen on the same attestation bundle in a single operation, there is no TOCTOU gap between signature verification and content inspection.
+
+**Why we verify all provenance fields:** Checking only the Sigstore certificate identity (GitHub Actions + repository) is necessary but not sufficient. An attacker with write access to the repo could introduce a malicious workflow (e.g., `.github/workflows/evil.yml`). By also verifying the workflow path, build type, and workflow ref, we ensure the package was built by the specific expected CI pipeline from a release tag.
+
+**Additional fields extracted but not directly verified:** The provenance parser also extracts `runDetails.builder.id` from the attestation. This is available in the `NpmProvenanceData` result for logging and diagnostics but is not currently used as a verification gate.
+
+### Step 4: Download and verify tarball integrity
+
+**Action:**
+1. Run `npm pack @playwright/cli@{version}` to download the tarball
+2. Compute SHA-512 hash of the downloaded tarball
+3. Compare against the SRI integrity hash obtained in Step 1
+
+**What this establishes:** That the tarball we have on disk is bit-for-bit identical to what the npm registry published for this version.
+
+**Trust basis:** Cryptographic hash comparison (SHA-512). If the hash matches, the content is the same regardless of how it was delivered.
+
+**Relationship to Step 3:** The Sigstore attestations verified in Step 3 are bound to the package version and its published content. The integrity hash in the registry packument is the canonical identifier for the tarball content. By verifying our tarball matches this hash, we establish that our tarball is the same artifact that the Sigstore attestations cover.
+
+### Step 5: Install globally from verified tarball
+
+**Action:** Run `npm install -g {tarballPath}` to install the verified tarball as a global tool.
+
+**What this establishes:** The tool is installed and available on the user's PATH.
+
+**Trust basis:** All preceding verification steps have passed. The tarball content has been verified against the registry's published hash (Step 4), the Sigstore attestations for that content are cryptographically valid (Step 3), and the attestations confirm the correct source repository, workflow, and build system (Step 3).
+
+### Step 6: Generate and mirror skill files
+
+**Action:** Run `playwright-cli install --skills` to generate agent skill files in the primary skill directory (`.claude/skills/playwright-cli/`), then mirror the skill directory to all other detected agent environment skill directories (e.g., `.github/skills/playwright-cli/`, `.opencode/skill/playwright-cli/`). The mirror is a full sync — files are created, updated, and stale files are removed so all environments have identical skill content.
+
+**What this establishes:** The Playwright CLI skill files are available for all configured agent environments.
+
+## Verification Chain Summary
+
+```text
+ ┌──────────────────────────────┐
+ │ Hardcoded expectations │
+ │ • Package: @playwright/cli │
+ │ • Version range: >=0.1.1 │
+ │ • Source: microsoft/ │
+ │ playwright-cli │
+ │ • Workflow: .github/ │
+ │ workflows/publish.yml │
+ │ • Build type: GitHub Actions │
+ │ workflow/v1 │
+ └──────────────┬────────────────┘
+ │
+ ┌──────────────▼────────────────┐
+ │ Step 1: Resolve version + │
+ │ integrity hash from registry │
+ └──────────────┬────────────────┘
+ │
+ ┌────────────────────┼────────────────────┐
+ │ │
+ ┌──────────▼──────────────┐ ┌─────────▼─────────┐
+ │ Step 3: Sigstore verify │ │ Step 4: npm pack │
+ │ + provenance checks │ │ + SHA-512 check │
+ │ (in-process via Sigstore │ │ (tarball │
+ │ .NET library + TUF) │ │ integrity) │
+ └──────────┬───────────────┘ └─────────┬─────────┘
+ │ │
+ │ Attestation is authentic + │ Tarball matches
+ │ built from expected repo + │ published hash
+ │ expected pipeline │
+ └────────────────────┬────────────────────┘
+ │
+ ┌──────────────▼────────────────┐
+ │ Step 5: npm install -g │
+ │ (from verified tarball) │
+ └───────────────────────────────┘
+```
+
+## Residual Risks
+
+### 1. Time-of-check-to-time-of-use (TOCTOU)
+
+**Risk:** The package could be replaced on the registry between our verification steps and the global install.
+
+**Mitigation:** We verify the SHA-512 hash of the tarball we actually install (Step 4), and we install from the local tarball file (not from the registry again). The verified tarball is the same file that gets installed.
+
+### 2. Transitive dependency attacks
+
+**Risk:** `@playwright/cli` has dependencies that could be compromised.
+
+**Mitigation:** The `--ignore-scripts` flag prevents execution of install scripts. However, the dependencies' code runs when the tool is invoked. This is partially mitigated by Sigstore attestations covering the dependency tree, but comprehensive supply chain verification of all transitive dependencies is out of scope.
+
+## Implementation Constants
+
+```csharp
+internal const string PackageName = "@playwright/cli";
+internal const string VersionRange = ">=0.1.1";
+internal const string ExpectedSourceRepository = "https://github.com/microsoft/playwright-cli";
+internal const string ExpectedWorkflowPath = ".github/workflows/publish.yml";
+internal const string ExpectedBuildType = "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1";
+internal const string NpmRegistryAttestationsBaseUrl = "https://registry.npmjs.org/-/npm/v1/attestations";
+internal const string SlsaProvenancePredicateType = "https://slsa.dev/provenance/v1";
+```
+
+## Configuration
+
+Two break-glass configuration keys are available via `aspire config set`:
+
+| Key | Effect |
+|---|---|
+| `disablePlaywrightCliPackageValidation` | When `"true"`, skips all Sigstore, provenance, and integrity checks. Use only for debugging npm service issues. |
+| `playwrightCliVersion` | When set, overrides the version range and pins to the specified exact version. |
+
+## Future Improvements
+
+1. **Pinned tarball hash** — Ship a known-good SRI hash with each Aspire release, eliminating the need to trust the registry for the hash at all.
diff --git a/eng/AfterSolutionBuild.targets b/eng/AfterSolutionBuild.targets
index 3288a591e3d..411f444dd5e 100644
--- a/eng/AfterSolutionBuild.targets
+++ b/eng/AfterSolutionBuild.targets
@@ -50,7 +50,7 @@
```
-->
-
+
<_CombinedRunsheetFile>$(ArtifactsTmpDir)/combined_runsheet.json
<_Command>
@@ -105,4 +105,26 @@
+
+
+
+
+
+
+ <_TestMatrixOutputPath>$([MSBuild]::NormalizePath('$(RepoRoot)', '$(TestMatrixOutputPath)'))
+ <_BuildMatrixScript>$([MSBuild]::NormalizePath($(RepoRoot), 'eng', 'scripts', 'build-test-matrix.ps1'))
+
+
+
+
+
+
+
+
diff --git a/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets
new file mode 100644
index 00000000000..849b1a73b9f
--- /dev/null
+++ b/eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.targets
@@ -0,0 +1,122 @@
+
+
+
+
+
+ <_NormalizedProjectDirectory>$([System.String]::Copy('$(MSBuildProjectDirectory)').Replace('\','/'))
+
+ <_ShouldSkipProject Condition="'$(_ShouldSkipProject)' == '' and $(_NormalizedProjectDirectory.Contains('tests/Shared'))">true
+ <_ShouldSkipProject Condition="'$(_ShouldSkipProject)' == '' and $(_NormalizedProjectDirectory.Contains('tests/testproject'))">true
+ <_ShouldSkipProject Condition="'$(_ShouldSkipProject)' == '' and $(_NormalizedProjectDirectory.Contains('tests/TestingAppHost1'))">true
+
+ <_ShouldSkipProject Condition="'$(_ShouldSkipProject)' == '' and '$(IncludeTemplateTests)' != 'true' and '$(MSBuildProjectName)' == 'Aspire.Templates.Tests'">true
+ <_ShouldSkipProject Condition="'$(_ShouldSkipProject)' == '' and '$(IncludeCliE2ETests)' != 'true' and '$(MSBuildProjectName)' == 'Aspire.Cli.EndToEnd.Tests'">true
+
+
+ <_ShouldSkipProject Condition="'$(_ShouldSkipProject)' == '' and '$(OnlyDeploymentTests)' != 'true' and '$(MSBuildProjectName)' == 'Aspire.Deployment.EndToEnd.Tests'">true
+
+ <_ShouldSkipProject Condition="'$(_ShouldSkipProject)' == '' and '$(OnlyDeploymentTests)' == 'true' and '$(MSBuildProjectName)' != 'Aspire.Deployment.EndToEnd.Tests'">true
+
+ <_ShouldSkipProject Condition="'$(_ShouldSkipProject)' == '' and '$(SkipTests)' == 'true'">true
+
+
+
+
+
+
+
+
+
+
+ <_RelativeProjectPath>$([System.String]::Copy('$(MSBuildProjectFullPath)').Replace('$(RepoRoot)', ''))
+
+ <_RelativeProjectPath Condition="$(_RelativeProjectPath.StartsWith('/')) or $(_RelativeProjectPath.StartsWith('\\'))">$(_RelativeProjectPath.Substring(1))
+
+
+ <_RequiresNugets>false
+ <_RequiresNugets Condition="'$(RequiresNugets)' == 'true'">true
+
+
+ <_RequiresTestSdk>false
+ <_RequiresTestSdk Condition="'$(RequiresTestSdk)' == 'true'">true
+
+
+ <_RequiresCliArchive>false
+ <_RequiresCliArchive Condition="'$(RequiresCliArchive)' == 'true'">true
+
+
+ <_EnablePlaywrightInstall>false
+ <_EnablePlaywrightInstall Condition="'$(EnablePlaywrightInstall)' == 'true'">true
+
+
+ <_SupportedOSesJson />
+ <_SupportedOSesJson Condition="'$(RunOnGithubActionsWindows)' == 'true'">$(_SupportedOSesJson)"windows",
+ <_SupportedOSesJson Condition="'$(RunOnGithubActionsLinux)' == 'true'">$(_SupportedOSesJson)"linux",
+ <_SupportedOSesJson Condition="'$(RunOnGithubActionsMacOS)' == 'true'">$(_SupportedOSesJson)"macos",
+
+ <_SupportedOSesJson Condition="'$(_SupportedOSesJson)' != ''">$(_SupportedOSesJson.TrimEnd(','))
+
+
+ <_TestSessionTimeout Condition="'$(TestSessionTimeout)' != ''">$(TestSessionTimeout)
+ <_TestSessionTimeout Condition="'$(TestSessionTimeout)' == ''">20m
+ <_TestHangTimeout Condition="'$(TestHangTimeout)' != ''">$(TestHangTimeout)
+ <_TestHangTimeout Condition="'$(TestHangTimeout)' == ''">10m
+ <_UncollectedTestsSessionTimeout Condition="'$(UncollectedTestsSessionTimeout)' != ''">$(UncollectedTestsSessionTimeout)
+ <_UncollectedTestsSessionTimeout Condition="'$(UncollectedTestsSessionTimeout)' == ''">15m
+ <_UncollectedTestsHangTimeout Condition="'$(UncollectedTestsHangTimeout)' != ''">$(UncollectedTestsHangTimeout)
+ <_UncollectedTestsHangTimeout Condition="'$(UncollectedTestsHangTimeout)' == ''">10m
+
+
+ <_MetadataJson>{
+ "projectName": "$(MSBuildProjectName)",
+ "shortName": "$(_ShortName)",
+ "testClassNamesPrefix": "$(MSBuildProjectName)",
+ "testProjectPath": "$(_RelativeProjectPath)",
+ "requiresNugets": "$(_RequiresNugets.ToLowerInvariant())",
+ "requiresTestSdk": "$(_RequiresTestSdk.ToLowerInvariant())",
+ "requiresCliArchive": "$(_RequiresCliArchive.ToLowerInvariant())",
+ "enablePlaywrightInstall": "$(_EnablePlaywrightInstall.ToLowerInvariant())",
+ "splitTests": "$(SplitTestsOnCI)",
+ "supportedOSes": [$(_SupportedOSesJson)],
+ "testSessionTimeout": "$(_TestSessionTimeout)",
+ "testHangTimeout": "$(_TestHangTimeout)",
+ "uncollectedTestsSessionTimeout": "$(_UncollectedTestsSessionTimeout)",
+ "uncollectedTestsHangTimeout": "$(_UncollectedTestsHangTimeout)"
+}
+
+
+ <_MetadataOutputDir Condition="'$(TestArchiveTestsDir)' != ''">$(TestArchiveTestsDir)
+ <_MetadataOutputDir Condition="'$(TestArchiveTestsDir)' == ''">$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix'))
+ <_MetadataFile>$(_MetadataOutputDir)$(MSBuildProjectName).tests-metadata.json
+
+
+
+
+
+
+
+
+
+
diff --git a/eng/TestRunsheetBuilder/TestRunsheetBuilder.targets b/eng/TestRunsheetBuilder/TestRunsheetBuilder.targets
deleted file mode 100644
index 7ca8ca2b8c1..00000000000
--- a/eng/TestRunsheetBuilder/TestRunsheetBuilder.targets
+++ /dev/null
@@ -1,124 +0,0 @@
-
-
-
-
-
-
-
- <_RequiresPackages Condition=" '$(MSBuildProjectName)' == 'Aspire.EndToEnd.Tests' ">true
-
- <_CreateRunsheet>false
- <_CreateRunsheet Condition=" '$(_RequiresPackages)' == 'true' and '$(FullE2e)' == 'true' ">true
- <_CreateRunsheet Condition=" '$(_RequiresPackages)' != 'true' and '$(FullE2e)' != 'true' ">true
-
- <_CreateRunsheet Condition=" '$(MSBuildProjectName)' == 'Aspire.Templates.Tests' ">false
-
- <_PreCommand>$(TestRunnerPreCommand)
-
-
-
-
- <_TestRunsheet>$([System.String]::Copy('$(MSBuildProjectName)').Replace('Aspire.', ''))
- <_TestRunsheetFileNameWindows>$(ArtifactsTmpDir)/$(_TestRunsheet).win.runsheet.json
- <_TestRunsheetFileNameLinux>$(ArtifactsTmpDir)/$(_TestRunsheet).linux.runsheet.json
- <_TestRunsheetFileNameMacOS>$(ArtifactsTmpDir)/$(_TestRunsheet).macos.runsheet.json
-
- <_TestBinLog>$([MSBuild]::NormalizePath($(ArtifactsLogDir), '$(_TestRunsheet).binlog'))
-
- <_RelativeTestProjectPath>$([System.String]::Copy('$(MSBuildProjectFullPath)').Replace('$(RepoRoot)', '%24(pwd)/'))
- <_RelativeTestBinLog>$([System.String]::Copy('$(_TestBinLog)').Replace('$(RepoRoot)', '%24(pwd)/'))
-
- <_TestRunnerWindows>./eng/build.ps1
- <_TestRunnerLinux>./eng/build.sh
- <_TestRunnerMacOS>./eng/build.sh
- <_TestCommand>-restore -build -test -projects "$(_RelativeTestProjectPath)" /bl:"$(_RelativeTestBinLog)" -c $(Configuration) -ci
-
-
- <_TestCommand Condition=" '$(_RequiresPackages)' == 'true' ">$(_TestCommand) /p:TestsRunningOutsideOfRepo=true
-
-
- <_PreCommand>$([System.String]::Copy($(_PreCommand)).Replace("\", "/").Replace('"', '\"'))
- <_TestCommand>$([System.String]::Copy($(_TestCommand)).Replace("\", "/").Replace('"', '\"'))
-
- <_TestRunsheetWindows>{ "label": "w: $(_TestRunsheet)", "project": "$(_TestRunsheet)", "os": "windows-latest", "command": "./eng/build.ps1 $(_TestCommand)" }
- <_TestRunsheetLinux>{ "label": "l: $(_TestRunsheet)", "project": "$(_TestRunsheet)", "os": "ubuntu-latest", "command": "$(_PreCommand)./eng/build.sh $(_TestCommand)" }
- <_TestRunsheetMacOS>{ "label": "m: $(_TestRunsheet)", "project": "$(_TestRunsheet)", "os": "macos-latest", "command": "$(_PreCommand)./eng/build.sh $(_TestCommand)" }
-
-
-
- <_OutputFiles Include="$(_TestRunsheetFileNameWindows)" />
- <_OutputFiles Include="$(_TestRunsheetFileNameLinux)" />
- <_OutputFiles Include="$(_TestRunsheetFileNameMacOS)" />
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/eng/Testing.props b/eng/Testing.props
index d24ccae9410..4af4e6a93b0 100644
--- a/eng/Testing.props
+++ b/eng/Testing.props
@@ -2,6 +2,12 @@
true
+
+ 20m
+ 10m
+ 15m
+ 10m
+
true
true
@@ -15,6 +21,9 @@
<_NonQuarantinedTestRunAdditionalArgs>--filter-not-trait "quarantined=true"
<_OuterloopTestRunAdditionalArgs>--filter-trait "outerloop=true"
<_NonOuterloopTestRunAdditionalArgs>--filter-not-trait "outerloop=true"
+
+ <_ShortName Condition="'$(TestShortName)' != ''">$(TestShortName)
+ <_ShortName Condition="'$(_ShortName)' == ''">$([System.IO.Path]::GetFileNameWithoutExtension('$(MSBuildProjectName)').Replace('Aspire.', '').Replace('.Tests', ''))
diff --git a/eng/Testing.targets b/eng/Testing.targets
index 853a3949d3b..f9e66d2b601 100644
--- a/eng/Testing.targets
+++ b/eng/Testing.targets
@@ -43,6 +43,8 @@
If we haven't detected the tests are run on build agents, then we presume we're running tests locally.
-->
true
+
+ false
@@ -108,5 +110,4 @@
-
diff --git a/eng/pipelines/templates/BuildAndTest.yml b/eng/pipelines/templates/BuildAndTest.yml
index ab5a2b4977c..7abce19a784 100644
--- a/eng/pipelines/templates/BuildAndTest.yml
+++ b/eng/pipelines/templates/BuildAndTest.yml
@@ -237,6 +237,7 @@ steps:
-configuration ${{ parameters.buildConfig }}
-pack
/p:PrepareForHelix=${{ lower(eq(parameters.runHelixTests, 'true')) }}
+ /p:ArchiveTests=${{ lower(eq(parameters.runHelixTests, 'true')) }}
/bl:${{ parameters.repoLogPath }}/build.binlog
$(_OfficialBuildIdArgs)
displayName: Build
diff --git a/eng/scripts/build-test-matrix.ps1 b/eng/scripts/build-test-matrix.ps1
new file mode 100644
index 00000000000..68f74d29f55
--- /dev/null
+++ b/eng/scripts/build-test-matrix.ps1
@@ -0,0 +1,330 @@
+<#
+.SYNOPSIS
+ Builds the canonical test matrix from test enumeration files.
+
+.DESCRIPTION
+ This script processes test metadata files and generates a canonical test matrix
+ that can be consumed by any CI platform (GitHub Actions, Azure DevOps, etc.).
+
+ The script:
+ 1. Collects all .tests-metadata.json files from the artifacts directory
+ 2. Processes regular and split test projects
+ 3. Applies default values for missing properties
+ 4. Normalizes boolean values
+ 5. Outputs a canonical JSON format with supportedOSes arrays (not expanded)
+
+ The output format is platform-agnostic. Each CI platform should have a thin
+ script to expand supportedOSes into platform-specific runner configurations.
+
+.PARAMETER ArtifactsDir
+ Path to the artifacts directory containing .tests-metadata.json files.
+
+.PARAMETER OutputMatrixFile
+ Path to write the canonical test matrix JSON file.
+
+.NOTES
+ PowerShell 7+
+
+ Output format:
+ {
+ "tests": [ { entry with supportedOSes array and requiresNugets boolean }, ... ]
+ }
+#>
+
+[CmdletBinding()]
+param(
+ [Parameter(Mandatory=$true)]
+ [string]$ArtifactsDir,
+
+ [Parameter(Mandatory=$true)]
+ [string]$OutputMatrixFile
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version Latest
+
+# Default values applied to all entries
+$script:defaults = @{
+ extraTestArgs = ''
+ requiresNugets = $false
+ requiresTestSdk = $false
+ testSessionTimeout = '20m'
+ testHangTimeout = '10m'
+ supportedOSes = @('windows', 'linux', 'macos')
+}
+
+Write-Host "Building canonical test matrix"
+Write-Host "Artifacts directory: $ArtifactsDir"
+
+# Helper function to normalize boolean values
+function ConvertTo-Boolean {
+ param($Value)
+ return ($Value -eq 'true' -or $Value -eq $true)
+}
+
+# Helper function to apply defaults and normalize an entry
+function Complete-EntryWithDefaults {
+ param([Parameter(Mandatory=$true)]$Entry)
+
+ # Apply defaults for missing properties
+ if (-not $Entry['testSessionTimeout']) { $Entry['testSessionTimeout'] = $script:defaults.testSessionTimeout }
+ if (-not $Entry['testHangTimeout']) { $Entry['testHangTimeout'] = $script:defaults.testHangTimeout }
+ if (-not $Entry.Contains('extraTestArgs')) { $Entry['extraTestArgs'] = $script:defaults.extraTestArgs }
+ if (-not $Entry['supportedOSes'] -or $Entry['supportedOSes'].Count -eq 0) {
+ $Entry['supportedOSes'] = $script:defaults.supportedOSes
+ }
+
+ # Normalize boolean values
+ $Entry['requiresNugets'] = if ($Entry.Contains('requiresNugets')) { ConvertTo-Boolean $Entry['requiresNugets'] } else { $false }
+ $Entry['requiresTestSdk'] = if ($Entry.Contains('requiresTestSdk')) { ConvertTo-Boolean $Entry['requiresTestSdk'] } else { $false }
+ $Entry['requiresCliArchive'] = if ($Entry.Contains('requiresCliArchive')) { ConvertTo-Boolean $Entry['requiresCliArchive'] } else { $false }
+
+ return $Entry
+}
+
+# Helper function to create matrix entry for regular (non-split) tests
+function New-RegularTestEntry {
+ param(
+ [Parameter(Mandatory=$false)]
+ $Metadata = $null
+ )
+
+ $entry = [ordered]@{
+ type = 'regular'
+ projectName = $Metadata.projectName
+ name = $Metadata.shortName
+ shortname = $Metadata.shortName
+ testProjectPath = $Metadata.testProjectPath
+ workitemprefix = $Metadata.projectName
+ splitTests = $false
+ }
+
+ # Add metadata if available
+ if ($Metadata) {
+ if ($Metadata.PSObject.Properties['testSessionTimeout']) { $entry['testSessionTimeout'] = $Metadata.testSessionTimeout }
+ if ($Metadata.PSObject.Properties['testHangTimeout']) { $entry['testHangTimeout'] = $Metadata.testHangTimeout }
+ if ($Metadata.PSObject.Properties['requiresNugets']) { $entry['requiresNugets'] = $Metadata.requiresNugets }
+ if ($Metadata.PSObject.Properties['requiresTestSdk']) { $entry['requiresTestSdk'] = $Metadata.requiresTestSdk }
+ if ($Metadata.PSObject.Properties['requiresCliArchive']) { $entry['requiresCliArchive'] = $Metadata.requiresCliArchive }
+ if ($Metadata.PSObject.Properties['extraTestArgs'] -and $Metadata.extraTestArgs) { $entry['extraTestArgs'] = $Metadata.extraTestArgs }
+ }
+
+ # Add supported OSes
+ $entry['supportedOSes'] = @($Metadata.supportedOSes)
+
+ return Complete-EntryWithDefaults $entry
+}
+
+# Helper function to create matrix entry for collection-based split tests
+function New-CollectionTestEntry {
+ param(
+ [Parameter(Mandatory=$true)]
+ [string]$CollectionName,
+ [Parameter(Mandatory=$true)]
+ $Metadata,
+ [Parameter(Mandatory=$true)]
+ [bool]$IsUncollected
+ )
+
+ $suffix = if ($IsUncollected) { 'uncollected' } else { $CollectionName }
+ $baseShortName = if ($Metadata.shortName) { $Metadata.shortName } else { $Metadata.projectName }
+
+ $entry = [ordered]@{
+ type = 'collection'
+ projectName = $Metadata.projectName
+ name = if ($IsUncollected) { $baseShortName } else { "$baseShortName-$suffix" }
+ shortname = if ($IsUncollected) { $baseShortName } else { "$baseShortName-$suffix" }
+ testProjectPath = $Metadata.testProjectPath
+ workitemprefix = "$($Metadata.projectName)_$suffix"
+ collection = $CollectionName
+ splitTests = $true
+ }
+
+ # Use uncollected timeouts if available, otherwise use regular
+ if ($IsUncollected) {
+ if ($Metadata.PSObject.Properties['uncollectedTestsSessionTimeout']) {
+ $entry['testSessionTimeout'] = $Metadata.uncollectedTestsSessionTimeout
+ } elseif ($Metadata.PSObject.Properties['testSessionTimeout']) {
+ $entry['testSessionTimeout'] = $Metadata.testSessionTimeout
+ }
+
+ if ($Metadata.PSObject.Properties['uncollectedTestsHangTimeout']) {
+ $entry['testHangTimeout'] = $Metadata.uncollectedTestsHangTimeout
+ } elseif ($Metadata.PSObject.Properties['testHangTimeout']) {
+ $entry['testHangTimeout'] = $Metadata.testHangTimeout
+ }
+ } else {
+ if ($Metadata.PSObject.Properties['testSessionTimeout']) { $entry['testSessionTimeout'] = $Metadata.testSessionTimeout }
+ if ($Metadata.PSObject.Properties['testHangTimeout']) { $entry['testHangTimeout'] = $Metadata.testHangTimeout }
+ }
+
+ if ($Metadata.PSObject.Properties['requiresNugets']) { $entry['requiresNugets'] = $Metadata.requiresNugets }
+ if ($Metadata.PSObject.Properties['requiresTestSdk']) { $entry['requiresTestSdk'] = $Metadata.requiresTestSdk }
+ if ($Metadata.PSObject.Properties['requiresCliArchive']) { $entry['requiresCliArchive'] = $Metadata.requiresCliArchive }
+
+ # Add test filter for collection-based splitting
+ if ($IsUncollected) {
+ $entry['extraTestArgs'] = '--filter-not-trait "Partition=*"'
+ } else {
+ $entry['extraTestArgs'] = "--filter-trait `"Partition=$CollectionName`""
+ }
+
+ # Add supported OSes from metadata
+ if ($Metadata.PSObject.Properties['supportedOSes']) {
+ $entry['supportedOSes'] = @($Metadata.supportedOSes)
+ }
+
+ return Complete-EntryWithDefaults $entry
+}
+
+# Helper function to create matrix entry for class-based split tests
+function New-ClassTestEntry {
+ param(
+ [Parameter(Mandatory=$true)]
+ [string]$ClassName,
+ [Parameter(Mandatory=$true)]
+ $Metadata
+ )
+
+ # Extract short class name (last segment after last dot)
+ $shortClassName = $ClassName.Split('.')[-1]
+ $baseShortName = if ($Metadata.shortName) { $Metadata.shortName } else { $Metadata.projectName }
+
+ $entry = [ordered]@{
+ type = 'class'
+ projectName = $Metadata.projectName
+ name = "$baseShortName-$shortClassName"
+ shortname = "$baseShortName-$shortClassName"
+ testProjectPath = $Metadata.testProjectPath
+ workitemprefix = "$($Metadata.projectName)_$shortClassName"
+ classname = $ClassName
+ splitTests = $true
+ }
+
+ if ($Metadata.PSObject.Properties['testSessionTimeout']) { $entry['testSessionTimeout'] = $Metadata.testSessionTimeout }
+ if ($Metadata.PSObject.Properties['testHangTimeout']) { $entry['testHangTimeout'] = $Metadata.testHangTimeout }
+ if ($Metadata.PSObject.Properties['requiresNugets']) { $entry['requiresNugets'] = $Metadata.requiresNugets }
+ if ($Metadata.PSObject.Properties['requiresTestSdk']) { $entry['requiresTestSdk'] = $Metadata.requiresTestSdk }
+ if ($Metadata.PSObject.Properties['requiresCliArchive']) { $entry['requiresCliArchive'] = $Metadata.requiresCliArchive }
+
+ # Add test filter for class-based splitting
+ $entry['extraTestArgs'] = "--filter-class `"$ClassName`""
+
+ # Add supported OSes from metadata
+ if ($Metadata.PSObject.Properties['supportedOSes']) {
+ $entry['supportedOSes'] = @($Metadata.supportedOSes)
+ }
+
+ return Complete-EntryWithDefaults $entry
+}
+
+# 1. Collect all metadata files
+$metadataFiles = @(Get-ChildItem -Path $ArtifactsDir -Filter "*.tests-metadata.json" -Recurse -ErrorAction SilentlyContinue)
+
+if ($metadataFiles.Count -eq 0) {
+ Write-Warning "No test metadata files found in $ArtifactsDir"
+ # Create empty canonical matrix
+ $canonicalMatrix = @{
+ tests = @()
+ }
+ $canonicalMatrix | ConvertTo-Json -Depth 10 | Set-Content -Path $OutputMatrixFile -Encoding UTF8
+ Write-Host "Created empty test matrix: $OutputMatrixFile"
+ exit 0
+}
+
+Write-Host "Found $($metadataFiles.Count) test metadata file(s)"
+
+# 2. Build matrix entries
+$matrixEntries = [System.Collections.Generic.List[object]]::new()
+
+foreach ($metadataFile in $metadataFiles) {
+ $metadataObject = Get-Content -Raw -Path $metadataFile.FullName | ConvertFrom-Json
+
+ Write-Host "Processing: $($metadataObject.projectName)"
+
+ # Check if this is a split test with metadata
+ if ($metadataObject.splitTests -eq 'true') {
+ Write-Host " → Split test (processing partitions/classes)"
+
+ $metaFile = $metadataFile.FullName
+ $partitionsFile = $metaFile -replace '\.tests-metadata\.json$', '.tests-partitions.json'
+
+ if (-not (Test-Path $partitionsFile)) {
+ throw "Test partitions file not found: $partitionsFile"
+ }
+
+ $metadata = Get-Content -Raw -Path $metaFile | ConvertFrom-Json
+
+ # Add supported OSes to metadata from enumeration
+ $metadata | Add-Member -Force -MemberType NoteProperty -Name 'supportedOSes' -Value $metadataObject.supportedOSes
+
+ # Extract the array testPartitions
+ $partitionsJson = Get-Content -Raw -Path $partitionsFile | ConvertFrom-Json
+ $testPartitions = $partitionsJson.testPartitions
+
+ $partitionCount = 0
+ $classCount = 0
+
+ foreach ($testPartition in $testPartitions) {
+ $testPartition = $testPartition.Trim()
+ if ([string]::IsNullOrWhiteSpace($testPartition)) { continue }
+
+ if ($testPartition -match '^collection:(.+)$') {
+ # Collection/partition entry
+ $collectionName = $Matches[1]
+ $entry = New-CollectionTestEntry -CollectionName $collectionName -Metadata $metadata -IsUncollected $false
+ $matrixEntries.Add($entry)
+ $partitionCount++
+ }
+ elseif ($testPartition -match '^uncollected:\*$') {
+ # Uncollected tests entry
+ $entry = New-CollectionTestEntry -CollectionName '*' -Metadata $metadata -IsUncollected $true
+ $matrixEntries.Add($entry)
+ $partitionCount++
+ }
+ elseif ($testPartition -match '^class:(.+)$') {
+ # Class-based entry
+ $className = $Matches[1]
+ $entry = New-ClassTestEntry -ClassName $className -Metadata $metadata
+ $matrixEntries.Add($entry)
+ $classCount++
+ }
+ }
+
+ Write-Host " ✓ Added $partitionCount partition(s) and $classCount class(es)"
+ }
+ else {
+ # Regular (non-split) test
+ $entry = New-RegularTestEntry -Metadata $metadataObject
+ $matrixEntries.Add($entry)
+ }
+}
+
+# 3. Sort entries and output
+Write-Host ""
+Write-Host "Generated $($matrixEntries.Count) total matrix entries"
+
+$sortedEntries = @($matrixEntries | Sort-Object -Property projectName, name)
+
+$requiresNugetsCount = @($sortedEntries | Where-Object { $_.requiresNugets -eq $true }).Count
+$noNugetsCount = @($sortedEntries | Where-Object { $_.requiresNugets -ne $true }).Count
+
+Write-Host " - Requiring NuGets: $requiresNugetsCount"
+Write-Host " - Not requiring NuGets: $noNugetsCount"
+
+# 4. Write canonical matrix
+$canonicalMatrix = [ordered]@{
+ tests = $sortedEntries
+}
+
+$outputDir = [System.IO.Path]::GetDirectoryName($OutputMatrixFile)
+if ($outputDir -and -not (Test-Path $outputDir)) {
+ New-Item -ItemType Directory -Path $outputDir -Force | Out-Null
+}
+
+$canonicalMatrix | ConvertTo-Json -Depth 10 | Set-Content -Path $OutputMatrixFile -Encoding UTF8
+
+Write-Host ""
+Write-Host "✓ Canonical matrix written to: $OutputMatrixFile"
+Write-Host ""
+Write-Host "Matrix build complete!"
diff --git a/eng/scripts/expand-test-matrix-github.ps1 b/eng/scripts/expand-test-matrix-github.ps1
new file mode 100644
index 00000000000..936351d613b
--- /dev/null
+++ b/eng/scripts/expand-test-matrix-github.ps1
@@ -0,0 +1,140 @@
+<#
+.SYNOPSIS
+ Expands the canonical test matrix for GitHub Actions.
+
+.DESCRIPTION
+ This script takes the canonical test matrix (output by build-test-matrix.ps1)
+ and transforms it for GitHub Actions consumption by:
+ 1. Expanding each entry for every OS in its supportedOSes array
+ 2. Mapping OS names to GitHub runner names (linux -> ubuntu-latest, etc.)
+ 3. Outputting a single all_tests matrix with all entries
+
+ This is the platform-specific layer for GitHub Actions. Azure DevOps would
+ have a similar script with different runner mappings and output format.
+
+ Downstream consumers (e.g., tests.yml) are responsible for splitting the
+ matrix by dependency type and handling overflow.
+
+.PARAMETER CanonicalMatrixFile
+ Path to the canonical test matrix JSON file (output of build-test-matrix.ps1).
+
+.PARAMETER OutputMatrixFile
+ Output file path for the matrix JSON. When set, writes the all_tests matrix.
+
+.PARAMETER OutputToGitHubEnv
+ If set, outputs to GITHUB_OUTPUT environment file instead of files.
+
+.NOTES
+ PowerShell 7+
+#>
+
+[CmdletBinding()]
+param(
+ [Parameter(Mandatory=$true)]
+ [string]$CanonicalMatrixFile,
+
+ [Parameter(Mandatory=$false)]
+ [string]$OutputMatrixFile = "",
+
+ [Parameter(Mandatory=$false)]
+ [switch]$OutputToGitHubEnv
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version Latest
+
+# GitHub runner mappings
+$runnerMap = @{
+ 'windows' = 'windows-latest'
+ 'linux' = 'ubuntu-latest'
+ 'macos' = 'macos-latest'
+}
+
+# Valid OS values
+$validOSes = @('windows', 'linux', 'macos')
+
+function Expand-MatrixEntriesByOS {
+ param(
+ [Parameter(Mandatory=$true)]
+ [array]$Entries
+ )
+
+ $expandedEntries = @()
+
+ foreach ($entry in $Entries) {
+ # Get supported OSes (default to all if not specified)
+ $hasSupportedOSes = [bool]($entry.PSObject.Properties.Name -contains 'supportedOSes')
+ $supportedOSes = if ($hasSupportedOSes -and $entry.supportedOSes -and $entry.supportedOSes.Count -gt 0) {
+ $entry.supportedOSes
+ } else {
+ $validOSes
+ }
+
+ # Validate and expand for each OS
+ foreach ($os in $supportedOSes) {
+ $osLower = $os.ToLowerInvariant()
+
+ if ($osLower -notin $validOSes) {
+ Write-Warning "Invalid OS '$os' in supportedOSes for test '$($entry.name)'. Skipping."
+ continue
+ }
+
+ # Create a copy of the entry for this OS
+ $expandedEntry = [ordered]@{}
+ foreach ($prop in $entry.PSObject.Properties) {
+ if ($prop.Name -ne 'supportedOSes') {
+ $expandedEntry[$prop.Name] = $prop.Value
+ }
+ }
+
+ # Add GitHub-specific runner
+ $expandedEntry['runs-on'] = $runnerMap[$osLower]
+
+ $expandedEntries += [PSCustomObject]$expandedEntry
+ }
+ }
+
+ return $expandedEntries
+}
+
+# Read canonical matrix
+if (-not (Test-Path $CanonicalMatrixFile)) {
+ Write-Error "Canonical matrix file not found: $CanonicalMatrixFile"
+ exit 1
+}
+
+Write-Host "Reading canonical matrix from: $CanonicalMatrixFile"
+$canonicalMatrix = Get-Content -Raw $CanonicalMatrixFile | ConvertFrom-Json
+
+# Expand matrix entries by OS
+$allEntries = @()
+
+if ($canonicalMatrix.PSObject.Properties.Name -contains 'tests' -and $canonicalMatrix.tests) {
+ $allEntries = Expand-MatrixEntriesByOS -Entries $canonicalMatrix.tests
+}
+
+Write-Host "Expanded matrix: $($allEntries.Count) total entries"
+
+$allTestsMatrix = ConvertTo-Json @{ include = @($allEntries) } -Compress -Depth 10
+
+# Output results
+if ($OutputToGitHubEnv) {
+ # Output to GitHub Actions environment
+ if ($env:GITHUB_OUTPUT) {
+ "all_tests=$allTestsMatrix" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
+ Write-Host "✓ Matrix written to GITHUB_OUTPUT ($($allEntries.Count) entries)"
+ } else {
+ Write-Error "GITHUB_OUTPUT environment variable not set"
+ exit 1
+ }
+} elseif ($OutputMatrixFile) {
+ $allTestsMatrix | Set-Content -Path $OutputMatrixFile -Encoding UTF8
+ Write-Host "✓ Matrix written to: $OutputMatrixFile"
+} else {
+ # Output to console for debugging
+ Write-Host ""
+ Write-Host "All tests: $allTestsMatrix"
+}
+
+Write-Host ""
+Write-Host "GitHub Actions matrix expansion complete!"
diff --git a/eng/scripts/split-test-matrix-by-deps.ps1 b/eng/scripts/split-test-matrix-by-deps.ps1
new file mode 100644
index 00000000000..6d04cc57729
--- /dev/null
+++ b/eng/scripts/split-test-matrix-by-deps.ps1
@@ -0,0 +1,142 @@
+<#
+.SYNOPSIS
+ Splits an all_tests matrix by dependency type for GitHub Actions.
+
+.DESCRIPTION
+ Takes a flat all_tests matrix JSON (already OS-expanded) and splits it into
+ four dependency-based matrices for GitHub Actions consumption:
+ 1. tests_matrix_no_nugets (primary) — tests with no package dependencies
+ 2. tests_matrix_no_nugets_overflow — overflow when primary exceeds threshold
+ 3. tests_matrix_requires_nugets — tests needing built NuGet packages
+ 4. tests_matrix_requires_cli_archive — tests needing CLI native archives
+
+ The overflow mechanism keeps each matrix under GitHub Actions' 256-job limit.
+
+.PARAMETER AllTestsMatrix
+ JSON string of the all_tests matrix ({"include": [...]}).
+
+.PARAMETER AllTestsMatrixFile
+ Path to a JSON file containing the all_tests matrix.
+ Mutually exclusive with AllTestsMatrix.
+
+.PARAMETER OutputToGitHubEnv
+ If set, outputs to GITHUB_OUTPUT environment file.
+
+.PARAMETER OverflowThreshold
+ Maximum entries in the no_nugets primary bucket before overflow kicks in.
+ Defaults to 250 (GitHub Actions hard limit is 256).
+
+.NOTES
+ PowerShell 7+
+#>
+
+[CmdletBinding()]
+param(
+ [Parameter(Mandatory=$false)]
+ [string]$AllTestsMatrix = "",
+
+ [Parameter(Mandatory=$false)]
+ [string]$AllTestsMatrixFile = "",
+
+ [Parameter(Mandatory=$false)]
+ [switch]$OutputToGitHubEnv,
+
+ [Parameter(Mandatory=$false)]
+ [int]$OverflowThreshold = 250
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version Latest
+
+$maxMatrixSize = 256
+
+# Read input
+if ($AllTestsMatrixFile) {
+ if (-not (Test-Path $AllTestsMatrixFile)) {
+ Write-Error "Matrix file not found: $AllTestsMatrixFile"
+ exit 1
+ }
+ $AllTestsMatrix = Get-Content -Raw $AllTestsMatrixFile
+}
+
+if (-not $AllTestsMatrix) {
+ Write-Error "Either -AllTestsMatrix or -AllTestsMatrixFile must be provided."
+ exit 1
+}
+
+$matrix = $AllTestsMatrix | ConvertFrom-Json
+$allEntries = @()
+if ($matrix.include -and $matrix.include.Count -gt 0) {
+ $allEntries = @($matrix.include)
+}
+
+Write-Host "Input matrix: $($allEntries.Count) total entries"
+
+# Split into categories based on dependency requirements
+$cliArchiveEntries = @($allEntries | Where-Object { $_.PSObject.Properties.Name -contains 'requiresCliArchive' -and $_.requiresCliArchive -eq $true })
+$nugetEntries = @($allEntries | Where-Object {
+ ($_.PSObject.Properties.Name -contains 'requiresNugets' -and $_.requiresNugets -eq $true) -and
+ -not ($_.PSObject.Properties.Name -contains 'requiresCliArchive' -and $_.requiresCliArchive -eq $true)
+})
+$noNugetEntries = @($allEntries | Where-Object {
+ -not ($_.PSObject.Properties.Name -contains 'requiresNugets' -and $_.requiresNugets -eq $true) -and
+ -not ($_.PSObject.Properties.Name -contains 'requiresCliArchive' -and $_.requiresCliArchive -eq $true)
+})
+
+Write-Host " - No nugets: $($noNugetEntries.Count)"
+Write-Host " - Requires nugets: $($nugetEntries.Count)"
+Write-Host " - Requires CLI archive: $($cliArchiveEntries.Count)"
+
+# Split no_nugets into primary + overflow
+$noNugetPrimary = @()
+$noNugetOverflow = @()
+
+if ($noNugetEntries.Count -le $OverflowThreshold) {
+ $noNugetPrimary = $noNugetEntries
+} else {
+ $noNugetPrimary = @($noNugetEntries[0..($OverflowThreshold - 1)])
+ $noNugetOverflow = @($noNugetEntries[$OverflowThreshold..($noNugetEntries.Count - 1)])
+ Write-Host " ↳ no_nugets overflow: $($noNugetPrimary.Count) primary + $($noNugetOverflow.Count) overflow"
+}
+
+# Validate no bucket exceeds the hard limit
+$buckets = @{
+ 'tests_matrix_no_nugets' = $noNugetPrimary
+ 'tests_matrix_no_nugets_overflow' = $noNugetOverflow
+ 'tests_matrix_requires_nugets' = $nugetEntries
+ 'tests_matrix_requires_cli_archive' = $cliArchiveEntries
+}
+
+foreach ($name in $buckets.Keys) {
+ if ($buckets[$name].Count -gt $maxMatrixSize) {
+ Write-Error "$name has $($buckets[$name].Count) entries, exceeding the GitHub Actions limit of $maxMatrixSize."
+ exit 1
+ }
+}
+
+# Convert to JSON
+$results = @{}
+foreach ($name in $buckets.Keys) {
+ $results[$name] = ConvertTo-Json @{ include = @($buckets[$name]) } -Compress -Depth 10
+}
+
+# Output
+if ($OutputToGitHubEnv) {
+ if ($env:GITHUB_OUTPUT) {
+ foreach ($name in $results.Keys) {
+ "$name=$($results[$name])" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
+ }
+ Write-Host "✓ Split matrices written to GITHUB_OUTPUT"
+ } else {
+ Write-Error "GITHUB_OUTPUT environment variable not set"
+ exit 1
+ }
+} else {
+ # Output to console for debugging
+ foreach ($name in $results.Keys) {
+ Write-Host "${name}: $($results[$name])"
+ }
+}
+
+Write-Host ""
+Write-Host "Matrix split complete!"
diff --git a/eng/scripts/split-test-projects-for-ci.ps1 b/eng/scripts/split-test-projects-for-ci.ps1
new file mode 100644
index 00000000000..b7a87364c1f
--- /dev/null
+++ b/eng/scripts/split-test-projects-for-ci.ps1
@@ -0,0 +1,186 @@
+<#
+.SYNOPSIS
+ Discovers test partitions or classes for CI test splitting.
+
+.DESCRIPTION
+ Determines how to split a test project for parallel CI execution:
+
+ 1. Extracts partitions using ExtractTestPartitions tool:
+ - Scans assembly for [Trait("Partition", "name")] attributes
+ - If partitions found → partition mode
+ - Fails if the tool cannot be built or run
+
+ 2. Uses class-based splitting if no partitions are found:
+ - Runs --list-tests to enumerate test classes
+ - Creates one entry per test class
+
+ Outputs a .tests-partitions.json file with entries like:
+ Partition mode: ["collection:Name", ..., "uncollected:*"]
+ Class mode: ["class:Full.Namespace.ClassName", ...]
+
+ The uncollected:* entry ensures tests without partition traits still run.
+
+.PARAMETER TestAssemblyPath
+ Path to the test assembly DLL for extracting partition attributes.
+
+.PARAMETER RunCommand
+ The command to run the test assembly (e.g., "dotnet exec ").
+ Only invoked if partition extraction fails and class-based splitting is needed.
+
+.PARAMETER TestClassNamePrefixForCI
+ Namespace prefix used to recognize test classes (e.g., Aspire.Hosting.Tests).
+
+.PARAMETER TestPartitionsJsonFile
+ Path to write the .tests-partitions.json output file.
+
+.PARAMETER RepoRoot
+ Path to the repository root (for locating the ExtractTestPartitions tool).
+
+.NOTES
+ PowerShell 7+
+ Fails fast if ExtractTestPartitions cannot be built or run.
+ Fails fast if zero test classes discovered when in class mode.
+ Only runs --list-tests when no partitions are found in the assembly.
+#>
+
+[CmdletBinding()]
+param(
+ [Parameter(Mandatory=$true)]
+ [string]$TestAssemblyPath,
+
+ [Parameter(Mandatory=$true)]
+ [string]$RunCommand,
+
+ [Parameter(Mandatory=$true)]
+ [string]$TestClassNamePrefixForCI,
+
+ [Parameter(Mandatory=$true)]
+ [string]$TestPartitionsJsonFile,
+
+ [Parameter(Mandatory=$true)]
+ [string]$RepoRoot
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version Latest
+
+if (-not (Test-Path $TestAssemblyPath)) {
+ Write-Error "TestAssemblyPath not found: $TestAssemblyPath"
+}
+
+$collections = [System.Collections.Generic.HashSet[string]]::new()
+$classes = [System.Collections.Generic.HashSet[string]]::new()
+
+# Extract partitions using the ExtractTestPartitions tool
+$partitionsFile = Join-Path ([System.IO.Path]::GetTempPath()) "partitions-$([System.Guid]::NewGuid()).txt"
+try {
+ $toolPath = Join-Path $RepoRoot "artifacts/bin/ExtractTestPartitions/Release/net8.0/ExtractTestPartitions.dll"
+
+ # Build the tool if it doesn't exist
+ if (-not (Test-Path $toolPath)) {
+ Write-Host "Building ExtractTestPartitions tool..."
+ $toolProjectPath = Join-Path $RepoRoot "tools/ExtractTestPartitions/ExtractTestPartitions.csproj"
+ & dotnet build $toolProjectPath -c Release --nologo -v quiet
+ if ($LASTEXITCODE -ne 0) {
+ Write-Error "Failed to build ExtractTestPartitions tool."
+ }
+ }
+
+ if (-not (Test-Path $toolPath)) {
+ Write-Error "ExtractTestPartitions tool not found at $toolPath after build."
+ }
+
+ Write-Host "Extracting partitions from assembly: $TestAssemblyPath"
+ $toolOutput = & dotnet $toolPath --assembly-path $TestAssemblyPath --output-file $partitionsFile 2>&1
+ $toolExitCode = $LASTEXITCODE
+
+ # Display tool output (informational)
+ if ($toolOutput) {
+ $toolOutput | Write-Host
+ }
+
+ if ($toolExitCode -ne 0) {
+ Write-Error "ExtractTestPartitions failed with exit code $toolExitCode."
+ }
+
+ # Read partitions if the file was created
+ if (Test-Path $partitionsFile) {
+ $partitionLines = Get-Content $partitionsFile -ErrorAction SilentlyContinue
+ if ($partitionLines) {
+ foreach ($partition in $partitionLines) {
+ if (-not [string]::IsNullOrWhiteSpace($partition)) {
+ $collections.Add($partition.Trim()) | Out-Null
+ }
+ }
+ Write-Host "Found $($collections.Count) partition(s) via attribute extraction"
+ }
+ }
+}
+finally {
+ # Clean up temp file
+ if (Test-Path $partitionsFile) {
+ Remove-Item $partitionsFile -ErrorAction SilentlyContinue
+ }
+}
+
+# Determine mode: if we have partitions, use collection mode; otherwise fall back to class mode
+$mode = if ($collections.Count -gt 0) { 'collection' } else { 'class' }
+
+# Only run --list-tests if we need class-based splitting (no partitions found)
+if ($mode -eq 'class') {
+ Write-Host "No partitions found. Running --list-tests to extract class names..."
+
+ # Run the test assembly with --list-tests to get all test names
+ $testOutput = & $RunCommand --filter-not-trait category=failing --list-tests 2>&1
+
+ if ($LASTEXITCODE -ne 0) {
+ Write-Error "Test listing command failed with exit code $LASTEXITCODE"
+ }
+
+ # Extract class names from test listing
+ # Match everything up to the last segment (method name), capturing the full class name
+ $classNamePattern = '^\s*(' + [Regex]::Escape($TestClassNamePrefixForCI) + '\..+)\.[^\.]+$'
+
+ foreach ($line in $testOutput) {
+ $lineStr = $line.ToString().Trim()
+ # Extract class name from test name
+ # Format: "Namespace.SubNs.ClassName.MethodName(...)" or "Namespace.ClassName.MethodName"
+ # Strip any trailing parenthesized arguments: "Method(arg1, arg2)" → "Method"
+ $cleanLine = $lineStr -replace '\(.*\)$', ''
+ if ($cleanLine -match $classNamePattern) {
+ $className = $Matches[1]
+ $classes.Add($className) | Out-Null
+ }
+ }
+
+ if ($classes.Count -eq 0) {
+ Write-Error "No test classes discovered matching prefix '$TestClassNamePrefixForCI'."
+ }
+}
+
+$lines = [System.Collections.Generic.List[string]]::new()
+
+if ($mode -eq 'collection') {
+ foreach ($c in ($collections | Sort-Object)) {
+ $lines.Add("collection:$c")
+ }
+ $lines.Add("uncollected:*")
+} else {
+ foreach ($cls in ($classes | Sort-Object)) {
+ $lines.Add("class:$cls")
+ }
+}
+
+# Create tests partitions json file
+try {
+ $testPartitionsJson = @{}
+ $testPartitionsJson | Add-Member -Force -MemberType NoteProperty -Name 'testPartitions' -Value @($lines)
+ $testPartitionsJson | ConvertTo-Json -Depth 20 | Set-Content -Path $TestPartitionsJsonFile -Encoding UTF8
+} catch {
+ Write-Warning "Failed updating metadata JSON: $_"
+}
+
+Write-Host "Mode: $mode"
+Write-Host "Collections discovered: $($collections.Count)"
+Write-Host "Classes discovered: $($classes.Count)"
+Write-Host "Test partitions JSON: $TestPartitionsJsonFile"
diff --git a/eng/testing/github-ci-trigger-patterns.txt b/eng/testing/github-ci-trigger-patterns.txt
new file mode 100644
index 00000000000..0d0ae248697
--- /dev/null
+++ b/eng/testing/github-ci-trigger-patterns.txt
@@ -0,0 +1,49 @@
+# CI trigger patterns
+#
+# This file lists glob patterns for files whose changes do NOT require the full CI
+# to run (e.g. documentation, non-build pipeline scripts, or specific workflow files
+# that are unrelated to the build and test process).
+#
+# When all files changed in a pull request match at least one pattern here, the CI
+# workflow is skipped.
+#
+# Pattern syntax:
+# ** matches any path including directory separators (recursive)
+# * matches any characters except a directory separator
+# . is treated as a literal dot (no escaping needed)
+# All other characters are treated as literals.
+#
+# Lines starting with '#' and blank lines are ignored.
+
+# This file itself - changing CI-skip patterns doesn't require a CI run.
+# Note: this also means a syntax error introduced here won't be caught by CI,
+# so take care when editing. Pattern conversion is validated by the
+# check-changed-files action at runtime.
+eng/testing/github-ci-trigger-patterns.txt
+
+# Documentation
+**.md
+
+# Engineering pipeline scripts (Azure DevOps, not used in the GitHub CI build)
+eng/pipelines/**
+eng/test-configuration.json
+
+.github/instructions/**
+.github/skills/**
+
+# GitHub workflow files that do not affect the CI build or test process
+.github/workflows/apply-test-attributes.yml
+.github/workflows/backmerge-release.yml
+.github/workflows/backport.yml
+.github/workflows/dogfood-comment.yml
+.github/workflows/generate-api-diffs.yml
+.github/workflows/generate-ats-diffs.yml
+.github/workflows/labeler-*.yml
+.github/workflows/markdownlint*.yml
+.github/workflows/pr-review-needed.yml
+.github/workflows/refresh-manifests.yml
+.github/workflows/reproduce-flaky-tests.yml
+.github/workflows/specialized-test-runner.yml
+.github/workflows/tests-outerloop.yml
+.github/workflows/tests-quarantine.yml
+.github/workflows/update-*.yml
diff --git a/extension/README.md b/extension/README.md
index 0fe685b2275..7d8442ae1e8 100644
--- a/extension/README.md
+++ b/extension/README.md
@@ -7,16 +7,19 @@ The Aspire VS Code extension provides a set of commands and tools to help you wo
The extension adds the following commands to VS Code:
-| Command | Description | Availability |
-|---------|-------------|--------------|
-| Aspire: New Aspire project | Create a new Aspire apphost or starter app from a template. | Available |
-| Aspire: Add an integration | Add a hosting integration (`Aspire.Hosting.*`) to the Aspire apphost. | Available |
-| Aspire: Configure launch.json | Adds the default Aspire debugger launch configuration to your workspace's `launch.json`, which will detect and run the apphost in the workspace. | Available |
-| Aspire: Manage configuration settings | Manage configuration settings including feature flags. | Available |
-| Aspire: Open Aspire terminal | Open an Aspire VS Code terminal for working with Aspire projects. | Available |
-| Aspire: Publish deployment artifacts | Generates deployment artifacts for an Aspire apphost. | Preview |
-| Aspire: Deploy app | Deploy the contents of an Aspire apphost to its defined deployment targets. | Preview |
-| Aspire: Update integrations | Update hosting integrations and Aspire SDK in the apphost. | Preview |
+| Command | Description |
+|---------|-------------|
+| Aspire: New Aspire project | Create a new Aspire apphost or starter app from a template. |
+| Aspire: Initialize Aspire | Initialize Aspire in an existing project. |
+| Aspire: Add an integration | Add a hosting integration (`Aspire.Hosting.*`) to the Aspire apphost. |
+| Aspire: Update integrations | Update hosting integrations and Aspire SDK in the apphost. |
+| Aspire: Publish deployment artifacts | Generate deployment artifacts for an Aspire apphost. |
+| Aspire: Deploy app | Deploy the contents of an Aspire apphost to its defined deployment targets. |
+| Aspire: Configure launch.json file | Add the default Aspire debugger launch configuration to your workspace's `launch.json`. |
+| Aspire: Extension settings | Open Aspire extension settings. |
+| Aspire: Open local Aspire settings | Open the local `.aspire/settings.json` file for the current workspace. |
+| Aspire: Open global Aspire settings | Open the global `~/.aspire/globalsettings.json` file. |
+| Aspire: Open Aspire terminal | Open an Aspire VS Code terminal for working with Aspire projects. |
All commands are available from the Command Palette (`Cmd+Shift+P` or `Ctrl+Shift+P`) and are grouped under the "Aspire" category.
diff --git a/extension/loc/xlf/aspire-vscode.xlf b/extension/loc/xlf/aspire-vscode.xlf
index bbe98c2cc15..ea53f379a7f 100644
--- a/extension/loc/xlf/aspire-vscode.xlf
+++ b/extension/loc/xlf/aspire-vscode.xlf
@@ -7,6 +7,9 @@
Add an integration
+
+ Aspire
+
Aspire CLI Version: {0}.
@@ -130,6 +133,9 @@
Error: {0}
+
+ Execute resource command
+
Extension context is not initialized.
@@ -214,6 +220,9 @@
No output from msbuild.
+
+ No running Aspire apphosts found.
[Run an apphost](command:aspire-vscode.runAppHost)
+
No watch task found. Please ensure a watch task is defined in your workspace.
@@ -229,6 +238,9 @@
Open Aspire terminal
+
+ Open Dashboard
+
Open codespaces URL
@@ -256,21 +268,42 @@
RPC server is not initialized.
+
+ Refresh running apphosts
+
Required capability: {0}.
+
+ Restart
+
Run Aspire apphost
Run {0}
+
+ Running apphosts
+
See CLI installation instructions
Select the default apphost to launch when starting an Aspire debug session
+
+ Start
+
+
+ Stop
+
+
+ Stop
+
+
+ The Aspire CLI is not installed or does not support this feature. Install or update the Aspire CLI to get started.
[Update Aspire CLI](command:aspire-vscode.updateSelf)
[Refresh](command:aspire-vscode.refreshRunningAppHosts)
+
The apphost is not compatible. Consider upgrading the apphost or Aspire CLI.
@@ -280,9 +313,15 @@
This field is required.
+
+ Update Aspire CLI
+
Update integrations
+
+ View logs
+
Watch {0} ({1})
@@ -292,5 +331,11 @@
Yes
+
+ Select directory
+
+
+ Select file
+