diff --git a/.github/instructions/helix-device-tests.instructions.md b/.github/instructions/helix-device-tests.instructions.md index 354984c36f46..6691c81e4733 100644 --- a/.github/instructions/helix-device-tests.instructions.md +++ b/.github/instructions/helix-device-tests.instructions.md @@ -71,6 +71,26 @@ dotnet tool restore ./build.sh -restore -build -configuration Release /p:BuildDeviceTests=true /bl:BuildDeviceTests.binlog -warnAsError false ``` +### Using the run-device-tests Skill (Recommended) + +The easiest way to run device tests locally is using the `run-device-tests` skill: + +```bash +# Run Controls tests on iOS simulator +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Controls -Platform ios + +# Run only Button category tests +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Controls -Platform ios -TestFilter "Category=Button" + +# Run on Android emulator +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Controls -Platform android -TestFilter "Category=Button" + +# Run on MacCatalyst +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Core -Platform maccatalyst +``` + +See `.github/skills/run-device-tests/SKILL.md` for full documentation. + ### Submit to Helix Set required environment variables: @@ -118,6 +138,37 @@ To validate the helix proj without submitting (requires built artifacts): dotnet msbuild eng/helix_xharness.proj /t:DiscoverTestBundles /p:TargetOS=ios /p:_MauiDotNetTfm=net10.0 /p:RepoRoot=$(pwd)/ -v:n ``` +## Test Filtering Implementation + +Test category filtering is implemented in `src/Core/tests/DeviceTests.Shared/DeviceTestSharedHelpers.cs`. The `GetExcludedTestCategories()` method reads the `TestFilter` value and converts it to a list of categories to skip. + +### Filter Syntax + +| Format | Description | Example | +|--------|-------------|---------| +| `Category=X` | Run only category X (skip all others) | `Category=Button` | +| `SkipCategories=X,Y,Z` | Skip specific categories | `SkipCategories=Shell,CollectionView` | + +### Platform-Specific Filter Passing + +| Platform | XHarness Argument | How App Reads It | +|----------|-------------------|------------------| +| **iOS/MacCatalyst** | `--set-env=TestFilter=...` | `NSProcessInfo.ProcessInfo.Environment["TestFilter"]` | +| **Android** | `--arg TestFilter=...` | `MauiTestInstrumentation.Current.Arguments.GetString("TestFilter")` | +| **Windows** | `--filter "Category=..."` | Native vstest filter | + +**Important**: iOS uses `--set-env` (environment variable), while Android uses `--arg` (instrumentation argument). These are NOT interchangeable. + +### Example XHarness Commands with Filters + +```bash +# iOS - uses --set-env +xharness apple test --target ios-simulator-64_18.5 --device UDID --set-env=TestFilter=Category=Button ... + +# Android - uses --arg +xharness android test --package-name com.microsoft.maui.controls.devicetests --arg TestFilter=Category=Button ... +``` + ## Configuration Details The `eng/helix_xharness.proj` configuration includes: diff --git a/.github/skills/run-device-tests/SKILL.md b/.github/skills/run-device-tests/SKILL.md new file mode 100644 index 000000000000..bd8d7b842ffa --- /dev/null +++ b/.github/skills/run-device-tests/SKILL.md @@ -0,0 +1,299 @@ +--- +name: run-device-tests +description: "Build and run .NET MAUI device tests locally with category filtering. Supports iOS, MacCatalyst, Android on macOS; Android, Windows on Windows. Use TestFilter to run specific test categories." +metadata: + author: dotnet-maui + version: "2.1" +compatibility: Requires xharness CLI (for iOS/MacCatalyst/Android), Xcode (for Apple platforms), Android SDK (for Android), and .NET SDK with platform workloads. +--- + +# Run Device Tests Skill + +Build and run .NET MAUI device tests locally on iOS simulators, MacCatalyst, Android emulators, or Windows. + +## Platform Support + +| Host OS | Supported Platforms | +|---------|---------------------| +| macOS | ios, maccatalyst, android | +| Windows | android, windows | + +## Tools Required + +This skill uses `bash` together with `pwsh` (PowerShell 7+) to run the PowerShell scripts. Requires: +- `xharness` - Global dotnet tool for running tests on iOS/MacCatalyst/Android +- `dotnet` - .NET SDK with platform workloads installed +- **iOS/MacCatalyst**: Xcode with iOS simulators +- **Android**: Android SDK with emulator +- **Windows**: Windows SDK + +## Dependencies + +This skill uses shared infrastructure scripts: +- `.github/scripts/shared/Start-Emulator.ps1` - Detects and boots iOS simulators / Android emulators +- `.github/scripts/shared/shared-utils.ps1` - Common utility functions + +These are automatically loaded by the Run-DeviceTests.ps1 script. + +## When to Use + +- User wants to run device tests locally +- User wants to verify iOS/MacCatalyst/Android/Windows compatibility +- User wants to test on a specific iOS version (e.g., iOS 26) +- User asks "run device tests for Controls/Core/Essentials/Graphics" +- User asks "test on iOS simulator" or "test on Android emulator" +- User asks "run device tests on MacCatalyst" +- User wants to run only specific test categories (e.g., "run Button tests") + +## Available Test Projects + +| Project | Path | +|---------|------| +| Controls | `src/Controls/tests/DeviceTests/Controls.DeviceTests.csproj` | +| Core | `src/Core/tests/DeviceTests/Core.DeviceTests.csproj` | +| Essentials | `src/Essentials/test/DeviceTests/Essentials.DeviceTests.csproj` | +| Graphics | `src/Graphics/tests/DeviceTests/Graphics.DeviceTests.csproj` | +| BlazorWebView | `src/BlazorWebView/tests/DeviceTests/MauiBlazorWebView.DeviceTests.csproj` | + +## Scripts + +All scripts are in `.github/skills/run-device-tests/scripts/` + +### Run Device Tests (Full Workflow) + +```bash +# Run Controls device tests on iOS simulator (default on macOS) +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Controls -Platform ios + +# Run Core device tests on MacCatalyst +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Core -Platform maccatalyst + +# Run Controls device tests on Android emulator +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Controls -Platform android + +# Run Controls device tests on Windows (default on Windows) +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Controls -Platform windows + +# Run on specific iOS version +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Core -Platform ios -iOSVersion 26 + +# Run with test filter +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Controls -Platform ios -TestFilter "Category=Button" + +# Run other test projects +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Essentials -Platform android +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Graphics -Platform maccatalyst +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project BlazorWebView -Platform ios +``` + +### Build Only (No Test Run) + +```bash +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Controls -Platform ios -BuildOnly +``` + +### List Available Simulators/Emulators + +```bash +# iOS simulators +xcrun simctl list devices available + +# Android emulators +emulator -list-avds +``` + +## Workflow + +1. **Run tests**: `scripts/Run-DeviceTests.ps1 -Project -Platform ` automatically detects/boots device, builds, and runs tests +2. **Check results**: Look at the console output or `artifacts/log/` directory for detailed test results + +## Output + +- Build artifacts: `artifacts/bin/.DeviceTests////` +- Test logs: `artifacts/log/` +- Test results summary is printed to console + +## Prerequisites + +- `xharness` global tool: `dotnet tool install --global Microsoft.DotNet.XHarness.CLI` +- .NET SDK with platform workloads +- **iOS/MacCatalyst**: Xcode with simulators installed +- **Android**: Android SDK with emulator configured +- **Windows**: Windows SDK +- For iOS 26: macOS Tahoe (26) with Xcode 26 + +## Examples + +```bash +# Quick test run for Controls on iOS (default on macOS) +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Controls -Platform ios + +# Test on MacCatalyst +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Controls -Platform maccatalyst + +# Test on Android emulator (works on both macOS and Windows) +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Controls -Platform android + +# Test on Windows (default on Windows) +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Controls -Platform windows + +# Test on iOS 26 specifically +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Controls -Platform ios -iOSVersion 26 + +# Run only Button category tests on iOS +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Controls -Platform ios -TestFilter "Category=Button" + +# Build for Android without running tests +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Core -Platform android -BuildOnly +``` + +## Notes + +- The script automatically detects and boots an iOS simulator / Android emulator using the shared Start-Emulator.ps1 infrastructure +- Default iOS simulator is iPhone Xs with iOS 18.5 (same as UI tests) +- Default Android emulator priority: API 30 Nexus > API 30 > Nexus > First available +- MacCatalyst runs directly on the Mac (no simulator needed) +- Windows tests run directly on the local machine +- Simulator/emulator selection and boot logic is handled by `.github/scripts/shared/Start-Emulator.ps1` +- xharness manages test execution and reporting for iOS/MacCatalyst/Android +- Windows uses vstest for test execution + +## Test Filtering + +The `-TestFilter` parameter allows running specific test categories instead of all tests. This is useful for quick iteration during development. + +### Filter Syntax + +| Format | Description | Example | +|--------|-------------|---------| +| `Category=X` | Run only category X | `Category=Button` | +| `SkipCategories=X,Y,Z` | Skip categories X, Y, Z | `SkipCategories=Shell,CollectionView` | + +### Examples + +```bash +# Run only Button tests on iOS +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Controls -Platform ios -TestFilter "Category=Button" + +# Run only Button tests on Android +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Controls -Platform android -TestFilter "Category=Button" + +# Skip heavy test categories +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Controls -Platform ios -TestFilter "SkipCategories=Shell,CollectionView,HybridWebView" +``` + +### How Test Filtering Works + +Test filtering is implemented in `src/Core/tests/DeviceTests.Shared/DeviceTestSharedHelpers.cs`: + +| Platform | How Filter is Passed | How Filter is Read | +|----------|---------------------|-------------------| +| **iOS/MacCatalyst** | `--set-env=TestFilter=...` | `NSProcessInfo.ProcessInfo.Environment["TestFilter"]` | +| **Android** | `--arg TestFilter=...` | `MauiTestInstrumentation.Current.Arguments.GetString("TestFilter")` | +| **Windows** | `--filter "Category=..."` | Native vstest filter | + +### Available Test Categories + +Common categories in Controls.DeviceTests: +- `Button`, `Label`, `Entry`, `Editor` - Individual control tests +- `CollectionView`, `ListView`, `CarouselView` - Collection controls (heavy) +- `Shell`, `Navigation`, `TabbedPage` - Navigation tests (heavy) +- `Layout`, `FlexLayout` - Layout tests +- `Memory` - Memory leak tests +- `Accessibility` - Accessibility tests +- `Gesture` - Gesture recognition tests + +To see all categories, check `src/Controls/tests/DeviceTests/TestCategory.cs`. + +## Troubleshooting + +### Xcode Version Mismatch + +If you see errors about Xcode version mismatch (e.g., "Xcode 26.2 installed but 26.0 required"): + +```bash +# Use -SkipXcodeVersionCheck to bypass validation +pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Controls -Platform ios -SkipXcodeVersionCheck +``` + +### MacCatalyst App Bundle Not Found + +MacCatalyst apps use display names (e.g., "Controls Tests.app") not assembly names. The script handles this automatically by searching for `.app` bundles in the output directory. + +### Android Package Name Issues + +Android package names must be lowercase (e.g., `com.microsoft.maui.controls.devicetests`). The script has a mapping of project names to correct package names. + +### Test Filter Not Working + +1. **Ensure full rebuild**: The test filter code must be compiled into the app. Delete artifacts and rebuild: + ```bash + rm -rf artifacts/bin/.DeviceTests + pwsh .github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 -Project Controls -Platform android -TestFilter "Category=Button" + ``` + +2. **Check the console output**: Look for `TestFilter: Category=Button` in the test output to confirm filter was applied. + +3. **Verify category exists**: Ensure the category name matches exactly (case-sensitive). + +## XHarness Device Detection + +The script automatically handles XHarness device targeting for iOS and Android: + +### iOS +1. **Simulator Boot**: Start-Emulator.ps1 detects and boots appropriate iOS simulator +2. **UDID Extraction**: Script extracts simulator UDID from Start-Emulator.ps1 output +3. **iOS Version Detection**: Script queries `xcrun simctl` to get iOS version from booted simulator +4. **XHarness Targeting**: Passes both `--target ios-simulator-64_VERSION` and `--device UDID` to xharness for explicit targeting + +### MacCatalyst +- Runs directly on the Mac, no device targeting needed +- XHarness uses `--target maccatalyst` + +### Android +1. **Emulator Boot**: Start-Emulator.ps1 detects running device or boots emulator +2. **Device ID Extraction**: Script gets device ID (e.g., `emulator-5554`) +3. **XHarness Targeting**: Passes `--device-id` to xharness android command + +### Windows +- No device/emulator needed +- Uses vstest (`dotnet test`) for test execution + +**Why both --target and --device for iOS?** +- XHarness requires `--target ios-simulator-64` (or `ios-simulator-64_VERSION`) to specify platform type +- Adding `--device UDID` explicitly tells xharness which simulator to use +- This combination ensures reliable device selection even on ARM64 Macs where automatic detection can fail + +**Example xharness invocations:** +```bash +# iOS with test filter +dotnet xharness apple test \ + --app path/to/app \ + --target ios-simulator-64_18.5 \ + --device 56AE278D-60F7-4892-9DE0-6341357CA068 \ + -o artifacts/log \ + --timeout 01:00:00 \ + -v \ + --set-env=TestFilter=Category=Button + +# MacCatalyst with test filter +dotnet xharness apple test \ + --app path/to/app \ + --target maccatalyst \ + -o artifacts/log \ + --timeout 01:00:00 \ + -v \ + --set-env=TestFilter=Category=Button + +# Android with test filter +dotnet xharness android test \ + --app path/to/app.apk \ + --package-name com.microsoft.maui.controls.devicetests \ + --device-id emulator-5554 \ + -o artifacts/log \ + --timeout 01:00:00 \ + -v \ + --arg TestFilter=Category=Button +``` + +This ensures tests run on the correct device with proper version targeting and filtering. diff --git a/.github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 b/.github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 new file mode 100644 index 000000000000..f94b2d5b044e --- /dev/null +++ b/.github/skills/run-device-tests/scripts/Run-DeviceTests.ps1 @@ -0,0 +1,619 @@ +<# +.SYNOPSIS + Builds and runs .NET MAUI device tests locally using xharness (Apple/Android) or vstest (Windows). + +.DESCRIPTION + This script builds a specified MAUI device test project for the target platform + and runs the tests. It handles device/emulator/simulator selection, build configuration, + and test execution. + + Platform support by OS: + - macOS: ios, maccatalyst, android + - Windows: android, windows + +.PARAMETER Project + The device test project to run. Valid values: Controls, Core, Essentials, Graphics, BlazorWebView + +.PARAMETER Platform + Target platform. Valid values depend on OS: + - macOS: ios (default), maccatalyst, android + - Windows: android, windows (default) + +.PARAMETER iOSVersion + Optional iOS version to target (e.g., "26", "18"). Only applies to ios platform. + +.PARAMETER Configuration + Build configuration. Defaults to "Release". + +.PARAMETER TestFilter + Optional test filter to run specific tests (e.g., "Category=Button"). + +.PARAMETER BuildOnly + If specified, only builds the project without running tests. + +.PARAMETER OutputDirectory + Directory for test logs and results. Defaults to "artifacts/log". + +.PARAMETER Timeout + Test timeout in format HH:MM:SS. Defaults to "01:00:00" (1 hour). + +.PARAMETER DeviceUdid + Optional specific device UDID to use. If not provided, auto-detects appropriate device. + +.EXAMPLE + ./Run-DeviceTests.ps1 -Project Controls -Platform ios + +.EXAMPLE + ./Run-DeviceTests.ps1 -Project Core -Platform maccatalyst + +.EXAMPLE + ./Run-DeviceTests.ps1 -Project Controls -Platform android + +.EXAMPLE + ./Run-DeviceTests.ps1 -Project Controls -Platform windows + +.EXAMPLE + ./Run-DeviceTests.ps1 -Project Controls -Platform ios -iOSVersion 26 + +.EXAMPLE + ./Run-DeviceTests.ps1 -Project Controls -Platform ios -TestFilter "Category=Button" + +.EXAMPLE + ./Run-DeviceTests.ps1 -Project Controls -Platform android -BuildOnly +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true, Position = 0)] + [ValidateSet("Controls", "Core", "Essentials", "Graphics", "BlazorWebView")] + [string]$Project, + + [Parameter(Mandatory = $false)] + [ValidateSet("ios", "maccatalyst", "android", "windows")] + [string]$Platform, + + [Parameter(Mandatory = $false)] + [string]$iOSVersion, + + [Parameter(Mandatory = $false)] + [string]$Configuration = "Release", + + [Parameter(Mandatory = $false)] + [string]$TestFilter, + + [Parameter(Mandatory = $false)] + [switch]$BuildOnly, + + [Parameter(Mandatory = $false)] + [string]$OutputDirectory = "artifacts/log", + + [Parameter(Mandatory = $false)] + [string]$Timeout = "01:00:00", + + [Parameter(Mandatory = $false)] + [string]$DeviceUdid, + + [Parameter(Mandatory = $false)] + [switch]$SkipXcodeVersionCheck +) + +$ErrorActionPreference = "Stop" + +# Determine default platform based on OS +if (-not $Platform) { + if ($IsWindows) { + $Platform = "windows" + } else { + $Platform = "ios" + } +} + +# Validate platform availability on current OS +$validPlatforms = if ($IsWindows) { @("android", "windows") } else { @("ios", "maccatalyst", "android") } +if ($Platform -notin $validPlatforms) { + Write-Error "Platform '$Platform' is not supported on this OS. Valid platforms: $($validPlatforms -join ', ')" + exit 1 +} + +# iOSVersion only applies to ios platform +if ($iOSVersion -and $Platform -ne "ios") { + Write-Warning "-iOSVersion parameter is only applicable to ios platform. Ignoring." + $iOSVersion = $null +} + +# Project paths mapping +$ProjectPaths = @{ + "Controls" = "src/Controls/tests/DeviceTests/Controls.DeviceTests.csproj" + "Core" = "src/Core/tests/DeviceTests/Core.DeviceTests.csproj" + "Essentials" = "src/Essentials/test/DeviceTests/Essentials.DeviceTests.csproj" + "Graphics" = "src/Graphics/tests/DeviceTests/Graphics.DeviceTests.csproj" + "BlazorWebView" = "src/BlazorWebView/tests/DeviceTests/MauiBlazorWebView.DeviceTests.csproj" +} + +$AppNames = @{ + "Controls" = "Microsoft.Maui.Controls.DeviceTests" + "Core" = "Microsoft.Maui.Core.DeviceTests" + "Essentials" = "Microsoft.Maui.Essentials.DeviceTests" + "Graphics" = "Microsoft.Maui.Graphics.DeviceTests" + "BlazorWebView" = "Microsoft.Maui.MauiBlazorWebView.DeviceTests" +} + +# Android package names (lowercase) +$AndroidPackageNames = @{ + "Controls" = "com.microsoft.maui.controls.devicetests" + "Core" = "com.microsoft.maui.core.devicetests" + "Essentials" = "com.microsoft.maui.essentials.devicetests" + "Graphics" = "com.microsoft.maui.graphics.devicetests" + "BlazorWebView" = "com.microsoft.maui.mauiblazorwebview.devicetests" +} + +# Platform-specific configurations +$PlatformConfigs = @{ + "ios" = @{ + Tfm = "net10.0-ios" + RuntimeIdentifier = "iossimulator-arm64" + AppExtension = ".app" + XHarnessTarget = "ios-simulator-64" + UsesXHarness = $true + EmulatorPlatform = "ios" + } + "maccatalyst" = @{ + Tfm = "net10.0-maccatalyst" + RuntimeIdentifier = "maccatalyst-arm64" + AppExtension = ".app" + XHarnessTarget = "maccatalyst" + UsesXHarness = $true + EmulatorPlatform = $null # No emulator needed for Mac Catalyst + } + "android" = @{ + Tfm = "net10.0-android" + RuntimeIdentifier = $null # Let MSBuild choose + AppExtension = "-Signed.apk" + XHarnessTarget = "android-emulator-64" + UsesXHarness = $true + EmulatorPlatform = "android" + } + "windows" = @{ + Tfm = "net10.0-windows10.0.19041.0" + RuntimeIdentifier = "win10-x64" + AppExtension = ".exe" + XHarnessTarget = $null + UsesXHarness = $false + EmulatorPlatform = $null # No emulator needed for Windows + } +} + +# Find repository root +$RepoRoot = $PSScriptRoot +while ($RepoRoot -and -not (Test-Path (Join-Path $RepoRoot ".git"))) { + $RepoRoot = Split-Path $RepoRoot -Parent +} + +if (-not $RepoRoot) { + Write-Error "Could not find repository root. Run this script from within the maui repository." + exit 1 +} + +# Import shared utilities +$SharedScriptsDir = Join-Path $RepoRoot ".github/scripts/shared" +. (Join-Path $SharedScriptsDir "shared-utils.ps1") + +Push-Location $RepoRoot + +$platformConfig = $PlatformConfigs[$Platform] + +try { + # Validate prerequisites + Write-Host "═══════════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-Host " MAUI Device Tests Runner" -ForegroundColor Cyan + Write-Host "═══════════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-Host "" + + # Check for xharness if needed (try local tool first, then global) + $useLocalXharness = $false + if ($platformConfig.UsesXHarness) { + $xharness = Get-Command "xharness" -ErrorAction SilentlyContinue + + if (-not $xharness) { + # Try dotnet tool (local tool manifest) + try { + $null = & dotnet xharness help 2>&1 + Write-Host "✓ xharness found: local dotnet tool" -ForegroundColor Green + $useLocalXharness = $true + } catch { + Write-Error "xharness is not installed. Install with: dotnet tool install --global Microsoft.DotNet.XHarness.CLI" + exit 1 + } + } else { + Write-Host "✓ xharness found: $($xharness.Source)" -ForegroundColor Green + } + } + + # Check for dotnet + $dotnet = Get-Command "dotnet" -ErrorAction SilentlyContinue + if (-not $dotnet) { + Write-Error "dotnet is not installed." + exit 1 + } + Write-Host "✓ dotnet found: $($dotnet.Source)" -ForegroundColor Green + + $projectPath = $ProjectPaths[$Project] + $appName = $AppNames[$Project] + + Write-Host "" + Write-Host "Project: $Project" -ForegroundColor Yellow + Write-Host "Project Path: $projectPath" -ForegroundColor Yellow + Write-Host "Platform: $Platform" -ForegroundColor Yellow + Write-Host "Configuration: $Configuration" -ForegroundColor Yellow + if ($iOSVersion) { + Write-Host "iOS Version: $iOSVersion" -ForegroundColor Yellow + } + if ($TestFilter) { + Write-Host "Test Filter: $TestFilter" -ForegroundColor Yellow + } + Write-Host "" + + # ═══════════════════════════════════════════════════════════ + # BUILD PHASE + # ═══════════════════════════════════════════════════════════ + Write-Host "═══════════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-Host " Building $Project Device Tests for $Platform" -ForegroundColor Cyan + Write-Host "═══════════════════════════════════════════════════════════" -ForegroundColor Cyan + + $buildArgs = @( + "build" + $projectPath + "-c", $Configuration + "-f", $platformConfig.Tfm + "/p:TreatWarningsAsErrors=false" + ) + + # Add RuntimeIdentifier if specified + if ($platformConfig.RuntimeIdentifier) { + $buildArgs += "-r", $platformConfig.RuntimeIdentifier + } + + # Platform-specific build properties + switch ($Platform) { + "ios" { + $buildArgs += "/p:CodesignRequireProvisioningProfile=false" + if ($SkipXcodeVersionCheck) { + $buildArgs += "/p:ValidateXcodeVersion=false" + } + } + "maccatalyst" { + $buildArgs += "/p:CodesignRequireProvisioningProfile=false" + if ($SkipXcodeVersionCheck) { + $buildArgs += "/p:ValidateXcodeVersion=false" + } + } + "android" { + $buildArgs += "/p:AndroidPackageFormat=apk" + } + "windows" { + $buildArgs += "/p:WindowsPackageType=None" + $buildArgs += "/p:WindowsAppSDKSelfContained=true" + } + } + + Write-Host "Running: dotnet $($buildArgs -join ' ')" -ForegroundColor Gray + Write-Host "" + + & dotnet @buildArgs + + if ($LASTEXITCODE -ne 0) { + Write-Error "Build failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE + } + + Write-Host "" + Write-Host "✓ Build succeeded" -ForegroundColor Green + + # Find the built app + $tfmFolder = $platformConfig.Tfm + $ridFolder = if ($platformConfig.RuntimeIdentifier) { $platformConfig.RuntimeIdentifier } else { "" } + + # Construct app path based on platform + switch ($Platform) { + "ios" { + $appPath = "artifacts/bin/$Project.DeviceTests/$Configuration/$tfmFolder/$ridFolder/$appName.app" + } + "maccatalyst" { + # MacCatalyst apps may have different names - search for .app bundle + $appSearchPath = "artifacts/bin/$Project.DeviceTests/$Configuration/$tfmFolder/$ridFolder" + $appBundle = Get-ChildItem -Path $appSearchPath -Filter "*.app" -Directory -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($appBundle) { + $appPath = $appBundle.FullName + } else { + $appPath = "$appSearchPath/$appName.app" + } + } + "android" { + # Android APK path - look for signed APK + $apkSearchPath = "artifacts/bin/$Project.DeviceTests/$Configuration/$tfmFolder" + $apkFile = Get-ChildItem -Path $apkSearchPath -Filter "*-Signed.apk" -Recurse -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($apkFile) { + $appPath = $apkFile.FullName + } else { + # Fall back to unsigned APK + $apkFile = Get-ChildItem -Path $apkSearchPath -Filter "*.apk" -Recurse -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($apkFile) { + $appPath = $apkFile.FullName + } else { + $appPath = "$apkSearchPath/$appName.apk" + } + } + } + "windows" { + $appPath = "artifacts/bin/$Project.DeviceTests/$Configuration/$tfmFolder/$ridFolder/$appName.exe" + } + } + + if (-not (Test-Path $appPath)) { + Write-Error "Built app not found at: $appPath" + Write-Info "Searching for app in artifacts..." + Get-ChildItem -Path "artifacts/bin/$Project.DeviceTests" -Recurse -ErrorAction SilentlyContinue | + Where-Object { $_.Name -match "$appName" } | + ForEach-Object { Write-Host " Found: $($_.FullName)" } + exit 1 + } + + Write-Host "✓ App found: $appPath" -ForegroundColor Green + + if ($BuildOnly) { + Write-Host "" + Write-Host "═══════════════════════════════════════════════════════════" -ForegroundColor Green + Write-Host " Build completed (BuildOnly mode)" -ForegroundColor Green + Write-Host "═══════════════════════════════════════════════════════════" -ForegroundColor Green + exit 0 + } + + # ═══════════════════════════════════════════════════════════ + # DEVICE/EMULATOR SETUP (if needed) + # ═══════════════════════════════════════════════════════════ + $deviceUdidToUse = $DeviceUdid + $DetectedIOSVersion = $null + + if ($platformConfig.EmulatorPlatform) { + Write-Host "" + Write-Host "═══════════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-Host " Starting $Platform Device/Emulator" -ForegroundColor Cyan + Write-Host "═══════════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-Host "" + + # Use Start-Emulator.ps1 to detect/boot device + $startEmulatorPath = Join-Path $SharedScriptsDir "Start-Emulator.ps1" + + $emulatorArgs = @("-File", $startEmulatorPath, "-Platform", $platformConfig.EmulatorPlatform) + if ($DeviceUdid) { + $emulatorArgs += "-DeviceUdid", $DeviceUdid + } + + $emulatorOutput = & pwsh @emulatorArgs 2>&1 + + # Extract UDID from output (last line, trimmed) + $deviceUdidToUse = ($emulatorOutput | Select-Object -Last 1).ToString().Trim() + + # Validate UDID format based on platform + $validUdid = $false + switch ($Platform) { + "ios" { + $validUdid = $deviceUdidToUse -match '^[A-F0-9]{8}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{12}$' + } + "android" { + $validUdid = $deviceUdidToUse -match '^emulator-\d+$' -or $deviceUdidToUse -match '^[a-zA-Z0-9]+$' + } + } + + if (-not $validUdid) { + Write-Error "Failed to get valid device UDID. Got: $deviceUdidToUse" + Write-Host "Full output:" -ForegroundColor Red + $emulatorOutput | ForEach-Object { Write-Host $_ } + exit 1 + } + + Write-Host "" + Write-Host "✓ Device ready: $deviceUdidToUse" -ForegroundColor Green + + # Extract iOS version from the booted simulator for XHarness targeting + if ($Platform -eq "ios" -and -not $iOSVersion) { + Write-Host "" + Write-Host "Detecting iOS version from simulator..." -ForegroundColor Gray + + try { + $simListJson = xcrun simctl list devices available -j | ConvertFrom-Json + + foreach ($runtime in $simListJson.devices.PSObject.Properties) { + $device = $runtime.Value | Where-Object { $_.udid -eq $deviceUdidToUse } + if ($device) { + # Extract version from runtime key (e.g., "com.apple.CoreSimulator.SimRuntime.iOS-18-5" -> "18.5") + if ($runtime.Name -match 'iOS-(\d+)-(\d+)') { + $DetectedIOSVersion = "$($matches[1]).$($matches[2])" + Write-Host "✓ Detected iOS version: $DetectedIOSVersion" -ForegroundColor Green + } + break + } + } + } catch { + Write-Warning "Could not detect iOS version from simulator. Continuing without version in target." + } + } + } + + # ═══════════════════════════════════════════════════════════ + # TEST PHASE + # ═══════════════════════════════════════════════════════════ + Write-Host "" + Write-Host "═══════════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-Host " Running Tests" -ForegroundColor Cyan + Write-Host "═══════════════════════════════════════════════════════════" -ForegroundColor Cyan + + # Create output directory + if (-not (Test-Path $OutputDirectory)) { + New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null + } + + $testExitCode = 0 + + if ($platformConfig.UsesXHarness) { + # ═══════════════════════════════════════════════════════════ + # XHARNESS TEST EXECUTION (iOS, MacCatalyst, Android) + # ═══════════════════════════════════════════════════════════ + + # Determine target + $target = $platformConfig.XHarnessTarget + + # Add iOS version to target if available + if ($Platform -eq "ios") { + $targetVersion = if ($iOSVersion) { $iOSVersion } else { $DetectedIOSVersion } + if ($targetVersion) { + $target = "ios-simulator-64_$targetVersion" + } + } + + # Build xharness arguments based on platform + switch ($Platform) { + "ios" { + $xharnessArgs = @( + "apple", "test" + "--app", $appPath + "--target", $target + "--device", $deviceUdidToUse + "-o", $OutputDirectory + "--timeout", $Timeout + "-v" + ) + } + "maccatalyst" { + $xharnessArgs = @( + "apple", "test" + "--app", $appPath + "--target", "maccatalyst" + "-o", $OutputDirectory + "--timeout", $Timeout + "-v" + ) + } + "android" { + $androidPackageName = $AndroidPackageNames[$Project] + $xharnessArgs = @( + "android", "test" + "--app", $appPath + "--package-name", $androidPackageName + "--device-id", $deviceUdidToUse + "-o", $OutputDirectory + "--timeout", $Timeout + "-v" + ) + } + } + + if ($TestFilter) { + if ($Platform -eq "android") { + # Android uses --arg for instrumentation arguments + $xharnessArgs += "--arg", "TestFilter=$TestFilter" + } else { + # iOS/MacCatalyst uses --set-env + $xharnessArgs += "--set-env=TestFilter=$TestFilter" + } + } + + if ($useLocalXharness) { + $xharnessCommand = "dotnet xharness" + } else { + $xharnessCommand = "xharness" + } + + Write-Host "Running: $xharnessCommand $($xharnessArgs -join ' ')" -ForegroundColor Gray + Write-Host "" + Write-Host "Target: $target" -ForegroundColor Yellow + if ($deviceUdidToUse) { + Write-Host "Device: $deviceUdidToUse" -ForegroundColor Yellow + } + Write-Host "" + + if ($useLocalXharness) { + & dotnet xharness @xharnessArgs + } else { + & xharness @xharnessArgs + } + + $testExitCode = $LASTEXITCODE + } else { + # ═══════════════════════════════════════════════════════════ + # VSTEST EXECUTION (Windows) + # ═══════════════════════════════════════════════════════════ + + Write-Host "Running tests with vstest..." -ForegroundColor Gray + Write-Host "" + + $vstestArgs = @( + "test" + $projectPath + "-c", $Configuration + "-f", $platformConfig.Tfm + "--no-build" + "--logger", "trx;LogFileName=TestResults.trx" + "--results-directory", $OutputDirectory + ) + + if ($TestFilter) { + $vstestArgs += "--filter", $TestFilter + } + + Write-Host "Running: dotnet $($vstestArgs -join ' ')" -ForegroundColor Gray + Write-Host "" + + & dotnet @vstestArgs + + $testExitCode = $LASTEXITCODE + } + + # ═══════════════════════════════════════════════════════════ + # RESULTS + # ═══════════════════════════════════════════════════════════ + Write-Host "" + Write-Host "═══════════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-Host " Test Results" -ForegroundColor Cyan + Write-Host "═══════════════════════════════════════════════════════════" -ForegroundColor Cyan + + # Try to find and parse the log file + $logFile = Get-ChildItem -Path $OutputDirectory -Filter "$appName.log" -ErrorAction SilentlyContinue | Select-Object -First 1 + + if ($logFile) { + $logContent = Get-Content $logFile.FullName -Raw + $passCount = ([regex]::Matches($logContent, '\[PASS\]')).Count + $failCount = ([regex]::Matches($logContent, '\[FAIL\]')).Count + + Write-Host "" + Write-Host " Passed: $passCount" -ForegroundColor Green + Write-Host " Failed: $failCount" -ForegroundColor $(if ($failCount -gt 0) { "Red" } else { "Green" }) + Write-Host "" + Write-Host " Log file: $($logFile.FullName)" -ForegroundColor Gray + + if ($failCount -gt 0) { + Write-Host "" + Write-Host " Failed tests:" -ForegroundColor Red + $logContent -split "`n" | Where-Object { $_ -match '\[FAIL\]' } | + ForEach-Object { $_ -replace '.*\[FAIL\]\s*', '' } | + Select-Object -Unique | + ForEach-Object { Write-Host " - $_" -ForegroundColor Red } + } + } + + Write-Host "" + if ($testExitCode -eq 0) { + Write-Host "═══════════════════════════════════════════════════════════" -ForegroundColor Green + Write-Host " Tests completed successfully" -ForegroundColor Green + Write-Host "═══════════════════════════════════════════════════════════" -ForegroundColor Green + } else { + Write-Host "═══════════════════════════════════════════════════════════" -ForegroundColor Yellow + Write-Host " Tests completed with exit code: $testExitCode" -ForegroundColor Yellow + Write-Host "═══════════════════════════════════════════════════════════" -ForegroundColor Yellow + } + + exit $testExitCode + +} finally { + Pop-Location +} diff --git a/src/Core/tests/DeviceTests.Shared/DeviceTestSharedHelpers.cs b/src/Core/tests/DeviceTests.Shared/DeviceTestSharedHelpers.cs index bc350fb68cca..ee7a24ecbefb 100644 --- a/src/Core/tests/DeviceTests.Shared/DeviceTestSharedHelpers.cs +++ b/src/Core/tests/DeviceTests.Shared/DeviceTestSharedHelpers.cs @@ -1,8 +1,12 @@ +#nullable enable using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Reflection; +#if ANDROID +using Microsoft.Maui.TestUtils.DeviceTests.Runners.HeadlessRunner; +#endif namespace Microsoft.Maui.DeviceTests { @@ -16,7 +20,7 @@ public static string[] GetTestCategoryValues([DynamicallyAccessedMembers(Dynamic { if (field.FieldType == typeof(string)) { - values.Add((string)field.GetValue(null)); + values.Add((string)field.GetValue(null)!); } } @@ -25,38 +29,48 @@ public static string[] GetTestCategoryValues([DynamicallyAccessedMembers(Dynamic public static List GetExcludedTestCategories([DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicFields)] this Type testCategoryType) { + string? filterValue = null; + #if IOS || MACCATALYST foreach (var en in Foundation.NSProcessInfo.ProcessInfo.Environment) { string key = $"{en.Key}"; - string filterValue = $"{en.Value}"; - if (key == "TestFilter") { - // Support TestFilter=Category=X (run only category X) - if (filterValue.StartsWith("Category=")) - { - Console.WriteLine($"TestFilter: {filterValue}"); - string categoryToRun = $"{filterValue.Split('=')[1]}"; - var categories = new List(GetTestCategoryValues(testCategoryType)); - categories.Remove(categoryToRun); - return categories.Select(c => $"Category={c}").ToList(); - } - - // Support TestFilter=SkipCategories=X,Y,Z (skip categories X, Y, Z) - if (filterValue.StartsWith("SkipCategories=")) - { - Console.WriteLine($"TestFilter: {filterValue}"); - var categoriesToSkip = filterValue.Substring("SkipCategories=".Length) - .Split(new[] { ',', ';' }) - .Select(c => c.Trim()) - .Where(c => !string.IsNullOrWhiteSpace(c)) - .ToList(); - return categoriesToSkip.Select(c => $"Category={c}").ToList(); - } + filterValue = $"{en.Value}"; + break; } } +#elif ANDROID + // Read TestFilter from instrumentation arguments + filterValue = MauiTestInstrumentation.Current?.Arguments?.GetString("TestFilter"); #endif + + if (!string.IsNullOrEmpty(filterValue)) + { + // Support TestFilter=Category=X (run only category X) + if (filterValue.StartsWith("Category=")) + { + Console.WriteLine($"TestFilter: {filterValue}"); + string categoryToRun = $"{filterValue.Split('=')[1]}"; + var categories = new List(GetTestCategoryValues(testCategoryType)); + categories.Remove(categoryToRun); + return categories.Select(c => $"Category={c}").ToList(); + } + + // Support TestFilter=SkipCategories=X,Y,Z (skip categories X, Y, Z) + if (filterValue.StartsWith("SkipCategories=")) + { + Console.WriteLine($"TestFilter: {filterValue}"); + var categoriesToSkip = filterValue.Substring("SkipCategories=".Length) + .Split(new[] { ',', ';' }) + .Select(c => c.Trim()) + .Where(c => !string.IsNullOrWhiteSpace(c)) + .ToList(); + return categoriesToSkip.Select(c => $"Category={c}").ToList(); + } + } + return new List(); } }