From 2b7071fb161a8f340c8487ae463dfcd86081bf6e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 16 Sep 2026 08:57:54 -0700 Subject: [PATCH 01/12] test: extract manual dissemination performance tooling --- .github/scripts/dissemination-performance.ps1 | 283 +++++++++++ .../test-dissemination-performance.ps1 | 50 ++ .../workflows/dissemination-performance.yml | 121 +++++ .../README.md | 122 +++++ .../Shared/Protocol.cs | 86 ++++ .../Silo/ControlTarget.cs | 24 + .../Silo/Dissemination.Silo.csproj | 21 + .../Silo/FileMembershipTable.cs | 227 +++++++++ .../Silo/NewRuntime.cs | 47 ++ .../Silo/Observation.cs | 316 ++++++++++++ .../Silo/Program.cs | 325 +++++++++++++ ...semination.PerformanceHarness.Tests.csproj | 12 + .../Tests/MeasurementTests.cs | 349 +++++++++++++ .../Tests/ProcessCluster.cs | 457 ++++++++++++++++++ .../Tests/ScalingComparison.cs | 106 ++++ .../Tests/ScalingTests.cs | 392 +++++++++++++++ .../methodology.json | 80 +++ 17 files changed, 3018 insertions(+) create mode 100644 .github/scripts/dissemination-performance.ps1 create mode 100644 .github/scripts/test-dissemination-performance.ps1 create mode 100644 .github/workflows/dissemination-performance.yml create mode 100644 test/Dissemination.PerformanceHarness/README.md create mode 100644 test/Dissemination.PerformanceHarness/Shared/Protocol.cs create mode 100644 test/Dissemination.PerformanceHarness/Silo/ControlTarget.cs create mode 100644 test/Dissemination.PerformanceHarness/Silo/Dissemination.Silo.csproj create mode 100644 test/Dissemination.PerformanceHarness/Silo/FileMembershipTable.cs create mode 100644 test/Dissemination.PerformanceHarness/Silo/NewRuntime.cs create mode 100644 test/Dissemination.PerformanceHarness/Silo/Observation.cs create mode 100644 test/Dissemination.PerformanceHarness/Silo/Program.cs create mode 100644 test/Dissemination.PerformanceHarness/Tests/Dissemination.PerformanceHarness.Tests.csproj create mode 100644 test/Dissemination.PerformanceHarness/Tests/MeasurementTests.cs create mode 100644 test/Dissemination.PerformanceHarness/Tests/ProcessCluster.cs create mode 100644 test/Dissemination.PerformanceHarness/Tests/ScalingComparison.cs create mode 100644 test/Dissemination.PerformanceHarness/Tests/ScalingTests.cs create mode 100644 test/Dissemination.PerformanceHarness/methodology.json diff --git a/.github/scripts/dissemination-performance.ps1 b/.github/scripts/dissemination-performance.ps1 new file mode 100644 index 00000000000..004061b11a0 --- /dev/null +++ b/.github/scripts/dissemination-performance.ps1 @@ -0,0 +1,283 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory)] + [ValidatePattern('^[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9][A-Za-z0-9._-]*$')] + [string] $CandidateRepository, + [Parameter(Mandatory)] + [ValidatePattern('^[A-Za-z0-9][A-Za-z0-9._/-]{0,199}$')] + [string] $CandidateRef, + [ValidatePattern('^Artifacts[\\/][A-Za-z0-9][A-Za-z0-9._-]{0,79}$')] + [string] $ArtifactsDirectory = 'Artifacts\DisseminationPerformance', + [ValidatePattern('^\d+(,\d+){0,5}$')] + [string] $Sizes = '4,8', + [ValidateRange(3, 200)] + [int] $Iterations = 3, + [ValidateRange(1, 5)] + [int] $Repetitions = 2, + [ValidateSet('All', 'Current')] + [string] $RuntimePaths = 'All', + [ValidatePattern('^(stable|churn|partition)(,(stable|churn|partition)){0,2}$')] + [string] $Scenarios = 'stable,churn,partition', + [ValidateRange(0, 64)] + [int] $SiloProcessorCount = 0, + [ValidateRange(0, 9)] + [int] $GCConserveMemory = 0, + [switch] $SkipBuild +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +$baselineSha = '6739589254b746a8790cf53524e6abe372bb53d4' +$root = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..\..')) +$harnessSource = Join-Path $root 'test\Dissemination.PerformanceHarness' +$sizesArray = @($Sizes.Split(',') | ForEach-Object { [int]$_ }) +if (@($sizesArray | Select-Object -Unique).Count -ne $sizesArray.Count -or + @($sizesArray | Where-Object { $_ -lt 3 -or $_ -gt 128 }).Count -gt 0) { + throw 'Select one to six distinct silo counts between 3 and 128.' +} +if (@($Scenarios.Split(',') | Select-Object -Unique).Count -ne $Scenarios.Split(',').Count) { + throw 'Select each scenario at most once.' +} + +function Assert-NoLinks([string] $Path) { + $current = $Path + while ($current -and $current -ne $root) { + if (Test-Path -LiteralPath $current) { + $item = Get-Item -LiteralPath $current -Force + if ($item.Attributes -band [System.IO.FileAttributes]::ReparsePoint) { + throw "Artifact and runtime paths must use ordinary directories: $current" + } + } + $current = Split-Path -Parent $current + } +} + +function Invoke-DotNet([string[]] $Arguments) { + & dotnet @Arguments + if ($LASTEXITCODE -ne 0) { + throw "dotnet failed ($LASTEXITCODE): $($Arguments -join ' ')" + } +} + +function Get-Revision([string] $Directory) { + $revision = & git -C $Directory rev-parse HEAD + if ($LASTEXITCODE -ne 0) { + throw "Not a source checkout: $Directory" + } + return $revision.Trim() +} + +function Assert-Checkout([string] $Directory, [string] $Repository) { + Assert-NoLinks $Directory + $topLevel = & git -C $Directory rev-parse --show-toplevel + if ($LASTEXITCODE -ne 0 -or [System.IO.Path]::GetFullPath($topLevel.Trim()) -ne $Directory) { + throw "Runtime must have its own isolated checkout: $Directory" + } + $url = & git -C $Directory remote get-url origin + if ($LASTEXITCODE -ne 0 -or $url.Trim().TrimEnd('/') -notin @("https://github.com/$Repository", "https://github.com/$Repository.git")) { + throw "Runtime checkout must originate from https://github.com/$Repository." + } + $changes = & git -C $Directory status --porcelain --untracked-files=all + if ($LASTEXITCODE -ne 0 -or $changes) { + throw "Selected runtime checkout must be pristine before copying worker support: $Directory" + } +} + +function Copy-Worker([string] $Checkout) { + $destination = Join-Path $Checkout 'test\Dissemination.PerformanceHarness' + if (Test-Path -LiteralPath $destination) { + throw "Refusing to overwrite an existing runtime harness: $destination" + } + foreach ($part in @('Silo', 'Shared')) { + $partDestination = Join-Path $destination $part + New-Item -ItemType Directory -Path $partDestination -Force | Out-Null + Get-ChildItem -LiteralPath (Join-Path $harnessSource $part) -File | + Where-Object { $_.Extension -in '.cs', '.csproj' } | + Copy-Item -Destination $partDestination + } +} + +function Publish-Silo([string] $Checkout, [string] $Runtime, [string] $Revision) { + $project = Join-Path $Checkout 'test\Dissemination.PerformanceHarness\Silo\Dissemination.Silo.csproj' + $output = Join-Path $binaries $Runtime + Push-Location $Checkout + try { + Invoke-DotNet @( + 'publish', $project, '--configuration', 'Release', '--framework', 'net10.0', + '--output', $output, '--verbosity', 'minimal', + "-p:HarnessRuntime=$Runtime", "-p:SourceRevisionId=$Revision", + '-p:OfficialBuild=true', "-p:BUILD_SOURCEVERSION=$Revision", + "-bl:$(Join-Path $artifacts "build-$Runtime.binlog")" + ) + } + finally { + Pop-Location + } + $assemblyPath = Join-Path $output 'Orleans.Runtime.dll' + $identity = [System.Reflection.AssemblyName]::GetAssemblyName($assemblyPath) + $assemblies = @{} + foreach ($file in Get-ChildItem -LiteralPath $output -Filter 'Orleans*.dll' -File) { + $name = [System.Reflection.AssemblyName]::GetAssemblyName($file.FullName) + $assemblies[$name.Name] = @{ + AssemblyVersion = $name.Version.ToString() + Sha256 = (Get-FileHash -LiteralPath $file.FullName -Algorithm SHA256).Hash + } + } + @{ + Runtime = $Runtime + SourceRevision = $Revision + AssemblyName = $identity.Name + AssemblyVersion = $identity.Version.ToString() + Sha256 = (Get-FileHash -LiteralPath $assemblyPath -Algorithm SHA256).Hash + Assemblies = $assemblies + } | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath (Join-Path $output 'manifest.json') +} + +$artifacts = [System.IO.Path]::GetFullPath((Join-Path $root $ArtifactsDirectory)) +Assert-NoLinks $artifacts +if (Test-Path -LiteralPath $artifacts) { + foreach ($item in Get-ChildItem -LiteralPath $artifacts -Recurse -Force) { + if ($item.Attributes -band [System.IO.FileAttributes]::ReparsePoint) { + throw "Artifact contents must use ordinary files and directories: $($item.FullName)" + } + } +} +$binaries = Join-Path $artifacts 'bin' +if (!$SkipBuild) { + if ($env:GITHUB_ACTIONS -ne 'true') { + throw 'Runtime builds run in CI. Download the binary artifact and use -SkipBuild for local reproduction.' + } + if (Test-Path -LiteralPath $binaries) { + throw "Use a fresh artifact directory for a new build: $binaries" + } + $baseline = Join-Path $root '.dissemination-runtime-original' + $candidate = Join-Path $root '.dissemination-runtime-candidate' + Assert-Checkout $baseline 'dotnet/orleans' + Assert-Checkout $candidate $CandidateRepository + if ((Get-Revision $baseline) -ne $baselineSha) { + throw "Original runtime must be pinned to $baselineSha." + } + $candidateSha = Get-Revision $candidate + if ($CandidateRef -match '^[0-9a-fA-F]{40}$' -and $candidateSha -ne $CandidateRef) { + throw "Candidate checkout does not match requested commit $CandidateRef." + } + if (!(Test-Path -LiteralPath (Join-Path $candidate 'src\Orleans.Runtime\Dissemination'))) { + throw 'Select a candidate revision containing the dissemination runtime APIs.' + } + if (Test-Path -LiteralPath (Join-Path $baseline 'src\Orleans.Runtime\Dissemination')) { + throw 'Original runtime unexpectedly contains dissemination.' + } + New-Item -ItemType Directory -Force $binaries | Out-Null + Copy-Worker $baseline + Copy-Worker $candidate + Publish-Silo $baseline 'Old' $baselineSha + Publish-Silo $candidate 'New' $candidateSha + Invoke-DotNet @( + 'publish', (Join-Path $harnessSource 'Tests\Dissemination.PerformanceHarness.Tests.csproj'), + '--configuration', 'Release', '--framework', 'net10.0', + '--output', (Join-Path $binaries 'Runner'), '--verbosity', 'minimal', + "-bl:$(Join-Path $artifacts 'build-runner.binlog')" + ) + @{ + ToolingCommit = Get-Revision $root + BaselineRepository = 'dotnet/orleans' + BaselineCommit = $baselineSha + CandidateRepository = $CandidateRepository + CandidateRef = $CandidateRef + CandidateCommit = $candidateSha + DotNetInfo = (& dotnet --info | Out-String) + } | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $binaries 'selection.json') +} + +$selection = Get-Content -LiteralPath (Join-Path $binaries 'selection.json') -Raw | ConvertFrom-Json +if ($selection.CandidateRepository -ne $CandidateRepository -or $selection.CandidateRef -cne $CandidateRef -or + $selection.BaselineCommit -ne $baselineSha) { + throw 'Requested revisions must match the downloaded binary selection.json.' +} +foreach ($runtime in @('Old', 'New')) { + $manifest = Get-Content -LiteralPath (Join-Path $binaries "$runtime\manifest.json") -Raw | ConvertFrom-Json + $expected = if ($runtime -eq 'Old') { $baselineSha } else { $selection.CandidateCommit } + if ($manifest.SourceRevision -ne $expected -or $manifest.Runtime -ne $runtime) { + throw "Binary manifest does not match selected $runtime revision." + } +} + +$runner = Join-Path $binaries 'Runner\Dissemination.PerformanceHarness.Tests.dll' +if (!(Test-Path -LiteralPath $runner)) { + throw "Published runner not found: $runner" +} +$results = Join-Path $artifacts "results\$([DateTime]::UtcNow.ToString('yyyyMMddTHHmmss'))-$([Guid]::NewGuid().ToString('N'))" +New-Item -ItemType Directory -Force $results | Out-Null +Copy-Item -LiteralPath (Join-Path $harnessSource 'methodology.json') -Destination $results +Copy-Item -LiteralPath (Join-Path $binaries 'selection.json') -Destination $results + +$largestSize = ($sizesArray | Measure-Object -Maximum).Maximum +$requiredBytes = [long]$largestSize * 128MB + 1GB +if ($IsLinux) { + $memory = Get-Content -LiteralPath '/proc/meminfo' + $availableLine = $memory | Where-Object { $_ -match '^MemAvailable:\s+(\d+)\s+kB$' } + if (!$availableLine -or $availableLine -notmatch '^MemAvailable:\s+(\d+)\s+kB$') { + throw 'Cannot establish available host memory.' + } + $availableBytes = [long]$Matches[1] * 1024 +} +elseif ($IsWindows) { + $availableBytes = [long](Get-CimInstance Win32_OperatingSystem).FreePhysicalMemory * 1024 +} +else { + throw 'Resource preflight supports Linux CI and Windows/Linux local reproduction.' +} +@{ + AvailableBytes = $availableBytes + RequiredPlanningBytes = $requiredBytes + PlanningBytesPerSilo = 128MB + ControllerHeadroomBytes = 1GB + HostProcessorCount = [Environment]::ProcessorCount + SiloProcessorCount = $SiloProcessorCount + GCConserveMemory = $GCConserveMemory +} | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $results 'host-resources.json') +if ($availableBytes -lt $requiredBytes) { + throw "Insufficient memory for $largestSize silos: available=$availableBytes; planning requirement=$requiredBytes." +} + +$env:ORLEANS_DISSEMINATION_OLD = Join-Path $binaries 'Old' +$env:ORLEANS_DISSEMINATION_NEW = Join-Path $binaries 'New' +$env:ORLEANS_DISSEMINATION_RESULTS = $results +$env:ORLEANS_DISSEMINATION_CANDIDATE_SHA = $selection.CandidateCommit +$env:ORLEANS_DISSEMINATION_MANUAL_RUN = 'true' +$env:ORLEANS_DISSEMINATION_SIZES = $Sizes +$env:ORLEANS_DISSEMINATION_ITERATIONS = $Iterations.ToString([System.Globalization.CultureInfo]::InvariantCulture) +$env:ORLEANS_DISSEMINATION_REPETITIONS = $Repetitions.ToString([System.Globalization.CultureInfo]::InvariantCulture) +$env:ORLEANS_DISSEMINATION_RUNTIME_PATHS = $RuntimePaths +$env:ORLEANS_DISSEMINATION_SCENARIOS = $Scenarios +$env:ORLEANS_DISSEMINATION_SILO_PROCESSOR_COUNT = $SiloProcessorCount.ToString([System.Globalization.CultureInfo]::InvariantCulture) +$env:ORLEANS_DISSEMINATION_GC_CONSERVE_MEMORY = $GCConserveMemory.ToString([System.Globalization.CultureInfo]::InvariantCulture) +@{ + ToolingCommit = Get-Revision $root + Selection = $selection + Sizes = $Sizes + Iterations = $Iterations + Repetitions = $Repetitions + RuntimePaths = $RuntimePaths + Scenarios = $Scenarios + SiloProcessorCount = $SiloProcessorCount + GCConserveMemory = $GCConserveMemory + StartUtc = [DateTimeOffset]::UtcNow + ResultsDirectory = $results +} | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath (Join-Path $results 'invocation.json') + +Push-Location $root +try { + Invoke-DotNet @( + $runner, '--filter-class', 'Orleans.Dissemination.PerformanceHarness.MeasurementTests', + '--minimum-expected-tests', '15', '--report-trx', + '--results-directory', (Join-Path $results 'instrument-checks') + ) + Invoke-DotNet @( + $runner, '--filter-class', 'Orleans.Dissemination.PerformanceHarness.ScalingTests', + '--minimum-expected-tests', '1', '--report-trx', '--results-directory', $results + ) +} +finally { + Pop-Location +} diff --git a/.github/scripts/test-dissemination-performance.ps1 b/.github/scripts/test-dissemination-performance.ps1 new file mode 100644 index 00000000000..059b8891352 --- /dev/null +++ b/.github/scripts/test-dissemination-performance.ps1 @@ -0,0 +1,50 @@ +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +$script = Join-Path $PSScriptRoot 'dissemination-performance.ps1' +$errors = $null +$null = [System.Management.Automation.Language.Parser]::ParseFile($script, [ref]$null, [ref]$errors) +if ($errors) { + throw ($errors | Out-String) +} + +$cases = @( + @{ Name = 'minimum process count'; Parameter = 'Sizes'; Value = '2'; Error = '*distinct silo counts*' } + @{ Name = 'maximum process count'; Parameter = 'Sizes'; Value = '129'; Error = '*distinct silo counts*' } + @{ Name = 'duplicate sizes'; Parameter = 'Sizes'; Value = '4,4'; Error = '*distinct silo counts*' } + @{ Name = 'size cardinality'; Parameter = 'Sizes'; Value = '3,4,5,6,7,8,9'; Error = '*cannot validate argument*' } + @{ Name = 'minimum rounds'; Parameter = 'Iterations'; Value = 2; Error = '*cannot validate argument*' } + @{ Name = 'maximum rounds'; Parameter = 'Iterations'; Value = 201; Error = '*cannot validate argument*' } + @{ Name = 'maximum repetitions'; Parameter = 'Repetitions'; Value = 6; Error = '*cannot validate argument*' } + @{ Name = 'duplicate scenarios'; Parameter = 'Scenarios'; Value = 'stable,stable'; Error = '*at most once*' } + @{ Name = 'unknown scenario'; Parameter = 'Scenarios'; Value = 'unknown'; Error = '*cannot validate argument*' } + @{ Name = 'absolute artifact path'; Parameter = 'ArtifactsDirectory'; Value = 'C:\outside'; Error = '*cannot validate argument*' } + @{ Name = 'artifact traversal'; Parameter = 'ArtifactsDirectory'; Value = 'Artifacts\..\outside'; Error = '*cannot validate argument*' } + @{ Name = 'artifact root'; Parameter = 'ArtifactsDirectory'; Value = 'Artifacts'; Error = '*cannot validate argument*' } + @{ Name = 'nested artifact path'; Parameter = 'ArtifactsDirectory'; Value = 'Artifacts\run\bin'; Error = '*cannot validate argument*' } + @{ Name = 'repository URL'; Parameter = 'CandidateRepository'; Value = 'https://github.com/ReubenBond/orleans'; Error = '*cannot validate argument*' } + @{ Name = 'ref option'; Parameter = 'CandidateRef'; Value = '--upload-pack=command'; Error = '*cannot validate argument*' } + @{ Name = 'processor bound'; Parameter = 'SiloProcessorCount'; Value = 65; Error = '*cannot validate argument*' } + @{ Name = 'GC bound'; Parameter = 'GCConserveMemory'; Value = 10; Error = '*cannot validate argument*' } +) +foreach ($case in $cases) { + $arguments = @{ + CandidateRepository = 'ReubenBond/orleans' + CandidateRef = '3fc0a4610cd2eb5feef27e958fcca59658aaae47' + SkipBuild = $true + } + $arguments[$case.Parameter] = $case.Value + $rejected = $false + try { + & $script @arguments + } + catch { + if ($_.Exception.Message -notlike $case.Error) { + throw "Unexpected failure for $($case.Name): $_" + } + $rejected = $true + } + if (!$rejected) { + throw "Expected rejection: $($case.Name)" + } +} +Write-Output "Passed $($cases.Count) script-boundary checks." diff --git a/.github/workflows/dissemination-performance.yml b/.github/workflows/dissemination-performance.yml new file mode 100644 index 00000000000..1819728efd8 --- /dev/null +++ b/.github/workflows/dissemination-performance.yml @@ -0,0 +1,121 @@ +name: Dissemination performance + +on: + workflow_dispatch: + inputs: + candidate_repository: + description: 'Public runtime repository (owner/repository); selected code executes on this runner' + type: string + required: true + default: 'ReubenBond/orleans' + candidate_ref: + description: 'Runtime commit, tag or branch; resolved commit is preserved in selection.json' + type: string + required: true + default: '3fc0a4610cd2eb5feef27e958fcca59658aaae47' + sizes: + description: 'Distinct process counts (3..128, at most six)' + type: string + default: '4,8' + iterations: + description: 'Publication rounds per scenario (3..200)' + type: number + default: 3 + repetitions: + description: 'Paired repetitions with alternating forward/reverse order (1..5)' + type: number + default: 2 + runtime_paths: + description: 'All includes the pinned original; Current compares candidate off/on' + type: choice + options: [All, Current] + default: All + scenarios: + description: 'Distinct comma-separated scenarios: stable,churn,partition' + type: string + default: 'stable,churn,partition' + silo_processor_count: + description: 'Logical processors per silo (0 = inherit runner, maximum 64)' + type: number + default: 0 + gc_conserve_memory: + description: 'DOTNET_GCConserveMemory for every silo (0..9)' + type: number + default: 0 + +permissions: + contents: read + +concurrency: + group: dissemination-performance-${{ github.ref }} + cancel-in-progress: false + +env: + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + DOTNET_NOLOGO: true + +jobs: + measure: + name: Isolated original and candidate off/on measurements + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - name: Tooling checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + persist-credentials: false + - name: Original runtime checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + repository: dotnet/orleans + ref: 6739589254b746a8790cf53524e6abe372bb53d4 + path: .dissemination-runtime-original + persist-credentials: false + - name: Selected candidate runtime checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + repository: ${{ inputs.candidate_repository }} + ref: ${{ inputs.candidate_ref }} + path: .dissemination-runtime-candidate + persist-credentials: false + - uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5 + with: + global-json-file: global.json + - name: Check script input and artifact boundaries + shell: pwsh + run: ./.github/scripts/test-dissemination-performance.ps1 + - name: Build isolated binaries and measure selected workloads + shell: pwsh + env: + CANDIDATE_REPOSITORY: ${{ inputs.candidate_repository }} + CANDIDATE_REF: ${{ inputs.candidate_ref }} + PERFORMANCE_SIZES: ${{ inputs.sizes }} + PERFORMANCE_ITERATIONS: ${{ inputs.iterations }} + PERFORMANCE_REPETITIONS: ${{ inputs.repetitions }} + PERFORMANCE_PATHS: ${{ inputs.runtime_paths }} + PERFORMANCE_SCENARIOS: ${{ inputs.scenarios }} + PERFORMANCE_PROCESSORS: ${{ inputs.silo_processor_count }} + PERFORMANCE_GC: ${{ inputs.gc_conserve_memory }} + run: | + ./.github/scripts/dissemination-performance.ps1 ` + -CandidateRepository $env:CANDIDATE_REPOSITORY -CandidateRef $env:CANDIDATE_REF ` + -Sizes $env:PERFORMANCE_SIZES -Iterations ([int]$env:PERFORMANCE_ITERATIONS) ` + -Repetitions ([int]$env:PERFORMANCE_REPETITIONS) -RuntimePaths $env:PERFORMANCE_PATHS ` + -Scenarios $env:PERFORMANCE_SCENARIOS -SiloProcessorCount ([int]$env:PERFORMANCE_PROCESSORS) ` + -GCConserveMemory ([int]$env:PERFORMANCE_GC) + - name: Preserve raw measurements and build diagnostics + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: dissemination-performance-results + retention-days: 30 + path: | + Artifacts/DisseminationPerformance/results + Artifacts/DisseminationPerformance/*.binlog + - name: Preserve reproducible binaries and exact revision selection + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: dissemination-performance-binaries + retention-days: 7 + path: Artifacts/DisseminationPerformance/bin diff --git a/test/Dissemination.PerformanceHarness/README.md b/test/Dissemination.PerformanceHarness/README.md new file mode 100644 index 00000000000..5d97b2032af --- /dev/null +++ b/test/Dissemination.PerformanceHarness/README.md @@ -0,0 +1,122 @@ +# Dissemination performance harness + +This manually invoked harness measures the original Orleans runtime and a selected +candidate with dissemination explicitly off and on. Each silo is a separate OS +process using the real runtime serializer, transport, membership and load publisher. +The controller reports raw per-node costs and equal-work off/on comparisons. + +## Select runtime revisions + +Run the **Dissemination performance** workflow from the tooling branch with an +explicit `candidate_repository` and `candidate_ref`. Prefer a full commit SHA for +reproducibility. The initial comparison uses `ReubenBond/orleans` at +`3fc0a4610cd2eb5feef27e958fcca59658aaae47`; the original baseline is pinned to +`dotnet/orleans` at `6739589254b746a8790cf53524e6abe372bb53d4`. + +```powershell +gh workflow run dissemination-performance.yml --repo ReubenBond/orleans ` + --ref ` + -f candidate_repository=ReubenBond/orleans ` + -f candidate_ref=3fc0a4610cd2eb5feef27e958fcca59658aaae47 ` + -f sizes=4,8 -f iterations=3 -f repetitions=2 ` + -f runtime_paths=All -f scenarios=stable,churn,partition ` + -f silo_processor_count=0 -f gc_conserve_memory=0 +``` + +The workflow runs only through `workflow_dispatch`. GitHub requires a workflow +to be registered on the repository's default branch before it can be dispatched. +Select trusted public runtime revisions: their build code executes on the runner +with read-only repository permissions and checkout credentials disabled. + +CI creates separate original and candidate checkouts, verifies their origins and +clean state, and copies only `Silo` and `Shared` support sources into each checkout. +Each worker builds against that checkout's runtime projects and package settings. +The controller builds against the tooling checkout's shared protocol and instrumentation. +The tooling branch can therefore be based on main while the selected candidate +provides the dissemination APIs. `NewRuntime.cs` describes the expected internal API +shape; update this adapter deliberately when testing a candidate with changed APIs. + +Published manifests record DLL hashes, versions and source commits. Every worker +checks the loaded Orleans assemblies and exact process identity. Candidate off/on +runs share one binary directory. `selection.json` preserves the resolved candidate, +original and tooling commits and build SDK information. + +## Workload and bounds + +Defaults are 4 and 8 processes, three rounds, two paired repetitions, and all three +paths across `stable`, `churn` and `partition`. Sizes accept one to six distinct +counts from 3 through 128; rounds accept 3 through 200; repetitions accept 1 through +5. Successive repetitions reverse the complete execution order, including off/on. + +Every round invokes the actual load publisher once per active node and checks exact +state inventory and field values at every node. Churn restarts a process at the same +endpoint. Partition recovery drains connections at both endpoints, including +unfinished handshakes, then confirms bidirectional acknowledged control RPCs before +one publication round. This preserves the #11243 transport-readiness ordering. +Readiness time and RPC costs stay inside recovery and total cost windows. + +Automatic load publication is paused by disposing its timer on the publisher +scheduler. The timer field retains its disposed reference for original-runtime +shutdown. Enabled runs use the candidate's production cadence after peer support +confirmation; replacement startup and support confirmation remain inside churn costs. + +The Linux/Windows memory guard plans 128 MiB per silo plus 1 GiB controller headroom. +Commands, startup, shutdown and processes have deadlines; the workflow has a +90-minute budget. For an explicitly requested large stable curve, select +`sizes=8,16,32,64,100`, `runtime_paths=Current`, `scenarios=stable`, +`silo_processor_count=1`, `gc_conserve_memory=9`, and two repetitions on a suitably +provisioned host. These controls are equal across each off/on pair. + +## Results and interpretation + +`Artifacts\DisseminationPerformance\results\-` contains raw +`cost-results.json`, normalized `comparisons.json`, `selection.json`, invocation, +methodology, host resources, TRX, and per-cluster logs and snapshots. The workflow +also preserves build binlogs and `bin\{Old,New,Runner}` as separate downloadable +artifacts. Results retain partial samples and explicit incomplete pairs on failure. + +Metrics include RPC/message counts, complete serialized bytes, socket and transport +bytes, CPU, allocations, retained memory, publication/convergence/recovery latency, +and per-node topology. Load broadcast requests and logical value transmissions are +separate counters. Reports compare equal binary, environment, process count and +offered work, with costs per publication and median/P95 convergence. + +These are shared-runner, single-host loopback measurements. The workload is +closed-loop, with a convergence barrier after each round. Silo costs include control +and instrumentation work; controller costs, storage disk bytes, cross-machine latency, +TCP/IP wire overhead and application-grain workloads need separate experiments. +The 100 ms polling cadence and one-second socket sampling introduce boundary +uncertainty. Retained memory represents post-GC live bytes on surviving nodes. +Original-to-candidate changes include all intervening commits; same-candidate off/on +pairs isolate enablement. See [methodology.json](methodology.json) for exact windows. + +## Local reproduction and focused checks + +Download `dissemination-performance-binaries` from a successful dispatch into +`Artifacts\DisseminationPerformance\bin` in the same tooling revision. Install a +matching .NET 10/ASP.NET Core shared framework for Linux or Windows, then replay +the exact repository/ref recorded in `bin\selection.json`: + +```powershell +.\.github\scripts\dissemination-performance.ps1 -SkipBuild ` + -CandidateRepository ReubenBond/orleans ` + -CandidateRef 3fc0a4610cd2eb5feef27e958fcca59658aaae47 ` + -Sizes '4,8' -Iterations 3 -Repetitions 2 ` + -RuntimePaths All -Scenarios 'stable,churn,partition' +``` + +`-ArtifactsDirectory` accepts a single named directory under `Artifacts`; paths +containing existing links/reparse points are rejected. Builds require fresh binary +output; each replay creates its own results directory. + +Run only the lightweight controller measurement cases locally: + +```powershell +.\.github\scripts\test-dissemination-performance.ps1 +dotnet test --project test\Dissemination.PerformanceHarness\Tests\Dissemination.PerformanceHarness.Tests.csproj ` + --framework net10.0 --filter-class '*MeasurementTests' --minimum-expected-tests 15 +``` + +The harness projects are selected explicitly by the workflow and remain outside +the solution's ordinary test runs. The scaling test requires the script's explicit +manual-run marker and published binary selection. diff --git a/test/Dissemination.PerformanceHarness/Shared/Protocol.cs b/test/Dissemination.PerformanceHarness/Shared/Protocol.cs new file mode 100644 index 00000000000..96840b57006 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Shared/Protocol.cs @@ -0,0 +1,86 @@ +using System.Text.Json; + +namespace Orleans.Dissemination.PerformanceHarness; + +internal sealed record NodeConfiguration( + string Name, + string ClusterId, + string MembershipDirectory, + int SiloPort, + int ParentProcessId, + bool Enabled, + int SiloProcessorCount = 0, + int GCConserveMemory = 0); + +internal sealed record Command(int Id, string Operation, string? Peer = null); + +internal sealed record Response(int Id, NodeSnapshot? Snapshot, string? Error); + +internal sealed record BinaryIdentity( + int ProcessId, + string Runtime, + string SourceRevision, + string AssemblyName, + string AssemblyVersion, + string InformationalVersion, + string ModuleVersionId, + string Sha256, + string AssemblyPath, + bool HasDissemination) +{ + public Dictionary Assemblies { get; init; } = []; +} + +internal sealed record AssemblyProof( + string Version, + string InformationalVersion, + string ModuleVersionId, + string Sha256, + string Path); + +internal sealed record MetricValue(long Count, double Sum); + +internal static class StateComparison +{ + private static readonly JsonSerializerOptions Options = new() { IncludeFields = true }; + + public static string Serialize(T value) => JsonSerializer.Serialize(value, Options); +} + +internal sealed record NodeSnapshot +{ + public DateTimeOffset CapturedAtUtc { get; init; } + public required BinaryIdentity Identity { get; init; } + public required string Address { get; init; } + public bool Enabled { get; init; } + public bool NamespaceEnabled { get; init; } + public long MembershipVersion { get; init; } + public required SortedDictionary Membership { get; init; } + public required SortedDictionary Load { get; init; } + public required SortedDictionary LoadVersions { get; init; } + public required string[] ActiveMembers { get; init; } + public required string[] UnconfirmedPeers { get; init; } + public required string[] OriginatorTargets { get; init; } + public required string[] ForwardingTargets { get; init; } + public string[] TopologyMembers { get; init; } = []; + public int Fanout { get; init; } + public bool AggregationTree { get; init; } + public int AntiEntropyPeerCount { get; init; } + public double AntiEntropyIntervalMilliseconds { get; init; } + public required Dictionary Metrics { get; init; } + public long TransportBytesWritten { get; init; } + public long TransportBytesRead { get; init; } + public double SocketBytesSent { get; init; } + public double SocketBytesReceived { get; init; } + public long SocketCounterSamples { get; init; } + public double CpuMilliseconds { get; init; } + public long AllocatedBytes { get; init; } + public long ManagedHeapBytes { get; init; } + public long WorkingSetBytes { get; init; } + public long PrivateBytes { get; init; } + public int ProcessorCount { get; init; } + public bool ServerGC { get; init; } + public bool Partitioned { get; init; } + public int? RemoteProcessId { get; init; } + public string? ProbeError { get; init; } +} diff --git a/test/Dissemination.PerformanceHarness/Silo/ControlTarget.cs b/test/Dissemination.PerformanceHarness/Silo/ControlTarget.cs new file mode 100644 index 00000000000..e0a1acded30 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Silo/ControlTarget.cs @@ -0,0 +1,24 @@ +using Orleans.Runtime; + +namespace Orleans.Dissemination.PerformanceHarness; + +[Alias("dissemination-performance-control-v1")] +internal interface IControlTarget : ISystemTarget +{ + [Alias("echo-v1")] + Task Echo(CancellationToken cancellationToken); +} + +internal sealed class ControlTarget : SystemTarget, IControlTarget +{ + public static readonly GrainType TargetType = SystemTargetGrainId.CreateGrainType("test.dissemination-performance"); + + public ControlTarget(SystemTargetShared shared) : base(TargetType, shared) => + shared.ActivationDirectory.RecordNewTarget(this); + + public Task Echo(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult(Environment.ProcessId); + } +} diff --git a/test/Dissemination.PerformanceHarness/Silo/Dissemination.Silo.csproj b/test/Dissemination.PerformanceHarness/Silo/Dissemination.Silo.csproj new file mode 100644 index 00000000000..027816ec065 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Silo/Dissemination.Silo.csproj @@ -0,0 +1,21 @@ + + + net10.0 + Exe + + Orleans.Runtime.Tests + Orleans.Dissemination.PerformanceHarness + true + $(DefineConstants);NEW_RUNTIME + + + + + + + + + + + diff --git a/test/Dissemination.PerformanceHarness/Silo/FileMembershipTable.cs b/test/Dissemination.PerformanceHarness/Silo/FileMembershipTable.cs new file mode 100644 index 00000000000..0e6e88c20b4 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Silo/FileMembershipTable.cs @@ -0,0 +1,227 @@ +using System.Text.Json; +using Orleans.Runtime; + +namespace Orleans.Dissemination.PerformanceHarness; + +// A shared JSON schema and exclusive file lease coordinate the loopback silo processes. +internal sealed class FileMembershipTable(NodeConfiguration configuration) : IMembershipTable +{ + private readonly string _dataPath = Path.Combine(configuration.MembershipDirectory, "membership.json"); + private readonly string _lockPath = Path.Combine(configuration.MembershipDirectory, "membership.lock"); + +#if NEW_RUNTIME + Task IMembershipTable.InitializeMembershipTable(bool tryInitTableVersion) => + InitializeMembershipTableAsync(tryInitTableVersion, CancellationToken.None); + + Task IMembershipTable.DeleteMembershipTableEntries(string clusterId) => + DeleteMembershipTableEntriesAsync(clusterId, CancellationToken.None); + + Task IMembershipTable.CleanupDefunctSiloEntries(DateTimeOffset beforeDate) => + CleanupDefunctSiloEntriesAsync(beforeDate, CancellationToken.None); + + Task IMembershipTable.ReadRow(SiloAddress key) => ReadRowAsync(key, CancellationToken.None); + + Task IMembershipTable.ReadAll() => ReadAllAsync(CancellationToken.None); + + Task IMembershipTable.InsertRow(MembershipEntry entry, TableVersion tableVersion) => + InsertRowAsync(entry, tableVersion, CancellationToken.None); + + Task IMembershipTable.UpdateRow(MembershipEntry entry, string etag, TableVersion tableVersion) => + UpdateRowAsync(entry, etag, tableVersion, CancellationToken.None); + + Task IMembershipTable.UpdateIAmAlive(MembershipEntry entry) => UpdateIAmAliveAsync(entry, CancellationToken.None); +#else + public Task InitializeMembershipTable(bool tryInitTableVersion) => + InitializeMembershipTableAsync(tryInitTableVersion, CancellationToken.None); + + public Task DeleteMembershipTableEntries(string clusterId) => + DeleteMembershipTableEntriesAsync(clusterId, CancellationToken.None); + + public Task CleanupDefunctSiloEntries(DateTimeOffset beforeDate) => + CleanupDefunctSiloEntriesAsync(beforeDate, CancellationToken.None); + + public Task ReadRow(SiloAddress key) => ReadRowAsync(key, CancellationToken.None); + + public Task ReadAll() => ReadAllAsync(CancellationToken.None); + + public Task InsertRow(MembershipEntry entry, TableVersion tableVersion) => + InsertRowAsync(entry, tableVersion, CancellationToken.None); + + public Task UpdateRow(MembershipEntry entry, string etag, TableVersion tableVersion) => + UpdateRowAsync(entry, etag, tableVersion, CancellationToken.None); + + public Task UpdateIAmAlive(MembershipEntry entry) => UpdateIAmAliveAsync(entry, CancellationToken.None); +#endif + + public Task InitializeMembershipTableAsync(bool tryInitTableVersion, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Directory.CreateDirectory(configuration.MembershipDirectory); + return Access(static _ => true, write: true, cancellationToken); + } + + public Task DeleteMembershipTableEntriesAsync(string clusterId, CancellationToken cancellationToken) => Access(state => + { + if (!StringComparer.Ordinal.Equals(clusterId, configuration.ClusterId)) + { + throw new InvalidOperationException("The harness provider cannot delete another cluster."); + } + + state.Rows.Clear(); + state.Version++; + return true; + }, write: true, cancellationToken); + + public Task CleanupDefunctSiloEntriesAsync(DateTimeOffset beforeDate, CancellationToken cancellationToken) => Access(state => + { + foreach (var key in state.Rows.Where(pair => + pair.Value.Entry.Status == (int)SiloStatus.Dead + && pair.Value.Entry.IAmAliveTime < beforeDate.UtcDateTime).Select(pair => pair.Key).ToArray()) + { + state.Rows.Remove(key); + } + + return true; + }, write: true, cancellationToken); + + public async Task ReadRowAsync(SiloAddress key, CancellationToken cancellationToken) + { + var all = await ReadAllAsync(cancellationToken); + return new MembershipTableData(all.Members.Where(row => row.Item1.SiloAddress.Equals(key)).ToList(), all.Version); + } + + public async Task ReadAllAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var state = await Access(static state => state, write: false, cancellationToken); + return new MembershipTableData( + state.Rows.Values.Select(row => Tuple.Create(row.Entry.ToEntry(), row.Etag)).ToList(), + new TableVersion(state.Version, state.Etag)); + } + + public Task InsertRowAsync(MembershipEntry entry, TableVersion tableVersion, CancellationToken cancellationToken) => Access(state => + { + var key = entry.SiloAddress.ToParsableString(); + if (state.Rows.ContainsKey(key) || !StringComparer.Ordinal.Equals(tableVersion.VersionEtag, state.Etag)) + { + return false; + } + + state.Rows.Add(key, new Row(EntryData.From(entry), Guid.NewGuid().ToString("N"))); + state.Version = tableVersion.Version; + state.Etag = Guid.NewGuid().ToString("N"); + return true; + }, write: true, cancellationToken); + + public Task UpdateRowAsync(MembershipEntry entry, string etag, TableVersion tableVersion, CancellationToken cancellationToken) => Access(state => + { + var key = entry.SiloAddress.ToParsableString(); + if (!state.Rows.TryGetValue(key, out var row) + || !StringComparer.Ordinal.Equals(row.Etag, etag) + || !StringComparer.Ordinal.Equals(tableVersion.VersionEtag, state.Etag)) + { + return false; + } + + state.Rows[key] = new Row(EntryData.From(entry), Guid.NewGuid().ToString("N")); + state.Version = tableVersion.Version; + state.Etag = Guid.NewGuid().ToString("N"); + return true; + }, write: true, cancellationToken); + + public Task UpdateIAmAliveAsync(MembershipEntry entry, CancellationToken cancellationToken) => Access(state => + { + if (state.Rows.TryGetValue(entry.SiloAddress.ToParsableString(), out var row)) + { + state.Rows[entry.SiloAddress.ToParsableString()] = row with + { + Entry = row.Entry with { IAmAliveTime = entry.IAmAliveTime }, + }; + } + + return true; + }, write: true, cancellationToken); + + private async Task Access(Func action, bool write, CancellationToken cancellationToken) + { + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(10)); + FileStream lease; + while (true) + { + deadline.Token.ThrowIfCancellationRequested(); + try + { + lease = new FileStream(_lockPath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + break; + } + catch (IOException) + { + await Task.Delay(10, deadline.Token); + } + } + + await using (lease) + { + var state = File.Exists(_dataPath) + ? JsonSerializer.Deserialize(await File.ReadAllTextAsync(_dataPath, deadline.Token))! + : new TableState(); + deadline.Token.ThrowIfCancellationRequested(); + var result = action(state); + if (write) + { + // The sibling file is in the owned artifact directory, never the system temporary directory. + var staging = _dataPath + ".next"; + await File.WriteAllTextAsync(staging, JsonSerializer.Serialize(state), deadline.Token); + deadline.Token.ThrowIfCancellationRequested(); + File.Move(staging, _dataPath, overwrite: true); + } + + return result; + } + } + + private sealed class TableState + { + public int Version { get; set; } + public string Etag { get; set; } = "0"; + public Dictionary Rows { get; set; } = []; + } + + private sealed record Row(EntryData Entry, string Etag); + + internal sealed record EntryData( + string Address, + int Status, + int ProxyPort, + string HostName, + string SiloName, + string? RoleName, + int UpdateZone, + int FaultZone, + DateTime StartTime, + DateTime IAmAliveTime, + Dictionary Suspects) + { + public static EntryData From(MembershipEntry entry) => new( + entry.SiloAddress.ToParsableString(), (int)entry.Status, entry.ProxyPort, + entry.HostName, entry.SiloName, entry.RoleName, entry.UpdateZone, entry.FaultZone, + entry.StartTime, entry.IAmAliveTime, + entry.SuspectTimes?.ToDictionary(pair => pair.Item1.ToParsableString(), pair => pair.Item2) ?? []); + + public MembershipEntry ToEntry() => new() + { + SiloAddress = SiloAddress.FromParsableString(Address), + Status = (SiloStatus)Status, + ProxyPort = ProxyPort, + HostName = HostName, + SiloName = SiloName, + RoleName = RoleName, + UpdateZone = UpdateZone, + FaultZone = FaultZone, + StartTime = StartTime, + IAmAliveTime = IAmAliveTime, + SuspectTimes = Suspects.Select(pair => Tuple.Create(SiloAddress.FromParsableString(pair.Key), pair.Value)).ToList(), + }; + } +} diff --git a/test/Dissemination.PerformanceHarness/Silo/NewRuntime.cs b/test/Dissemination.PerformanceHarness/Silo/NewRuntime.cs new file mode 100644 index 00000000000..4e19c9b1f84 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Silo/NewRuntime.cs @@ -0,0 +1,47 @@ +#if NEW_RUNTIME +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Runtime.Dissemination; + +namespace Orleans.Dissemination.PerformanceHarness; + +internal static class NewRuntime +{ + public static void Configure(IServiceCollection services, NodeConfiguration configuration) + { + services.Configure(options => options.Enabled = configuration.Enabled); + services.Configure(options => options.Dissemination.Enabled = configuration.Enabled); + services.Configure(options => options.Dissemination.Enabled = configuration.Enabled); + } + + public static NodeSnapshot Decorate(IServiceProvider services, NodeSnapshot snapshot) + { + var options = services.GetRequiredService>().Value; + if (!options.Enabled) + { + return snapshot with + { + NamespaceEnabled = services.GetRequiredService>().Value.Dissemination.Enabled, + }; + } + + var ns = services.GetRequiredService(); + var topology = services.GetRequiredService().CurrentSnapshots.ActiveMembers; + return snapshot with + { + Enabled = options.Enabled, + NamespaceEnabled = ns.Options.Enabled, + UnconfirmedPeers = services.GetRequiredService() + .GetUnconfirmedPeers(ns).Select(address => address.ToParsableString()).Order(StringComparer.Ordinal).ToArray(), + OriginatorTargets = topology.GetOriginatorTargets(ns.RoutingMode).Select(address => address.ToParsableString()).ToArray(), + ForwardingTargets = topology.GetForwardingTargets(ns.RoutingMode).Select(address => address.ToParsableString()).ToArray(), + TopologyMembers = topology.Members.Select(address => address.ToParsableString()).ToArray(), + Fanout = options.Overlay.GetFanOutFactor(topology.Members.Length), + AggregationTree = ns.RoutingMode == DisseminationRoutingMode.AggregationTree, + AntiEntropyPeerCount = options.Overlay.AntiEntropyPeerCount, + AntiEntropyIntervalMilliseconds = options.Overlay.AntiEntropyInterval.TotalMilliseconds, + }; + } +} +#endif diff --git a/test/Dissemination.PerformanceHarness/Silo/Observation.cs b/test/Dissemination.PerformanceHarness/Silo/Observation.cs new file mode 100644 index 00000000000..ec8b2472355 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Silo/Observation.cs @@ -0,0 +1,316 @@ +using System.Buffers; +using System.Diagnostics.Metrics; +using System.Diagnostics.Tracing; +using System.IO.Pipelines; +using Microsoft.AspNetCore.Connections; + +namespace Orleans.Dissemination.PerformanceHarness; + +internal sealed class Observation : IDisposable +{ + private readonly object _lock = new(); + private readonly MeterListener _meters = new(); + private readonly Dictionary _metrics = []; + private readonly SocketCounters _sockets = new(); + private readonly object _connectionLock = new(); + private readonly Dictionary _connections = []; + private readonly HashSet _drainingConnections = []; + private Func? _blockedConnections; + private int _partitioned; + private long _bytesWritten; + private long _bytesRead; + + public Observation() + { + _meters.InstrumentPublished = (instrument, listener) => + { + if (instrument.Meter.Name.StartsWith("Microsoft.Orleans", StringComparison.Ordinal) + && (instrument.Name is "orleans-messaging-sent-messages-size" or "orleans-messaging-received-messages-size" + || instrument.Name.StartsWith("orleans-dissemination-", StringComparison.Ordinal))) + { + listener.EnableMeasurementEvents(instrument); + } + }; + _meters.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); + _meters.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); + _meters.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); + _meters.Start(); + } + + public bool Partitioned => Volatile.Read(ref _partitioned) != 0; + public long BytesWritten => Interlocked.Read(ref _bytesWritten); + public long BytesRead => Interlocked.Read(ref _bytesRead); + public (double Sent, double Received, long Samples) SocketSnapshot => _sockets.Snapshot(); + + public void Partition() + { + ConnectionLifetime[] connections; + lock (_connectionLock) + { + Volatile.Write(ref _partitioned, 1); + connections = CaptureConnectionsUnsafe(static _ => true); + } + + foreach (var connection in connections) + { + connection.Context.Abort(new ConnectionAbortedException("Harness partition")); + } + } + + public async Task DrainConnections( + Func matches, + Func close, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + ConnectionLifetime[] connections; + lock (_connectionLock) + { + _blockedConnections = matches; + connections = CaptureConnectionsUnsafe(matches); + } + + var closing = new List(connections.Length * 2); + foreach (var connection in connections) + { + if (!connection.Completion.Task.IsCompleted) + { + connection.Context.Abort(new ConnectionAbortedException("Harness partition")); + } + + var closeTask = close(connection.Context); + lock (_connectionLock) + { + connection.CloseTask = closeTask; + } + + closing.Add(closeTask); + closing.Add(connection.Completion.Task); + } + + var drained = Task.WhenAll(closing); + _ = drained.ContinueWith( + static task => _ = task.Exception, + CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + await drained.WaitAsync(cancellationToken); + } + + public void ResumeConnections() + { + lock (_connectionLock) + { + if (_drainingConnections.Any(static connection => + !connection.Completion.Task.IsCompleted || connection.CloseTask is not { IsCompletedSuccessfully: true })) + { + throw new InvalidOperationException("Partitioned connection middleware must drain before reopening."); + } + + _drainingConnections.Clear(); + _blockedConnections = null; + Volatile.Write(ref _partitioned, 0); + } + } + + private ConnectionLifetime[] CaptureConnectionsUnsafe(Func matches) + { + foreach (var connection in _connections.Values) + { + if (matches(connection.Context)) + { + _drainingConnections.Add(connection); + } + } + + return [.. _drainingConnections]; + } + + public async Task Connection(ConnectionContext context, ConnectionDelegate next) + { + ConnectionLifetime? lifetime = null; + lock (_connectionLock) + { + if (!Partitioned && _blockedConnections?.Invoke(context) != true) + { + lifetime = new(context); + _connections.Add(context, lifetime); + } + } + + if (lifetime is null) + { + context.Abort(new ConnectionAbortedException("Harness partition")); + return; + } + + var original = context.Transport; + context.Transport = new CountingDuplexPipe(original, this); + try + { + await next(context); + } + finally + { + context.Transport = original; + lock (_connectionLock) + { + _connections.Remove(context); + lifetime.Completion.SetResult(); + } + } + } + + private sealed class ConnectionLifetime(ConnectionContext context) + { + public ConnectionContext Context { get; } = context; + public TaskCompletionSource Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public Task? CloseTask { get; set; } + } + + public Dictionary Metrics() + { + _meters.RecordObservableInstruments(); + lock (_lock) + { + return new(_metrics); + } + } + + private void Record(Instrument instrument, double value, ReadOnlySpan> tags) + { + var tagValues = new string[tags.Length]; + for (var index = 0; index < tags.Length; index++) + { + tagValues[index] = $"{tags[index].Key}={tags[index].Value}"; + } + + Array.Sort(tagValues, StringComparer.Ordinal); + var key = $"{instrument.Name}|{string.Join(';', tagValues)}"; + lock (_lock) + { + var previous = _metrics.GetValueOrDefault(key, new(0, 0)); + _metrics[key] = instrument.IsObservable + ? new(previous.Count + 1, value) + : new(previous.Count + 1, previous.Sum + value); + } + } + + public void Dispose() + { + _meters.Dispose(); + _sockets.Dispose(); + } + + private sealed class CountingDuplexPipe(IDuplexPipe inner, Observation owner) : IDuplexPipe + { + public PipeReader Input { get; } = new CountingReader(inner.Input, owner); + public PipeWriter Output { get; } = new CountingWriter(inner.Output, owner); + } + + private sealed class CountingWriter(PipeWriter inner, Observation owner) : PipeWriter + { + public override void Advance(int bytes) + { + inner.Advance(bytes); + Interlocked.Add(ref owner._bytesWritten, bytes); + } + + public override Memory GetMemory(int sizeHint = 0) => inner.GetMemory(sizeHint); + public override Span GetSpan(int sizeHint = 0) => inner.GetSpan(sizeHint); + public override void CancelPendingFlush() => inner.CancelPendingFlush(); + public override void Complete(Exception? exception = null) => inner.Complete(exception); + public override ValueTask FlushAsync(CancellationToken cancellationToken = default) => inner.FlushAsync(cancellationToken); + } + + private sealed class CountingReader(PipeReader inner, Observation owner) : PipeReader + { + private ReadOnlySequence _buffer; + + public override void AdvanceTo(SequencePosition consumed) => AdvanceTo(consumed, consumed); + + public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) + { + Interlocked.Add(ref owner._bytesRead, _buffer.Slice(0, consumed).Length); + inner.AdvanceTo(consumed, examined); + _buffer = default; + } + + public override async ValueTask ReadAsync(CancellationToken cancellationToken = default) + { + var result = await inner.ReadAsync(cancellationToken); + _buffer = result.Buffer; + return result; + } + + public override bool TryRead(out ReadResult result) + { + if (!inner.TryRead(out result)) + { + return false; + } + + _buffer = result.Buffer; + return true; + } + + public override void CancelPendingRead() => inner.CancelPendingRead(); + public override void Complete(Exception? exception = null) => inner.Complete(exception); + } + + private sealed class SocketCounters : EventListener + { + private readonly object _lock = new(); + private double _sent; + private double _received; + private long _samples; + + protected override void OnEventSourceCreated(EventSource eventSource) + { + if (eventSource.Name == "System.Net.Sockets") + { + EnableEvents(eventSource, EventLevel.LogAlways, EventKeywords.All, + new Dictionary { ["EventCounterIntervalSec"] = "1" }); + } + } + + protected override void OnEventWritten(EventWrittenEventArgs eventData) + { + if (eventData.EventName != "EventCounters" + || eventData.Payload?.Count != 1 + || eventData.Payload[0] is not IDictionary counter + || !counter.TryGetValue("Name", out var name)) + { + return; + } + + var incremental = counter.TryGetValue("Increment", out var measurement); + if (!incremental && !counter.TryGetValue("Mean", out measurement)) + { + return; + } + + lock (_lock) + { + var value = Convert.ToDouble(measurement, System.Globalization.CultureInfo.InvariantCulture); + if ((string)name == "bytes-sent") + { + _sent = incremental ? _sent + value : value; + _samples++; + } + else if ((string)name == "bytes-received") + { + _received = incremental ? _received + value : value; + } + } + } + + public (double Sent, double Received, long Samples) Snapshot() + { + lock (_lock) + { + return (_sent, _received, _samples); + } + } + } +} diff --git a/test/Dissemination.PerformanceHarness/Silo/Program.cs b/test/Dissemination.PerformanceHarness/Silo/Program.cs new file mode 100644 index 00000000000..e320f1f9185 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Silo/Program.cs @@ -0,0 +1,325 @@ +using System.Diagnostics; +using System.Net; +using System.Reflection; +using System.Security.Cryptography; +using System.Text.Json; +using Microsoft.AspNetCore.Connections; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.MembershipService; +using Orleans.Runtime.Messaging; +using Orleans.Runtime.Scheduler; +using Orleans.TestingHost; + +namespace Orleans.Dissemination.PerformanceHarness; + +internal static class Program +{ + private static readonly MethodInfo PublishMethod = typeof(DeploymentLoadPublisher).GetMethod( + "PublishStatistics", BindingFlags.Instance | BindingFlags.NonPublic)!; + + public static async Task Main(string[] args) + { + if (args is not ["--silo", var configurationPath]) + { + Console.Error.WriteLine("Expected --silo ."); + return 2; + } + + var configuration = JsonSerializer.Deserialize(await File.ReadAllTextAsync(configurationPath))!; +#if !NEW_RUNTIME + if (configuration.Enabled) + { + throw new InvalidOperationException("The pinned old runtime cannot be opted into dissemination."); + } +#endif + using var lifetime = new CancellationTokenSource(TimeSpan.FromMinutes(45)); + using var observation = new Observation(); + var membership = new FileMembershipTable(configuration); + using var host = TestClusterHostFactory.CreateSiloHost( + configuration.Name, + new ConfigurationBuilder().Build(), + builder => builder.ConfigureServices(services => + { + services.AddSingleton(configuration); + services.AddSingleton(observation); + services.AddSingleton(membership); + services.AddSingleton(membership); + services.AddSingleton(); + services.Configure(options => + { + options.ClusterId = configuration.ClusterId; + options.ServiceId = "dissemination-performance"; + }); + services.Configure(options => options.SiloName = configuration.Name); + services.Configure(options => + { + options.AdvertisedIPAddress = IPAddress.Loopback; + options.SiloPort = configuration.SiloPort; + options.GatewayPort = 0; + }); + services.Configure(options => + { + // Keep membership stable while the transport is partitioned. + options.LivenessEnabled = false; + options.TableRefreshTimeout = TimeSpan.FromHours(1); + options.IAmAliveTablePublishTimeout = TimeSpan.FromHours(1); + options.MaxJoinAttemptTime = TimeSpan.FromSeconds(60); + }); + services.Configure(options => + options.DeploymentLoadPublisherRefreshTime = TimeSpan.FromHours(1)); + services.Configure(options => options.ResponseTimeout = TimeSpan.FromSeconds(5)); + services.Configure(options => options.ShutdownTimeout = TimeSpan.FromSeconds(15)); + services.Configure(options => + { + options.ConfigureSiloInboundConnection(connection => connection.Use(next => context => observation.Connection(context, next))); + options.ConfigureSiloOutboundConnection(connection => connection.Use(next => context => observation.Connection(context, next))); + }); + services.AddLogging(logging => + { + logging.SetMinimumLevel(LogLevel.Warning).AddConsole(options => + options.LogToStandardErrorThreshold = LogLevel.Trace); + }); +#if NEW_RUNTIME + NewRuntime.Configure(services, configuration); +#endif + })); + + _ = host.Services.GetRequiredService(); + var monitor = MonitorParent(configuration.ParentProcessId, lifetime); + try + { + using var startup = CancellationTokenSource.CreateLinkedTokenSource(lifetime.Token); + startup.CancelAfter(TimeSpan.FromSeconds(90)); + await host.StartAsync(startup.Token); + var publisher = host.Services.GetRequiredService(); + // Commands own publication cadence; lifecycle shutdown retains its disposable timer reference. + await publisher.RunOrQueueTask(() => + { + var field = typeof(DeploymentLoadPublisher).GetField("_publishTimer", BindingFlags.Instance | BindingFlags.NonPublic)!; + ((IDisposable)field.GetValue(publisher)!).Dispose(); + return Task.CompletedTask; + }); + await Write(new Response(0, Snapshot(host.Services), null)); + + while (await Console.In.ReadLineAsync(lifetime.Token) is { } line) + { + var command = JsonSerializer.Deserialize(line)!; + using var operation = CancellationTokenSource.CreateLinkedTokenSource(lifetime.Token); + operation.CancelAfter(TimeSpan.FromSeconds(60)); + try + { + var result = await Execute(host.Services, command, operation.Token); + await Write(new Response(command.Id, result, null)); + if (command.Operation == "stop") + { + break; + } + } + catch (Exception exception) + { + await Write(new Response(command.Id, null, exception.ToString())); + } + } + + using var shutdown = new CancellationTokenSource(TimeSpan.FromSeconds(20)); + await host.StopAsync(shutdown.Token).WaitAsync(shutdown.Token); + // Drain the last one-second OS socket-counter bucket before recording shutdown cost. + await Task.Delay(TimeSpan.FromMilliseconds(1100)); + await Write(new Response(-1, Snapshot(host.Services, detailed: true), null)); + return 0; + } + finally + { + await lifetime.CancelAsync(); + await monitor; + } + } + + private static async Task Execute(IServiceProvider services, Command command, CancellationToken cancellationToken) + { + var manager = services.GetRequiredService(); + var publisher = services.GetRequiredService(); + switch (command.Operation) + { + case "snapshot": + case "stop": + break; + case "measure": + return Snapshot(services, detailed: true); + case "refresh": + await manager.Refresh(null, cancellationToken); + break; + case "publish": + await publisher.RunOrQueueTask(() => (Task)PublishMethod.Invoke(publisher, [cancellationToken])!) + .WaitAsync(cancellationToken); + + break; + case "partition": + services.GetRequiredService().Partition(); + break; + case "drain-connections": + await DrainConnections(services, command.Peer, cancellationToken); + break; + case "resume-connections": + services.GetRequiredService().ResumeConnections(); + break; + case "retained-memory": + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + break; + case "echo": + { + var peer = services.GetRequiredService().GetSystemTarget( + ControlTarget.TargetType, SiloAddress.FromParsableString(command.Peer!)); + var processId = await peer.Echo(cancellationToken).WaitAsync(cancellationToken); + return Snapshot(services) with { RemoteProcessId = processId }; + } + case "probe-echo": + { + var peer = services.GetRequiredService().GetSystemTarget( + ControlTarget.TargetType, SiloAddress.FromParsableString(command.Peer!)); + using var probeCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + try + { + var request = peer.Echo(probeCancellation.Token); + request.Ignore(); + var processId = await request.WaitAsync(TimeSpan.FromSeconds(2), cancellationToken); + return Snapshot(services) with { RemoteProcessId = processId }; + } + catch (Exception exception) when (exception is ConnectionFailedException + or SiloUnavailableException or OrleansMessageRejectionException or TimeoutException) + { + return Snapshot(services) with + { + ProbeError = $"{exception.GetType().FullName}: {exception.Message}", + }; + } + finally + { + await probeCancellation.CancelAsync(); + } + } + default: + throw new NotSupportedException(command.Operation); + } + + return Snapshot(services); + } + + private static Task DrainConnections(IServiceProvider services, string? peer, CancellationToken cancellationToken) + { + var address = peer is null ? null : SiloAddress.FromParsableString(peer); + return services.GetRequiredService().DrainConnections( + context => + { + var connection = (SiloConnection)context.Features.Get()!; + // Inbound handshakes have no remote silo identity until their preamble completes. + return address is null || connection.RemoteSiloAddress is null || connection.RemoteSiloAddress.Equals(address); + }, + context => context.Features.Get()!.CloseAsync(new ConnectionAbortedException("Harness partition")), + cancellationToken); + } + + internal static NodeSnapshot Snapshot(IServiceProvider services, bool detailed = false) + { + var manager = services.GetRequiredService(); + var publisher = services.GetRequiredService(); + var observation = services.GetRequiredService(); + var membership = manager.CurrentSnapshot; + using var process = Process.GetCurrentProcess(); + var socket = observation.SocketSnapshot; + var result = new NodeSnapshot + { + CapturedAtUtc = DateTimeOffset.UtcNow, + Identity = Identity(), + Address = services.GetRequiredService().SiloAddress.ToParsableString(), + MembershipVersion = membership.Version.Value, + Membership = new(membership.Entries.ToDictionary( + pair => pair.Key.ToParsableString(), + pair => JsonSerializer.Serialize(FileMembershipTable.EntryData.From(pair.Value))), StringComparer.Ordinal), + ActiveMembers = membership.Entries.Values.Where(entry => entry.Status == SiloStatus.Active) + .Select(entry => entry.SiloAddress.ToParsableString()).Order(StringComparer.Ordinal).ToArray(), + Load = new(publisher.PeriodicStatistics.ToDictionary( + pair => pair.Key.ToParsableString(), pair => StateComparison.Serialize(pair.Value)), StringComparer.Ordinal), + LoadVersions = new(publisher.PeriodicStatistics.ToDictionary( + pair => pair.Key.ToParsableString(), pair => pair.Value.DateTime.Ticks), StringComparer.Ordinal), + UnconfirmedPeers = [], + OriginatorTargets = [], + ForwardingTargets = [], + Metrics = detailed ? observation.Metrics() : [], + TransportBytesWritten = observation.BytesWritten, + TransportBytesRead = observation.BytesRead, + SocketBytesSent = socket.Sent, + SocketBytesReceived = socket.Received, + SocketCounterSamples = socket.Samples, + CpuMilliseconds = process.TotalProcessorTime.TotalMilliseconds, + AllocatedBytes = GC.GetTotalAllocatedBytes(precise: true), + ManagedHeapBytes = GC.GetTotalMemory(forceFullCollection: false), + WorkingSetBytes = process.WorkingSet64, + PrivateBytes = process.PrivateMemorySize64, + ProcessorCount = Environment.ProcessorCount, + ServerGC = System.Runtime.GCSettings.IsServerGC, + Partitioned = observation.Partitioned, + }; +#if NEW_RUNTIME + result = NewRuntime.Decorate(services, result); +#endif + return result; + } + + private static BinaryIdentity Identity() + { + // Hash once: periodic snapshots must not read the runtime image repeatedly. + return CachedIdentity.Value; + } + + private static readonly Lazy CachedIdentity = new(() => + { + var assembly = typeof(Silo).Assembly; + var metadata = typeof(Program).Assembly.GetCustomAttributes() + .ToDictionary(attribute => attribute.Key, attribute => attribute.Value); + return new( + Environment.ProcessId, metadata["HarnessRuntime"]!, metadata["HarnessSourceRevision"]!, + assembly.GetName().Name!, assembly.GetName().Version!.ToString(), + assembly.GetCustomAttribute()!.InformationalVersion, + assembly.ManifestModule.ModuleVersionId.ToString(), + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(assembly.Location))), + assembly.Location, + assembly.GetType("Orleans.Runtime.Dissemination.DisseminationSystemTarget") is not null) + { + Assemblies = AppDomain.CurrentDomain.GetAssemblies() + .Where(candidate => !candidate.IsDynamic && candidate.GetName().Name!.StartsWith("Orleans", StringComparison.Ordinal)) + .ToDictionary(candidate => candidate.GetName().Name!, candidate => new AssemblyProof( + candidate.GetName().Version!.ToString(), + candidate.GetCustomAttribute()?.InformationalVersion ?? "", + candidate.ManifestModule.ModuleVersionId.ToString(), + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(candidate.Location))), + candidate.Location)), + }; + }); + + private static async Task MonitorParent(int parentProcessId, CancellationTokenSource lifetime) + { + try + { + using var parent = Process.GetProcessById(parentProcessId); + await parent.WaitForExitAsync(lifetime.Token); + await lifetime.CancelAsync(); + } + catch (ArgumentException) + { + await lifetime.CancelAsync(); + } + catch (OperationCanceledException) when (lifetime.IsCancellationRequested) + { + } + } + + private static Task Write(Response response) => Console.Out.WriteLineAsync("HARNESS " + JsonSerializer.Serialize(response)); +} diff --git a/test/Dissemination.PerformanceHarness/Tests/Dissemination.PerformanceHarness.Tests.csproj b/test/Dissemination.PerformanceHarness/Tests/Dissemination.PerformanceHarness.Tests.csproj new file mode 100644 index 00000000000..39ef47e709d --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Tests/Dissemination.PerformanceHarness.Tests.csproj @@ -0,0 +1,12 @@ + + + net10.0 + enable + + + + + + + + diff --git a/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.cs b/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.cs new file mode 100644 index 00000000000..b0e1aa0e532 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.cs @@ -0,0 +1,349 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.IO.Pipelines; +using System.Net; +using System.Net.Sockets; +using System.Text.Json; +using Microsoft.AspNetCore.Connections; +using Xunit; + +namespace Orleans.Dissemination.PerformanceHarness; + +public sealed class MeasurementTests +{ + [Theory] + [InlineData(2)] + [InlineData(3)] + public void PairedRepetitionsReverseEveryRuntimePath(int count) + { + var forward = Enumerable.Range(0, count).Select(order => ScalingTests.ExecutionIndex(0, order, count)).ToArray(); + var reverse = Enumerable.Range(0, count).Select(order => ScalingTests.ExecutionIndex(1, order, count)).ToArray(); + var repeated = Enumerable.Range(0, count).Select(order => ScalingTests.ExecutionIndex(2, order, count)).ToArray(); + Assert.Equal(Enumerable.Range(0, count), forward); + Assert.Equal(forward.Reverse(), reverse); + Assert.Equal(forward, repeated); + } + + [Fact] + public void ScalingComparisonNormalizesEqualOfferedWorkAndReportsLatencyDistribution() + { + var environment = JsonSerializer.SerializeToElement(new { ProcessorCount = 4, SiloProcessorCount = 1, GCConserveMemory = 9 }); + var disabled = new ScalingSample( + "CurrentDisabled", 100, 100, "stable", 0, 4, 400, + new("candidate", "same-binary"), environment, + 39600, 800000, 4000000, 2000, 1000000, [40, 10, 30, 20]); + var enabled = disabled with + { + RuntimePath = "CurrentEnabledSupported", + TotalRpcs = 4000, + SerializedBytesSent = 400000, + AllocatedBytes = 2000000, + CpuMilliseconds = 1000, + ConvergenceMilliseconds = [30, 10, 20, 10], + }; + + var result = ScalingComparison.Create(JsonSerializer.Serialize(new[] { enabled, disabled })); + + Assert.Equal(0, result.IncompletePairs); + var pair = Assert.Single(result.Pairs); + Assert.Equal(100, pair.Size); + Assert.Equal(400, pair.OfferedPublications); + Assert.Equal(99, pair.Disabled.RpcsPerPublication); + Assert.Equal(10, pair.Enabled.RpcsPerPublication); + Assert.Equal(2000, pair.Disabled.SerializedBytesPerPublication); + Assert.Equal(1000, pair.Enabled.SerializedBytesPerPublication); + Assert.Equal(10000, pair.Disabled.AllocatedBytesPerPublication); + Assert.Equal(5000, pair.Enabled.AllocatedBytesPerPublication); + Assert.Equal(5, pair.Disabled.CpuMillisecondsPerPublication); + Assert.Equal(2.5, pair.Enabled.CpuMillisecondsPerPublication); + Assert.Equal(25, pair.Disabled.MedianConvergenceMilliseconds); + Assert.Equal(15, pair.Enabled.MedianConvergenceMilliseconds); + Assert.Equal(40, pair.Disabled.P95ConvergenceMilliseconds); + Assert.Equal(30, pair.Enabled.P95ConvergenceMilliseconds); + Assert.Contains("99.00 -> 10.00", result.ToMarkdown(), StringComparison.Ordinal); + } + + [Theory] + [InlineData("binary")] + [InlineData("workload")] + [InlineData("environment")] + [InlineData("silo-count")] + public void ScalingComparisonRejectsIncomparableMeasurements(string mismatch) + { + var sample = new ScalingSample( + "CurrentDisabled", 100, 100, "stable", 0, 3, 300, + new("candidate", "same-binary"), JsonSerializer.SerializeToElement(new { ProcessorCount = 4 }), + 100, 1000, 10000, 100, 1000, [1, 2, 3]); + var enabled = sample with { RuntimePath = "CurrentEnabledSupported" }; + enabled = mismatch switch + { + "binary" => enabled with { Runtime = new("different", "other-binary") }, + "workload" => enabled with { OfferedPublications = 301 }, + "environment" => enabled with { Environment = JsonSerializer.SerializeToElement(new { ProcessorCount = 1 }) }, + "silo-count" => enabled with { LiveSilos = 99 }, + _ => throw new InvalidOperationException(mismatch), + }; + + Assert.Throws(() => ScalingComparison.Create(JsonSerializer.Serialize(new[] { sample, enabled }))); + } + + [Fact] + public void ScalingComparisonReportsPartialPairsExplicitly() + { + var sample = new ScalingSample( + "CurrentDisabled", 8, 8, "stable", 0, 3, 24, + new("candidate", "same-binary"), JsonSerializer.SerializeToElement(new { ProcessorCount = 4 }), + 100, 1000, 10000, 100, 1000, [1, 2, 3]); + + var result = ScalingComparison.Create(JsonSerializer.Serialize(new[] { sample })); + + Assert.Empty(result.Pairs); + Assert.Equal(1, result.IncompletePairs); + } + + [Fact] + public void LoadStateComparisonRequiresExactInventoryAndValues() + { + var expected = new Dictionary + { + ["silo@2"] = "current", + ["peer@1"] = "peer-value", + }; + var actual = new Dictionary(expected); + Assert.True(ProcessCluster.MatchesExpectedLoad(actual, expected)); + + actual["silo@1"] = "retired"; + Assert.False(ProcessCluster.MatchesExpectedLoad(actual, expected)); + actual.Remove("silo@2"); + Assert.False(ProcessCluster.MatchesExpectedLoad(actual, expected)); + actual.Remove("silo@1"); + actual["silo@2"] = "older-value"; + Assert.False(ProcessCluster.MatchesExpectedLoad(actual, expected)); + actual["silo@2"] = "current"; + Assert.True(ProcessCluster.MatchesExpectedLoad(actual, expected)); + } + + [Fact] + public async Task ConnectionDrain_WaitsForInitializationAndCloseBeforeReopening() + { + using var observation = new Observation(); + await using var context = CreateConnectionContext("initializing"); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var closed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var middleware = observation.Connection(context, async _ => + { + entered.SetResult(); + await release.Task; + }); + await entered.Task.WaitAsync(TestContext.Current.CancellationToken); + observation.Partition(); + Assert.True(context.ConnectionClosed.IsCancellationRequested); + var closeCalls = 0; + var drain = observation.DrainConnections( + static _ => true, + connection => + { + Assert.Same(context, connection); + closeCalls++; + return closed.Task; + }, + TestContext.Current.CancellationToken); + try + { + Assert.Equal(1, closeCalls); + Assert.False(drain.IsCompleted); + Assert.Throws(observation.ResumeConnections); + await using var rejected = CreateConnectionContext("rejected"); + await observation.Connection(rejected, _ => throw new InvalidOperationException("A partitioned connection entered.")); + release.SetResult(); + await middleware; + Assert.False(drain.IsCompleted); + Assert.Throws(observation.ResumeConnections); + closed.SetResult(); + await drain.WaitAsync(TestContext.Current.CancellationToken); + observation.ResumeConnections(); + Assert.False(observation.Partitioned); + var resumed = false; + await using var reopened = CreateConnectionContext("reopened"); + await observation.Connection(reopened, _ => + { + resumed = true; + return Task.CompletedTask; + }); + Assert.True(resumed); + } + finally + { + release.TrySetResult(); + closed.TrySetResult(); + await middleware; + await drain; + } + } + + [Fact] + public async Task ConnectionDrain_CapturesUnidentifiedPeersAndPreservesHealthyConnections() + { + using var observation = new Observation(); + await using var unknown = CreateConnectionContext("unknown"); + await using var healthy = CreateConnectionContext("healthy"); + healthy.Items["peer"] = "healthy"; + var releaseUnknown = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseHealthy = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var unknownMiddleware = observation.Connection(unknown, _ => releaseUnknown.Task); + var healthyMiddleware = observation.Connection(healthy, _ => releaseHealthy.Task); + using var cancellation = new CancellationTokenSource(); + var closed = new List(); + var drain = observation.DrainConnections( + context => !context.Items.TryGetValue("peer", out var peer) || Equals(peer, "partitioned"), + context => + { + closed.Add(context.ConnectionId); + return Task.CompletedTask; + }, + cancellation.Token); + try + { + Assert.Equal(new[] { "unknown" }, closed); + Assert.True(unknown.ConnectionClosed.IsCancellationRequested); + Assert.False(healthy.ConnectionClosed.IsCancellationRequested); + Assert.False(healthyMiddleware.IsCompleted); + await cancellation.CancelAsync(); + await Assert.ThrowsAnyAsync(() => drain); + await using var rejected = CreateConnectionContext("new-unknown"); + await observation.Connection(rejected, _ => throw new InvalidOperationException("An unidentified connection entered.")); + releaseUnknown.SetResult(); + await unknownMiddleware; + observation.ResumeConnections(); + Assert.False(healthyMiddleware.IsCompleted); + } + finally + { + releaseUnknown.TrySetResult(); + releaseHealthy.TrySetResult(); + await Task.WhenAll(unknownMiddleware, healthyMiddleware); + } + } + + [Fact] + public void StateComparison_ObservesNestedPublicReadonlyFieldChanges() + { + var original = new { Version = 7, EnvironmentStatistics = new ReadonlyStatistics(20) }; + var changed = new { Version = 7, EnvironmentStatistics = new ReadonlyStatistics(35) }; + // This is the previous comparison's blind spot: properties alone produce identical JSON. + Assert.Equal(JsonSerializer.Serialize(original), JsonSerializer.Serialize(changed)); + var before = StateComparison.Serialize(original); + var after = StateComparison.Serialize(changed); + Assert.NotEqual(before, after); + using var beforeDocument = JsonDocument.Parse(before); + using var afterDocument = JsonDocument.Parse(after); + Assert.Equal(20, beforeDocument.RootElement.GetProperty("EnvironmentStatistics").GetProperty("RawCpuUsagePercentage").GetSingle()); + Assert.Equal(35, afterDocument.RootElement.GetProperty("EnvironmentStatistics").GetProperty("RawCpuUsagePercentage").GetSingle()); + } + + [Fact] + public void MessageObservation_KeepsRequestOneWayAndBytesDistinct() + { + using var observation = new Observation(); + using var meter = new Meter("Microsoft.Orleans.HarnessMeasurementTest"); + var histogram = meter.CreateHistogram("orleans-messaging-sent-messages-size"); + histogram.Record(97, new KeyValuePair("MessageDirection", "Request")); + histogram.Record(31, new KeyValuePair("MessageDirection", "OneWay")); + histogram.Record(17, new KeyValuePair("MessageDirection", "Response")); + var metrics = observation.Metrics(); + Assert.Equal(new MetricValue(1, 97), metrics["orleans-messaging-sent-messages-size|MessageDirection=Request"]); + Assert.Equal(new MetricValue(1, 31), metrics["orleans-messaging-sent-messages-size|MessageDirection=OneWay"]); + Assert.Equal(new MetricValue(1, 17), metrics["orleans-messaging-sent-messages-size|MessageDirection=Response"]); + } + + [Fact] + public async Task TransportObservation_CountsConsumedAndSubmittedBytes() + { + var cancellationToken = TestContext.Current.CancellationToken; + using var observation = new Observation(); + var input = new Pipe(); + var output = new Pipe(); + await using var context = new DefaultConnectionContext("instrumentation-test") + { + Transport = new TestPipe(input.Reader, output.Writer), + }; + var payload = new byte[] { 1, 2, 3, 4, 5 }; + await observation.Connection(context, async connection => + { + await input.Writer.WriteAsync(payload, cancellationToken); + var received = await connection.Transport.Input.ReadAsync(cancellationToken); + connection.Transport.Input.AdvanceTo(received.Buffer.End); + await connection.Transport.Output.WriteAsync(payload, cancellationToken); + }); + Assert.Equal(payload.Length, observation.BytesRead); + Assert.Equal(payload.Length, observation.BytesWritten); + await input.Reader.CompleteAsync(); + await input.Writer.CompleteAsync(); + await output.Reader.CompleteAsync(); + await output.Writer.CompleteAsync(); + } + + [Fact] + public async Task SocketObservation_ReportsActualLoopbackTransfer() + { + var cancellationToken = TestContext.Current.CancellationToken; + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + using var observation = new Observation(); + using var client = new TcpClient(); + var accepted = listener.AcceptTcpClientAsync(cancellationToken); + await client.ConnectAsync(IPAddress.Loopback, ((IPEndPoint)listener.LocalEndpoint).Port, cancellationToken); + using var server = await accepted; + await WaitFor(() => observation.SocketSnapshot.Samples > 0, cancellationToken); + var before = observation.SocketSnapshot; + var payload = new byte[4096]; + await client.GetStream().WriteAsync(payload, cancellationToken); + await server.GetStream().ReadExactlyAsync(payload, cancellationToken); + await WaitFor(() => + { + var snapshot = observation.SocketSnapshot; + return snapshot.Sent - before.Sent >= payload.Length && snapshot.Received - before.Received >= payload.Length; + }, cancellationToken); + } + + private static async Task WaitFor(Func predicate, CancellationToken cancellationToken) + { + var clock = Stopwatch.StartNew(); + while (!predicate()) + { + Assert.True(clock.Elapsed < TimeSpan.FromSeconds(8), "The OS socket EventCounters did not report the measured transfer."); + await Task.Delay(50, cancellationToken); + } + } + + private sealed record TestPipe(PipeReader Input, PipeWriter Output) : IDuplexPipe; + + private static DefaultConnectionContext CreateConnectionContext(string id) + { + var pipe = new Pipe(); + return new TestConnectionContext(id) { Transport = new TestPipe(pipe.Reader, pipe.Writer) }; + } + + // DefaultConnectionContext queues Cancel independently of Dispose. These tests own abort synchronously; + // separate barriers continue to control middleware and transport completion. + private sealed class TestConnectionContext : DefaultConnectionContext + { + private readonly CancellationTokenSource _closed = new(); + + public TestConnectionContext(string id) : base(id) => ConnectionClosed = _closed.Token; + + public override void Abort(ConnectionAbortedException abortReason) => _closed.Cancel(); + + public override ValueTask DisposeAsync() + { + _closed.Dispose(); + return base.DisposeAsync(); + } + } + + private readonly struct ReadonlyStatistics(float rawCpuUsagePercentage) + { + public readonly float RawCpuUsagePercentage = rawCpuUsagePercentage; + } +} diff --git a/test/Dissemination.PerformanceHarness/Tests/ProcessCluster.cs b/test/Dissemination.PerformanceHarness/Tests/ProcessCluster.cs new file mode 100644 index 00000000000..0ac07f17c4d --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Tests/ProcessCluster.cs @@ -0,0 +1,457 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Net; +using System.Net.Sockets; +using System.Text.Json; +using Xunit; + +namespace Orleans.Dissemination.PerformanceHarness; + +internal sealed record AssemblyManifest(string AssemblyVersion, string Sha256); +internal sealed record BuildManifest( + string Runtime, + string SourceRevision, + string AssemblyName, + string AssemblyVersion, + string Sha256, + Dictionary Assemblies); + +internal sealed class ProcessCluster : IAsyncDisposable +{ + public const string Baseline = "6739589254b746a8790cf53524e6abe372bb53d4"; + private readonly List _all = []; + private readonly string _membershipDirectory; + private int _sequence; + + public ProcessCluster(string scenario, int siloProcessorCount = 0, int gcConserveMemory = 0) + { + var artifactRoot = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_RESULTS") + ?? throw new InvalidOperationException("Run .github/scripts/dissemination-performance.ps1 first."); + Directory = Path.Combine(Path.GetFullPath(artifactRoot), $"{scenario}-{Guid.NewGuid():N}"); + System.IO.Directory.CreateDirectory(Directory); + _membershipDirectory = Path.Combine(Directory, "membership"); + System.IO.Directory.CreateDirectory(_membershipDirectory); + Assert.InRange(siloProcessorCount, 0, 64); + Assert.InRange(gcConserveMemory, 0, 9); + SiloProcessorCount = siloProcessorCount; + GCConserveMemory = gcConserveMemory; + } + + public string Directory { get; } + public int SiloProcessorCount { get; } + public int GCConserveMemory { get; } + public IReadOnlyList All => _all; + public SiloProcess[] Active => _all.Where(node => !node.Stopped).ToArray(); + public double LastPublicationMilliseconds { get; private set; } + public double LastConvergenceWaitMilliseconds { get; private set; } + + public async Task Start(string runtime, bool enabled, int? port = null) + { + var binaryDirectory = Path.GetFullPath(Environment.GetEnvironmentVariable($"ORLEANS_DISSEMINATION_{runtime.ToUpperInvariant()}") + ?? throw new InvalidOperationException($"Missing {runtime} published binary directory.")); + var manifest = JsonSerializer.Deserialize(await File.ReadAllTextAsync(Path.Combine(binaryDirectory, "manifest.json")))!; + Assert.Equal(runtime, manifest.Runtime); + if (runtime == "Old") + { + Assert.Equal(Baseline, manifest.SourceRevision); + } + else + { + Assert.Equal(Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_CANDIDATE_SHA"), manifest.SourceRevision); + Assert.NotEqual(Baseline, manifest.SourceRevision); + } + + var name = $"{runtime}-{_sequence++}-{(enabled ? "enabled" : "default")}"; + var node = new SiloProcess( + binaryDirectory, + new(name, Path.GetFileName(Directory), _membershipDirectory, port ?? AvailablePort(), Environment.ProcessId, + enabled, SiloProcessorCount, GCConserveMemory), + Directory, + manifest); + _all.Add(node); + await node.Start(); + Assert.DoesNotContain(_all.Where(other => !ReferenceEquals(other, node) && !other.Stopped), + other => other.Last.Identity.ProcessId == node.Last.Identity.ProcessId); + return node; + } + + public async Task Stabilize() + { + var expected = Active.Select(node => node.Last.Address).Order(StringComparer.Ordinal).ToArray(); + await Eventually("every process has the exact active membership", async () => + { + var snapshots = await Task.WhenAll(Active.Select(node => node.Send("refresh"))); + return snapshots.All(snapshot => snapshot.ActiveMembers.SequenceEqual(expected)); + }); + } + + public async Task AssertControlRpcs(bool allPairs = true) + { + var nodes = Active; + for (var index = 0; index < nodes.Length; index++) + { + var peers = allPairs ? nodes.Where(peer => !ReferenceEquals(peer, nodes[index])) : [nodes[(index + 1) % nodes.Length]]; + foreach (var peer in peers) + { + var result = await nodes[index].Send("echo", peer: peer.Last.Address); + Assert.Equal(peer.Last.Identity.ProcessId, result.RemoteProcessId); + Assert.NotEqual(nodes[index].Last.Identity.ProcessId, result.RemoteProcessId); + } + } + } + + public async Task PublishAndConverge() + { + var clock = Stopwatch.StartNew(); + var expected = new Dictionary(StringComparer.Ordinal); + foreach (var node in Active) + { + var previous = node.Last.LoadVersions.GetValueOrDefault(node.Last.Address); + var snapshot = await node.Send("publish"); + Assert.True(snapshot.LoadVersions[snapshot.Address] > previous, "The production publisher did not create a new sample."); + expected.Add(snapshot.Address, snapshot.Load[snapshot.Address]); + } + + var publicationMilliseconds = clock.Elapsed.TotalMilliseconds; + await Eventually("exact production load state at every active process", async () => + { + var snapshots = await Task.WhenAll(Active.Select(node => node.Send("snapshot"))); + return snapshots.All(snapshot => MatchesExpectedLoad(snapshot.Load, expected)); + }, expected: expected); + var elapsed = clock.Elapsed.TotalMilliseconds; + LastPublicationMilliseconds = publicationMilliseconds; + LastConvergenceWaitMilliseconds = elapsed - publicationMilliseconds; + return elapsed; + } + + internal static bool MatchesExpectedLoad( + IReadOnlyDictionary actual, + IReadOnlyDictionary expected) => + actual.Count == expected.Count + && expected.All(pair => actual.TryGetValue(pair.Key, out var value) + && StringComparer.Ordinal.Equals(pair.Value, value)); + + public async Task HealPartition(SiloProcess partitioned) + { + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(60)); + var cancellationToken = deadline.Token; + var phase = "retiring partitioned connections at both endpoints"; + try + { + var nodes = Active; + // Drain admitted middleware, including unfinished handshakes, before reopening any affected endpoint. + await Task.WhenAll(nodes.Select(node => node.Send( + cancellationToken, + "drain-connections", + peer: ReferenceEquals(node, partitioned) ? null : partitioned.Last.Address))); + await Task.WhenAll(nodes.Select(node => node.Send(cancellationToken, "resume-connections"))); + phase = "bidirectional control RPCs after transport healing"; + await Eventually(phase, async () => + { + foreach (var node in nodes) + { + foreach (var peer in nodes.Where(peer => !ReferenceEquals(peer, node))) + { + var result = await node.Send(cancellationToken, "probe-echo", peer: peer.Last.Address); + if (result.RemoteProcessId is not { } processId) + { + Assert.NotNull(result.ProbeError); + return false; + } + + Assert.Equal(peer.Last.Identity.ProcessId, processId); + Assert.NotEqual(node.Last.Identity.ProcessId, processId); + } + } + + return true; + }); + } + catch (OperationCanceledException) when (deadline.IsCancellationRequested && !TestContext.Current.CancellationToken.IsCancellationRequested) + { + await Save("failure-state.json", new { Phase = phase, Nodes = _all.Select(node => node.Last) }); + Assert.Fail($"Timed out during {phase}. Exact snapshots/logs: {Directory}"); + } + } + + public async Task Eventually(string phase, Func> assertion, TimeSpan? timeout = null, object? expected = null) + { + var clock = Stopwatch.StartNew(); + while (true) + { + if (await assertion()) + { + return; + } + + if (clock.Elapsed > (timeout ?? TimeSpan.FromSeconds(60))) + { + await Save("failure-state.json", new { Phase = phase, Expected = expected, Nodes = _all.Select(node => node.Last) }); + Assert.Fail($"Timed out during {phase}. Exact snapshots/logs: {Directory}"); + } + + await Task.Delay(100, TestContext.Current.CancellationToken); + } + } + + public Task Save(string filename, object value) => + File.WriteAllTextAsync(Path.Combine(Directory, filename), JsonSerializer.Serialize(value, new JsonSerializerOptions { WriteIndented = true })); + + public async ValueTask DisposeAsync() + { + List failures = []; + foreach (var node in _all.AsEnumerable().Reverse()) + { + try + { + await node.DisposeAsync(); + } + catch (Exception exception) + { + failures.Add(exception); + } + } + + await Save("final-processes.json", _all.Select(node => new { node.Configuration, node.Last, node.ShutdownMilliseconds, node.ForcedKill })); + if (failures.Count > 0) + { + throw new AggregateException("One or more silos exceeded shutdown bounds.", failures); + } + } + + private static int AvailablePort() + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + try + { + return ((IPEndPoint)listener.LocalEndpoint).Port; + } + finally + { + listener.Stop(); + } + } +} + +internal sealed class SiloProcess : IAsyncDisposable +{ + private readonly Process _process; + private readonly string _configurationPath; + private readonly string _logPath; + private readonly string _errorPath; + private readonly BuildManifest _manifest; + private readonly ConcurrentDictionary> _pending = new(); + private readonly SemaphoreSlim _commandLock = new(1); + private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously); + private Task _output = Task.CompletedTask; + private Task _error = Task.CompletedTask; + private int _commandId; + private bool _started; + private bool _disposed; + + public SiloProcess(string binaryDirectory, NodeConfiguration configuration, string artifacts, BuildManifest manifest) + { + Configuration = configuration; + _manifest = manifest; + _configurationPath = Path.Combine(artifacts, configuration.Name + ".configuration.json"); + _logPath = Path.Combine(artifacts, configuration.Name + ".stdout.jsonl"); + _errorPath = Path.Combine(artifacts, configuration.Name + ".stderr.log"); + _process = new Process + { + StartInfo = new ProcessStartInfo(Environment.GetEnvironmentVariable("DOTNET_HOST_PATH") ?? "dotnet") + { + WorkingDirectory = binaryDirectory, + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + ArgumentList = { Path.Combine(binaryDirectory, "Orleans.Runtime.Tests.dll"), "--silo", _configurationPath }, + }, + }; + _pending[0] = _ready; + if (configuration.SiloProcessorCount > 0) + { + _process.StartInfo.Environment["DOTNET_PROCESSOR_COUNT"] = + configuration.SiloProcessorCount.ToString(System.Globalization.CultureInfo.InvariantCulture); + } + + if (configuration.GCConserveMemory > 0) + { + _process.StartInfo.Environment["DOTNET_GCConserveMemory"] = + configuration.GCConserveMemory.ToString(System.Globalization.CultureInfo.InvariantCulture); + } + } + + public NodeConfiguration Configuration { get; } + public NodeSnapshot Last { get; private set; } = null!; + public NodeSnapshot Initial { get; private set; } = null!; + public bool Stopped { get; private set; } + public bool ForcedKill { get; private set; } + public double ShutdownMilliseconds { get; private set; } + + public async Task Start() + { + await File.WriteAllTextAsync(_configurationPath, JsonSerializer.Serialize(Configuration)); + Assert.True(_process.Start(), "Failed to launch silo OS process."); + _started = true; + _output = PumpOutput(); + _error = PumpErrors(); + var ready = await _ready.Task.WaitAsync(TimeSpan.FromSeconds(100), TestContext.Current.CancellationToken); + Assert.Null(ready.Error); + Last = ready.Snapshot!; + Initial = Last; + Assert.Equal(_process.Id, Last.Identity.ProcessId); + Assert.Equal(_manifest.Runtime, Last.Identity.Runtime); + Assert.Equal(_manifest.SourceRevision, Last.Identity.SourceRevision); + Assert.Equal(_manifest.AssemblyName, Last.Identity.AssemblyName); + Assert.Equal(_manifest.AssemblyVersion, Last.Identity.AssemblyVersion); + Assert.Equal(_manifest.Sha256, Last.Identity.Sha256, ignoreCase: true); + Assert.Contains(_manifest.SourceRevision, Last.Identity.InformationalVersion, StringComparison.OrdinalIgnoreCase); + Assert.Equal(_manifest.Runtime == "New", Last.Identity.HasDissemination); + Assert.Equal(Configuration.Enabled, Last.Enabled); + Assert.Equal(Configuration.Enabled, Last.NamespaceEnabled); + if (Configuration.SiloProcessorCount > 0) + { + Assert.Equal(Configuration.SiloProcessorCount, Last.ProcessorCount); + } + var actualDirectory = Path.GetDirectoryName(Path.GetFullPath(Last.Identity.AssemblyPath)); + Assert.Equal(Path.GetFullPath(_process.StartInfo.WorkingDirectory), actualDirectory); + foreach (var name in new[] { "Orleans.Runtime", "Orleans.Core", "Orleans.Core.Abstractions", "Orleans.Serialization" }) + { + Assert.Contains(name, Last.Identity.Assemblies.Keys); + } + + foreach (var (name, proof) in Last.Identity.Assemblies) + { + Assert.True(_manifest.Assemblies.TryGetValue(name, out var expected), $"Loaded unexpected assembly {name}."); + Assert.Equal(expected!.AssemblyVersion, proof.Version); + Assert.Equal(expected.Sha256, proof.Sha256, ignoreCase: true); + Assert.Equal(Path.GetFullPath(_process.StartInfo.WorkingDirectory), Path.GetDirectoryName(Path.GetFullPath(proof.Path))); + } + } + + public Task Send(string operation, string? peer = null) => + Send(TestContext.Current.CancellationToken, operation, peer); + + public async Task Send(CancellationToken cancellationToken, string operation, string? peer = null) + { + await _commandLock.WaitAsync(cancellationToken); + try + { + var command = new Command(Interlocked.Increment(ref _commandId), operation, peer); + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _pending[command.Id] = completed; + await _process.StandardInput.WriteLineAsync(JsonSerializer.Serialize(command).AsMemory(), cancellationToken); + await _process.StandardInput.FlushAsync(cancellationToken); + var response = await completed.Task.WaitAsync(TimeSpan.FromSeconds(70), cancellationToken); + Assert.True(response.Error is null, $"{Configuration.Name}/{operation}: {response.Error}; {_errorPath}"); + Last = response.Snapshot!; + return Last; + } + finally + { + _commandLock.Release(); + } + } + + public async Task Stop() + { + if (Stopped || !_started) + { + return; + } + + var clock = Stopwatch.StartNew(); + try + { + if (!_process.HasExited) + { + // Cleanup has its own bound, independent of a cancelled xUnit test token. + await _process.StandardInput.WriteLineAsync(JsonSerializer.Serialize(new Command(-2, "stop"))); + await _process.StandardInput.FlushAsync(); + await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(35)); + } + + await Task.WhenAll(_output, _error).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, _process.ExitCode); + } + catch + { + if (!_process.HasExited) + { + ForcedKill = true; + _process.Kill(entireProcessTree: true); + await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(10)); + } + + throw; + } + finally + { + ShutdownMilliseconds = clock.Elapsed.TotalMilliseconds; + Stopped = true; + } + } + + private async Task PumpOutput() + { + await using var log = File.CreateText(_logPath); + while (await _process.StandardOutput.ReadLineAsync() is { } line) + { + await log.WriteLineAsync(line); + await log.FlushAsync(); + if (!line.StartsWith("HARNESS ", StringComparison.Ordinal)) + { + continue; + } + + var response = JsonSerializer.Deserialize(line["HARNESS ".Length..])!; + if (response.Snapshot is not null) + { + Last = response.Snapshot; + } + + if (_pending.TryRemove(response.Id, out var completed)) + { + completed.TrySetResult(response); + } + } + + foreach (var completion in _pending.Values) + { + completion.TrySetException(new InvalidOperationException($"Silo stdout closed; inspect {_errorPath}.")); + } + } + + private async Task PumpErrors() + { + await using var log = File.CreateText(_errorPath); + while (await _process.StandardError.ReadLineAsync() is { } line) + { + await log.WriteLineAsync(line); + await log.FlushAsync(); + } + } + + public async ValueTask DisposeAsync() + { + if (_disposed) + { + return; + } + + _disposed = true; + try + { + await Stop(); + } + finally + { + _process.Dispose(); + _commandLock.Dispose(); + } + } +} diff --git a/test/Dissemination.PerformanceHarness/Tests/ScalingComparison.cs b/test/Dissemination.PerformanceHarness/Tests/ScalingComparison.cs new file mode 100644 index 00000000000..2482816057c --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Tests/ScalingComparison.cs @@ -0,0 +1,106 @@ +using System.Globalization; +using System.Text; +using System.Text.Json; + +namespace Orleans.Dissemination.PerformanceHarness; + +internal sealed record ScalingComparison(ScalingPair[] Pairs, int IncompletePairs) +{ + public static ScalingComparison Create(string json) + { + var samples = JsonSerializer.Deserialize(json) + ?? throw new InvalidOperationException("Missing scaling samples."); + var pairs = new List(); + var incomplete = 0; + foreach (var group in samples + .Where(sample => sample.RuntimePath is "CurrentDisabled" or "CurrentEnabledSupported") + .GroupBy(sample => (sample.Size, sample.Scenario, sample.Repetition))) + { + if (group.Count() == 1) + { + incomplete++; + continue; + } + + var disabled = group.Single(sample => sample.RuntimePath == "CurrentDisabled"); + var enabled = group.Single(sample => sample.RuntimePath == "CurrentEnabledSupported"); + Validate(disabled); + Validate(enabled); + if (disabled.Iterations != enabled.Iterations + || disabled.OfferedPublications != enabled.OfferedPublications + || disabled.Runtime.SourceRevision != enabled.Runtime.SourceRevision + || disabled.Runtime.Sha256 != enabled.Runtime.Sha256 + || disabled.Environment.GetRawText() != enabled.Environment.GetRawText()) + { + throw new InvalidOperationException($"Mismatched binary, workload, or environment for {group.Key}."); + } + + pairs.Add(new( + disabled.Size, disabled.Scenario, disabled.Repetition, disabled.Iterations, disabled.OfferedPublications, + disabled.Runtime.SourceRevision, disabled.Runtime.Sha256, disabled.Environment, + ScalingCost.From(disabled), ScalingCost.From(enabled))); + } + + return new(pairs.ToArray(), incomplete); + } + + private static void Validate(ScalingSample sample) + { + if (sample.Size < 3 || sample.LiveSilos != sample.Size || sample.OfferedPublications <= 0 + || sample.Runtime is null || string.IsNullOrEmpty(sample.Runtime.SourceRevision) || string.IsNullOrEmpty(sample.Runtime.Sha256) + || sample.Environment.ValueKind != JsonValueKind.Object + || sample.Iterations < 3 || sample.ConvergenceMilliseconds is null || sample.ConvergenceMilliseconds.Length != sample.Iterations + || sample.TotalRpcs <= 0 || sample.SerializedBytesSent <= 0 || sample.AllocatedBytes <= 0 + || sample.CpuMilliseconds < 0 || sample.ConvergenceMilliseconds.Any(value => !double.IsFinite(value) || value < 0)) + { + throw new InvalidOperationException($"Incomplete scaling measurement for {sample.RuntimePath}, {sample.Size} silos."); + } + } + + public string ToMarkdown() + { + var result = new StringBuilder(); + result.AppendLine("### Dissemination disabled vs enabled"); + result.AppendLine("Each cell is disabled -> enabled. Costs per offered publication include background and control traffic. Single-host, resource-shared measurements; raw totals, environment and per-node data are in the artifacts."); + result.AppendLine(); + result.AppendLine("| Silos / scenario / repetition | RPCs / publication | KiB / publication | Allocated KiB / publication | CPU ms / publication | Median ms | P95 ms |"); + result.AppendLine("|---|---:|---:|---:|---:|---:|---:|"); + foreach (var pair in Pairs) + { + result.AppendLine(FormattableString.Invariant( + $"| {pair.Size} / {pair.Scenario} / {pair.Repetition} | {pair.Disabled.RpcsPerPublication:F2} -> {pair.Enabled.RpcsPerPublication:F2} | {pair.Disabled.SerializedBytesPerPublication / 1024:F2} -> {pair.Enabled.SerializedBytesPerPublication / 1024:F2} | {pair.Disabled.AllocatedBytesPerPublication / 1024:F2} -> {pair.Enabled.AllocatedBytesPerPublication / 1024:F2} | {pair.Disabled.CpuMillisecondsPerPublication:F2} -> {pair.Enabled.CpuMillisecondsPerPublication:F2} | {pair.Disabled.MedianConvergenceMilliseconds:F2} -> {pair.Enabled.MedianConvergenceMilliseconds:F2} | {pair.Disabled.P95ConvergenceMilliseconds:F2} -> {pair.Enabled.P95ConvergenceMilliseconds:F2} |")); + } + + result.AppendLine(); + result.AppendLine($"Incomplete pairs: {IncompletePairs.ToString(CultureInfo.InvariantCulture)}."); + return result.ToString(); + } +} + +internal sealed record ScalingSample( + string RuntimePath, int Size, int LiveSilos, string Scenario, int Repetition, int Iterations, int OfferedPublications, + ScalingBinary Runtime, JsonElement Environment, long TotalRpcs, double SerializedBytesSent, + long AllocatedBytes, double CpuMilliseconds, long RetainedManagedBytes, double[] ConvergenceMilliseconds); + +internal sealed record ScalingBinary(string SourceRevision, string Sha256); + +internal sealed record ScalingPair( + int Size, string Scenario, int Repetition, int Iterations, int OfferedPublications, + string SourceRevision, string Sha256, JsonElement Environment, ScalingCost Disabled, ScalingCost Enabled); + +internal sealed record ScalingCost( + long TotalRpcs, double SerializedBytesSent, long AllocatedBytes, double CpuMilliseconds, long RetainedManagedBytes, + double RpcsPerPublication, double SerializedBytesPerPublication, double AllocatedBytesPerPublication, + double CpuMillisecondsPerPublication, double MedianConvergenceMilliseconds, double P95ConvergenceMilliseconds) +{ + public static ScalingCost From(ScalingSample sample) + { + var ordered = sample.ConvergenceMilliseconds.Order().ToArray(); + var median = (ordered[(ordered.Length - 1) / 2] + ordered[ordered.Length / 2]) / 2; + return new( + sample.TotalRpcs, sample.SerializedBytesSent, sample.AllocatedBytes, sample.CpuMilliseconds, sample.RetainedManagedBytes, + (double)sample.TotalRpcs / sample.OfferedPublications, sample.SerializedBytesSent / sample.OfferedPublications, + (double)sample.AllocatedBytes / sample.OfferedPublications, sample.CpuMilliseconds / sample.OfferedPublications, + median, ordered[(int)Math.Ceiling(0.95 * ordered.Length) - 1]); + } +} diff --git a/test/Dissemination.PerformanceHarness/Tests/ScalingTests.cs b/test/Dissemination.PerformanceHarness/Tests/ScalingTests.cs new file mode 100644 index 00000000000..66a9ec25d75 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Tests/ScalingTests.cs @@ -0,0 +1,392 @@ +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Text.Json; +using Xunit; + +namespace Orleans.Dissemination.PerformanceHarness; + +public sealed class ScalingTests +{ + [Fact] + [Trait("Category", "DisseminationScale")] + public async Task Scaling_SelectedRuntimePathsConverge() + { + Assert.Equal("true", Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_MANUAL_RUN")); + var sizes = (Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_SIZES") ?? "4,8") + .Split(',').Select(value => int.Parse(value, System.Globalization.CultureInfo.InvariantCulture)).ToArray(); + var iterations = Setting("ITERATIONS", 3, 3, 200); + var repetitions = Setting("REPETITIONS", 2, 1, 5); + Assert.All(sizes, size => Assert.InRange(size, 3, 128)); + Assert.InRange(sizes.Length, 1, 6); + Assert.Equal(sizes.Length, sizes.Distinct().Count()); + var scenarios = (Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_SCENARIOS") ?? "stable,churn,partition").Split(','); + Assert.InRange(scenarios.Length, 1, 3); + Assert.Equal(scenarios.Length, scenarios.Distinct().Count()); + Assert.All(scenarios, scenario => Assert.Contains(scenario, new[] { "stable", "churn", "partition" })); + var paths = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_RUNTIME_PATHS") ?? "All"; + Assert.Contains(paths, new[] { "All", "Current" }); + + var records = new List(); + var output = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_RESULTS")!; + RuntimeVariant[] variants = + [ + new("OriginalLegacy", "Old", Enabled: false), + new("CurrentDisabled", "New", Enabled: false), + new("CurrentEnabledSupported", "New", Enabled: true), + ]; + if (paths == "Current") + { + variants = variants[1..]; + } + + foreach (var size in sizes) + { + foreach (var scenario in scenarios) + { + for (var repetition = 0; repetition < repetitions; repetition++) + { + // Reverse the whole sequence so every off/on pair changes order. + for (var order = 0; order < variants.Length; order++) + { + var variant = variants[ExecutionIndex(repetition, order, variants.Length)]; + var record = await Measure(size, scenario, variant, iterations, repetition, order); + records.Add(record); + var json = JsonSerializer.Serialize(records, new JsonSerializerOptions { WriteIndented = true }); + await File.WriteAllTextAsync(Path.Combine(output, "cost-results.json"), json, TestContext.Current.CancellationToken); + await File.WriteAllTextAsync(Path.Combine(output, "comparisons.json"), + JsonSerializer.Serialize(ScalingComparison.Create(json), new JsonSerializerOptions { WriteIndented = true }), + TestContext.Current.CancellationToken); + } + } + } + } + + var report = ScalingComparison.Create(JsonSerializer.Serialize(records)); + Assert.Equal(0, report.IncompletePairs); + Assert.Equal(sizes.Length * scenarios.Length * repetitions, report.Pairs.Length); + if (Environment.GetEnvironmentVariable("GITHUB_STEP_SUMMARY") is { Length: > 0 } summary) + { + await File.AppendAllTextAsync(summary, report.ToMarkdown(), TestContext.Current.CancellationToken); + } + } + + private static async Task Measure(int size, string scenario, RuntimeVariant variant, int iterations, int repetition, int order) + { + var enabled = variant.Enabled; + var siloProcessorCount = Setting("SILO_PROCESSOR_COUNT", 0, 0, 64); + var gcConserveMemory = Setting("GC_CONSERVE_MEMORY", 0, 0, 9); + var bootstrap = Stopwatch.StartNew(); + await using var cluster = new ProcessCluster( + $"scale-{size}-{scenario}-{variant.Name}-{repetition}", siloProcessorCount, gcConserveMemory); + for (var index = 0; index < size; index++) + { + await cluster.Start(variant.Runtime, enabled); + } + + await cluster.Stabilize(); + await cluster.AssertControlRpcs(allPairs: false); + await cluster.PublishAndConverge(); + await cluster.PublishAndConverge(); + if (enabled) + { + await ConfirmSupport(cluster); + } + + await cluster.Save("warmup.json", await Task.WhenAll(cluster.Active.Select(node => node.Send("measure")))); + // Socket EventCounters arrive in one-second buckets. Every path has the same two-second margins. + await Task.Delay(TimeSpan.FromSeconds(2), TestContext.Current.CancellationToken); + var start = (await Task.WhenAll(cluster.Active.Select(node => node.Send("measure")))) + .ToDictionary(snapshot => snapshot.Identity.ProcessId); + Assert.Equal(size, start.Count); + Assert.All(start.Values, snapshot => Assert.Equal(size, snapshot.ActiveMembers.Length)); + var bootstrapMilliseconds = bootstrap.Elapsed.TotalMilliseconds; + var startedAt = DateTimeOffset.UtcNow; + var clock = Stopwatch.StartNew(); + var latencies = new List(); + var publicationLatencies = new List(); + var convergenceWaits = new List(); + var recoveryLatencies = new List(); + var offeredPublications = 0; + var restarts = 0; + var partitions = 0; + for (var iteration = 0; iteration < iterations; iteration++) + { + if (scenario == "churn" && iteration > 0 && iteration % 2 == 0) + { + var victim = cluster.Active[iteration % size]; + await victim.Stop(); + // Reuse the endpoint to exercise a real new silo incarnation and stale-generation cleanup. + await cluster.Start(variant.Runtime, enabled, victim.Configuration.SiloPort); + await cluster.Stabilize(); + if (enabled) + { + // Bootstrap/fallback work stays inside the churn window, but offered rounds use supported peers. + await ConfirmSupport(cluster); + } + + restarts++; + } + else if (scenario == "partition" && iteration % 3 == 1) + { + var victim = cluster.Active[iteration % size]; + await victim.Send("partition"); + partitions++; + foreach (var source in cluster.Active.Where(node => !ReferenceEquals(node, victim))) + { + await source.Send("publish"); + offeredPublications++; + } + + var recovery = Stopwatch.StartNew(); + // Establish delivery paths before offering the next sample, including on the one-way legacy path. + await cluster.HealPartition(victim); + await PublishRound(); + recoveryLatencies.Add(recovery.Elapsed.TotalMilliseconds); + continue; + } + + await PublishRound(); + } + + async Task PublishRound() + { + latencies.Add(await cluster.PublishAndConverge()); + publicationLatencies.Add(cluster.LastPublicationMilliseconds); + convergenceWaits.Add(cluster.LastConvergenceWaitMilliseconds); + offeredPublications += size; + } + + await Task.Delay(TimeSpan.FromSeconds(2), TestContext.Current.CancellationToken); + await Task.WhenAll(cluster.Active.Select(node => node.Send("measure"))); + var elapsed = clock.Elapsed.TotalMilliseconds; + var finishedAt = DateTimeOffset.UtcNow; + var finish = cluster.All.Select(node => node.Last).ToArray(); + var retained = await Task.WhenAll(cluster.Active.Select(node => node.Send("retained-memory"))); + var retainedByPid = retained.ToDictionary(snapshot => snapshot.Identity.ProcessId, snapshot => snapshot.ManagedHeapBytes); + Assert.Equal(scenario == "churn" ? (iterations - 1) / 2 : 0, restarts); + Assert.Equal(scenario == "partition" ? (iterations + 1) / 3 : 0, partitions); + if (enabled) + { + Assert.All(start.Values, snapshot => Assert.Empty(snapshot.UnconfirmedPeers)); + Assert.All(cluster.Active, node => Assert.Empty(node.Last.UnconfirmedPeers)); + } + + var nodes = finish.Select(snapshot => + { + start.TryGetValue(snapshot.Identity.ProcessId, out var baseline); + var allMessages = Delta(snapshot, baseline, "orleans-messaging-sent-messages-size|"); + var requestMessages = Delta(snapshot, baseline, "orleans-messaging-sent-messages-size|", "MessageDirection=Request"); + var oneWayMessages = Delta(snapshot, baseline, "orleans-messaging-sent-messages-size|", "MessageDirection=OneWay"); + var received = Delta(snapshot, baseline, "orleans-messaging-received-messages-size|"); + var topology = snapshot.TopologyMembers.Contains(snapshot.Address) + ? snapshot + : baseline ?? cluster.All.Single(node => node.Last.Identity.ProcessId == snapshot.Identity.ProcessId).Initial; + var index = Array.IndexOf(topology.TopologyMembers, snapshot.Address); + return new NodeCost( + snapshot.Identity.ProcessId, + snapshot.Address, + index >= 0 ? index : null, + index >= 0 && (topology.AggregationTree ? index <= topology.Fanout : index < topology.Fanout), + topology.OriginatorTargets, + topology.ForwardingTargets, + requestMessages.Count + oneWayMessages.Count, + allMessages.Count, + allMessages.Sum, + received.Count, + received.Sum, + snapshot.TransportBytesWritten - (baseline?.TransportBytesWritten ?? 0), + snapshot.TransportBytesRead - (baseline?.TransportBytesRead ?? 0), + snapshot.SocketBytesSent - (baseline?.SocketBytesSent ?? 0), + snapshot.SocketBytesReceived - (baseline?.SocketBytesReceived ?? 0), + snapshot.SocketCounterSamples - (baseline?.SocketCounterSamples ?? 0), + snapshot.CpuMilliseconds - (baseline?.CpuMilliseconds ?? 0), + snapshot.AllocatedBytes - (baseline?.AllocatedBytes ?? 0), + retainedByPid.GetValueOrDefault(snapshot.Identity.ProcessId), + snapshot.WorkingSetBytes, + snapshot.PrivateBytes, + Delta(snapshot, baseline, "orleans-dissemination-bytes-sent|").Sum, + Delta(snapshot, baseline, "orleans-dissemination-broadcast-sent|").Sum, + Delta(snapshot, baseline, "orleans-dissemination-anti-entropy-exchanges|", "direction=out").Sum, + Delta(snapshot, baseline, "orleans-dissemination-broadcast-sent|", "namespace=load").Sum, + Delta(snapshot, baseline, "orleans-dissemination-values-sent|", "namespace=load").Sum); + }).ToArray(); + Assert.All(nodes, node => + { + Assert.True(node.SentMessages > 0, "Orleans message histogram evidence is missing."); + Assert.True(node.SerializedBytesSent > 0, "Serialized byte evidence is missing."); + Assert.True(node.SocketSamples > 0 && node.SocketBytesSent > 0, "OS socket byte evidence is missing."); + }); + if (!enabled) + { + Assert.All(nodes, node => + { + Assert.Equal(0, node.DisseminationBroadcasts); + Assert.Equal(0, node.AntiEntropyExchanges); + }); + } + else + { + Assert.True(nodes.Sum(node => node.DisseminationBroadcasts) > 0, + "The supported-enabled cost window did not exercise any dissemination broadcasts."); + } + + var record = new + { + Schema = 1, + RuntimePath = variant.Name, + Runtime = finish[0].Identity, + Size = size, + LiveSilos = start.Count, + Scenario = scenario, + Enabled = enabled, + Iterations = iterations, + Repetition = repetition, + ExecutionOrder = order, + OfferedPublications = offeredPublications, + RestartedProcesses = restarts, + InjectedPartitions = partitions, + WindowMilliseconds = elapsed, + WindowStartedAtUtc = startedAt, + WindowFinishedAtUtc = finishedAt, + ConvergenceMilliseconds = latencies, + PublicationMilliseconds = publicationLatencies, + ConvergenceAfterPublicationMilliseconds = convergenceWaits, + PartitionRecoveryMilliseconds = recoveryLatencies, + TotalRpcs = nodes.Sum(node => node.Rpcs), + TotalMessages = nodes.Sum(node => node.SentMessages), + LoadBroadcastRequests = nodes.Sum(node => node.LoadBroadcastRequests), + LoadValueTransmissions = nodes.Sum(node => node.LoadValueTransmissions), + LoadBroadcastRequestsPerRound = nodes.Sum(node => node.LoadBroadcastRequests) / iterations, + LoadBroadcastRequestsPerSecond = nodes.Sum(node => node.LoadBroadcastRequests) / (elapsed / 1000), + SerializedBytesSent = nodes.Sum(node => node.SerializedBytesSent), + SocketBytesSent = nodes.Sum(node => node.SocketBytesSent), + TransportBytesSubmitted = nodes.Sum(node => node.TransportBytesSubmitted), + CpuMilliseconds = nodes.Sum(node => node.CpuMilliseconds), + AllocatedBytes = nodes.Sum(node => node.AllocatedBytes), + RetainedManagedBytes = nodes.Sum(node => node.RetainedManagedBytes), + BootstrapMilliseconds = bootstrapMilliseconds, + BootstrapCpuMilliseconds = start.Values.Sum(snapshot => snapshot.CpuMilliseconds), + BootstrapAllocatedBytes = start.Values.Sum(snapshot => snapshot.AllocatedBytes), + Nodes = nodes, + Environment = new + { + RuntimeInformation.OSDescription, + RuntimeInformation.FrameworkDescription, + System.Environment.ProcessorCount, + SiloProcessorCount = siloProcessorCount, + GCConserveMemory = gcConserveMemory, + WorkerProcessorCounts = start.Values.Select(snapshot => snapshot.ProcessorCount).Distinct().Order().ToArray(), + WorkerServerGC = start.Values.Select(snapshot => snapshot.ServerGC).Distinct().Order().ToArray(), + }, + Methodology = new + { + WarmupRounds = 2, + SettlingMarginSeconds = 2, + AutomaticLoadPublisherPaused = true, + ProductionDisseminationCadence = variant.Runtime == "New" && enabled, + ExplicitDisseminationOptOut = variant.Runtime == "New" && !enabled, + LoadNamespaceSupportConfirmedBeforeOfferedRounds = enabled, + IncludesUnconfirmedBootstrapTraffic = enabled && scenario == "churn", + LivenessFailureDetectionEnabled = false, + StorageProvider = "locked local JSON file; no external-service network", + Window = "after warmup/capability discovery, through offered load, churn/restarts and final 2-second drain; GC retained-size probe is outside CPU/allocation window", + RpcDefinition = "sent Orleans message-size observations tagged MessageDirection=Request OR MessageDirection=OneWay; not response messages", + SerializedBytes = "complete serialized Orleans messages (headers and bodies), not dissemination payload counters", + SocketBytes = "System.Net.Sockets bytes-sent/bytes-received EventCounters, 1-second sampling; cumulative values differenced across the window (incremental counters also supported); excludes TCP/IP headers and retransmits", + TransportBytes = "bytes advanced at the real connection transport pipe, including handshake; may include buffered bytes lost when a partition aborts a connection", + RetainedMemory = "full GC live managed bytes on surviving nodes only; not a peak; RSS/private bytes and retired-node snapshots also retained", + Includes = "all silo work during the window, including control/snapshot/instrumentation overhead and legacy fallback; restart startup/shutdown costs included for churn", + Observation = "identical state polling on selected paths; detailed counters only at window boundaries", + Topology = "per-node actual tree at end, or start for a retired node; raw command snapshots preserve intervening topology changes; role is not time-weighted", + Excludes = "controller process CPU/allocation, storage disk byte accounting, cross-machine latency, NIC/TCP wire overhead, production storage/placement/grain work", + PartitionComparison = "retire partitioned connections at both endpoints, then confirm healed connections with bidirectional acknowledged control RPCs before the same single publication round on all paths; teardown and readiness remain inside partition recovery time and total cost", + OriginalBaselineAttribution = "original-to-current disabled deltas include all intervening source changes, including lifecycle prerequisites and load notification/tombstone/dedup changes; only current off/on isolates enabling the subsystem on the same binary", + Interpretation = "separate original-legacy, current-explicitly-disabled and current-explicitly-enabled-supported measurements", + }, + }; + await cluster.Save("window-start.json", start); + await cluster.Save("window-finish.json", finish); + await cluster.Save("cost.json", record); + return record; + } + + internal static int ExecutionIndex(int repetition, int order, int count) => + repetition % 2 == 0 ? order : count - 1 - order; + + private static Task ConfirmSupport(ProcessCluster cluster) + { + var timeout = TimeSpan.FromSeconds(90); + foreach (var node in cluster.Active) + { + Assert.True(node.Last.AntiEntropyPeerCount > 0); + Assert.True(node.Last.AntiEntropyIntervalMilliseconds > 0); + var rounds = Math.Ceiling((cluster.Active.Length - 1.0) / node.Last.AntiEntropyPeerCount); + var rotationBudget = TimeSpan.FromMilliseconds(2 * rounds * node.Last.AntiEntropyIntervalMilliseconds) + + TimeSpan.FromSeconds(60); + if (rotationBudget > timeout) + { + timeout = rotationBudget; + } + } + + return cluster.Eventually("all enabled peers confirm support before offered publication rounds", async () => + { + var snapshots = await Task.WhenAll(cluster.Active.Select(node => node.Send("snapshot"))); + return snapshots.All(snapshot => snapshot.UnconfirmedPeers.Length == 0); + }, timeout); + } + + private sealed record RuntimeVariant(string Name, string Runtime, bool Enabled); + + private static MetricValue Delta(NodeSnapshot value, NodeSnapshot? baseline, string prefix, string? tag = null) + { + var entries = value.Metrics.Where(pair => pair.Key.StartsWith(prefix, StringComparison.Ordinal) + && (tag is null || pair.Key.Contains(tag, StringComparison.Ordinal))); + long count = 0; + double sum = 0; + foreach (var (key, metric) in entries) + { + var previous = baseline?.Metrics.GetValueOrDefault(key); + count += metric.Count - (previous?.Count ?? 0); + sum += metric.Sum - (previous?.Sum ?? 0); + } + + return new(count, sum); + } + + private static int Setting(string name, int defaultValue, int minimum, int maximum) + { + var value = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_" + name); + var result = value is null ? defaultValue : int.Parse(value, System.Globalization.CultureInfo.InvariantCulture); + Assert.InRange(result, minimum, maximum); + return result; + } + + private sealed record NodeCost( + int ProcessId, + string Address, + int? TreeRank, + bool UpperTree, + string[] OriginatorTargets, + string[] ForwardingTargets, + long Rpcs, + long SentMessages, + double SerializedBytesSent, + long ReceivedMessages, + double SerializedBytesReceived, + long TransportBytesSubmitted, + long TransportBytesRead, + double SocketBytesSent, + double SocketBytesReceived, + long SocketSamples, + double CpuMilliseconds, + long AllocatedBytes, + long RetainedManagedBytes, + long WorkingSetBytes, + long PrivateBytes, + double DisseminationPayloadBytes, + double DisseminationBroadcasts, + double AntiEntropyExchanges, + double LoadBroadcastRequests, + double LoadValueTransmissions); +} diff --git a/test/Dissemination.PerformanceHarness/methodology.json b/test/Dissemination.PerformanceHarness/methodology.json new file mode 100644 index 00000000000..73971d5035a --- /dev/null +++ b/test/Dissemination.PerformanceHarness/methodology.json @@ -0,0 +1,80 @@ +{ + "schema": 1, + "purpose": "Manually selected, real-process measurements of original legacy and candidate dissemination off/on costs.", + "entrypoint": ".github/scripts/dissemination-performance.ps1", + "workflow": ".github/workflows/dissemination-performance.yml", + "trigger": "workflow_dispatch", + "extractedFrom": "3fc0a4610cd2eb5feef27e958fcca59658aaae47", + "baseline": { + "repository": "dotnet/orleans", + "commit": "6739589254b746a8790cf53524e6abe372bb53d4" + }, + "selection": { + "candidate": "Explicit repository/ref inputs resolve to a commit in a separate checkout. Both worker checkouts receive only this tooling revision's Silo and Shared files.", + "build": "Each worker resolves runtime project references, SDK selection and package settings from its own checkout. The controller builds from the tooling checkout with shared protocol/instrumentation and MTP/xUnit.", + "identity": "selection.json preserves tooling/candidate/original commits and build dotnet --info. Each OS process checks its PID, source revision, assembly version, informational commit, dissemination presence, loaded Orleans DLL hashes and binary-directory paths against its published manifest.", + "comparison": "Candidate off/on pairs use one published binary directory and identical process counts, workload and environment. Original-to-candidate deltas include every intervening runtime change." + }, + "controls": { + "defaults": { "sizes": [4, 8], "iterations": 3, "repetitions": 2, "runtimePaths": "All", "scenarios": ["stable", "churn", "partition"] }, + "sizes": "One to six distinct counts, each 3..128", + "iterations": "3..200", + "repetitions": "1..5; even repetitions run original/off/on, odd repetitions run on/off/original; Current selects off/on", + "scenarios": "Distinct stable, churn and partition selections", + "siloProcessorCount": "0..64; 0 inherits the runner processor count", + "gcConserveMemory": "0..9; equal process environment for every path", + "memory": "Linux/Windows available-memory planning guard: 128 MiB per requested silo plus 1 GiB controller headroom; host-resources.json records the calculation. Job/process deadlines bound the run independently.", + "largeCurve": "For an explicitly requested large stable curve, use 8,16,32,64,100 with Current, two repetitions, one processor per silo and GCConserveMemory=9 on a sufficiently provisioned host." + }, + "measurement": { + "test": "Scaling_SelectedRuntimePathsConverge", + "paths": ["OriginalLegacy", "CurrentDisabled", "CurrentEnabledSupported"], + "enablement": "Candidate off/on explicitly sets the subsystem and both namespace flags. Overlay, repair, coalescing and namespace cadence use the selected runtime's settings.", + "warmup": "Two production publication rounds and complete enabled-peer support confirmation precede the cost window. Preparation time, CPU and allocation totals are reported separately.", + "publication": "Dispose the automatic DeploymentLoadPublisher timer on its scheduler while retaining the field reference for original-runtime lifecycle shutdown. Invoke the production PublishStatistics method once per active node, then require exact load dictionary inventory and values using IncludeFields=true.", + "stable": "Repeat equal closed-loop publication/convergence rounds.", + "churn": "Gracefully stop and restart one process at its original endpoint every other round. Confirm membership and supported enabled peers before the next offered round. Replacement startup, shutdown, discovery and fallback costs remain in the measured window.", + "partition": "Abort real inbound/outbound silo connections and reject new admissions at one endpoint. Publish once on each remaining node. Recovery drains middleware and close tasks at both endpoints, including unidentified handshakes, then confirms bidirectional acknowledged control RPCs before a single publication per active node. Teardown/readiness/retries remain inside recovery time and total cost, preserving the #11243 barrier.", + "window": "Identical two-second socket-sampling margins; costs include background runtime, control RPCs and instrumentation. The post-window full-GC retained-memory probe is separate from CPU/allocation costs.", + "membership": "Exclusive-file-lease JSON provider with atomic replacement, ETags and incarnation-aware rows. Liveness detection is disabled so the transport partition retains stable membership.", + "metrics": [ + "Request plus one-way Orleans RPC counts; all messages including responses reported separately", + "Load broadcast requests and logical value transmissions reported separately, with requests per round and per second", + "Complete serialized Orleans message bytes, including headers and bodies", + "Process-wide System.Net.Sockets sent/received counters and separate transport-pipe submitted/read bytes", + "Process CPU, allocated bytes, retained managed bytes, working set and private bytes", + "Publication latency, residual convergence wait, end-to-end convergence and partition recovery time", + "Per-process topology, originator/forwarding targets, tree rank, upper-tree role and namespace/anti-entropy counters" + ], + "reports": "cost-results.json keeps raw samples. comparisons.json and the job summary pair candidate off/on by size/scenario/repetition, checking equal live process count, binary/revision, environment, iterations and offered publications. Costs normalize per offered publication; median/P95 convergence and incomplete pairs remain explicit." + }, + "interpretation": { + "environment": "Loopback TCP processes share one runner's CPU, memory and local storage. Results describe a closed-loop control-plane workload with a convergence barrier after each round.", + "timing": "100 ms state polling and one-second socket sampling introduce boundary uncertainty; raw UTC snapshots and separate publication/wait durations preserve that context.", + "socketBytes": "Socket counters measure application bytes; transport counters include submitted buffered bytes, including bytes abandoned during connection abort.", + "memory": "Retained memory is a post-GC live-size sample for surviving processes. Artifacts also retain retired-process snapshots and per-process RSS/private-memory samples.", + "costScope": "The window covers silo processes. Controller CPU/allocation, storage disk bytes, TCP/IP headers, retransmission, NIC overhead, cross-machine latency and application-grain work require separate experiments.", + "attribution": "The candidate off/on pair isolates enablement on the same binaries. Original-to-candidate comparisons reflect all source changes between the selected revisions." + }, + "bounds": { + "startupSeconds": 90, + "controllerStartupSeconds": 100, + "siloCommandSeconds": 60, + "controllerCommandSeconds": 70, + "convergenceSeconds": 60, + "partitionRecoverySeconds": 60, + "capabilityDiscovery": "max(90 seconds, two complete peer rotations at actual repair cadence plus 60 seconds)", + "membershipLockSeconds": 10, + "shutdownSeconds": 35, + "processLifetimeMinutes": 45, + "workflowMinutes": 90, + "failure": "Preserve partial samples, logs, snapshots, binlogs and TRX. Cleanup kills only owned child processes which exceeded their shutdown deadline and reports that as a failure." + }, + "artifacts": { + "root": "Artifacts/DisseminationPerformance", + "results": "results/-: selection, invocation, methodology, host resources, TRX, comparisons, raw costs and per-cluster process snapshots/stdout/stderr", + "binaries": "bin/{Old,New,Runner} and bin/selection.json for explicit -SkipBuild replay", + "paths": "Artifact roots are single named directories under Artifacts; existing links/reparse points are rejected. Every invocation writes a fresh results directory.", + "retentionDays": { "results": 30, "binaries": 7 } + } +} From a4bd25c4030daae5cbf7cecbca38a24c5050eab4 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 16 Sep 2026 08:58:57 -0700 Subject: [PATCH 02/12] ci: bootstrap manual performance validation on feature branch --- .github/workflows/ci.yml | 943 ++------------------------------------- 1 file changed, 48 insertions(+), 895 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20a90248ade..5c943bdda4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,903 +1,56 @@ -name: .NET CI +name: Dissemination performance bootstrap on: workflow_dispatch: - push: - branches: - - main - pull_request: - branches: - - main - merge_group: - types: - - checks_requested permissions: contents: read env: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 DOTNET_NOLOGO: true - DOTNET_COVERAGE_VERSION: 18.5.2 - # Docker images started by workflow steps are centralized so pulls can be retried and mirrors can be swapped in one place. - AZURITE_IMAGE: mcr.microsoft.com/azure-storage/azurite:3.35.0@sha256:647c63a91102a9d8e8000aab803436e1fc85fbb285e7ce830a82ee5d6661cf37 - EVENTHUBS_EMULATOR_IMAGE: mcr.microsoft.com/azure-messaging/eventhubs-emulator:2.2.0@sha256:2c8e0d4dd93a5fc078df2721eeb3e211442d555d61293ac3972df931c6d9333a - COSMOSDB_EMULATOR_IMAGE: mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:vnext-preview@sha256:bf3bd3cbe3fae2005a0745f77b01d1fc8cd04112193f3ee32289ee15ee9ae5fa - CONSUL_IMAGE: public.ecr.aws/hashicorp/consul:1.19 - DYNAMODB_IMAGE: amazon/dynamodb-local:latest - ELASTICMQ_IMAGE: softwaremill/elasticmq-native:latest - # Keep in sync with SeaweedFSTestContainer - # so the fixtures use the pre-pulled image. - SEAWEEDFS_IMAGE: ghcr.io/chrislusf/seaweedfs:4.46@sha256:08d516132314207d10c8e37cbffc1f32b147d870169688734cc61c6231625b62 - MINISTACK_IMAGE: ministackorg/ministack:1.4.11@sha256:8b05715b9effc23d998d4a4e0c0a0c4af7f9435f0ff076a30cf27a77326f14b1 - MSSQL_IMAGE: mcr.microsoft.com/mssql/server:2022-CU26-ubuntu-22.04@sha256:ba4c8329f48fb8f02e1416be6a930ebfd71268caee78aa985f3af4315e457c89 - NATS_IMAGE: nats:latest - POSTGRES_IMAGE: postgres - REDIS_IMAGE: redis - TESTCONTAINERS_RYUK_IMAGE: testcontainers/ryuk:0.14.0 - ZOOKEEPER_IMAGE: zookeeper:3.9 jobs: - whitespace: - name: Whitespace formatting - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - with: - fetch-depth: 0 - - name: Setup .NET - uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5 - with: - global-json-file: global.json - - name: Verify .NET formatter - run: dotnet format --version - - name: Check solution - run: dotnet format whitespace Orleans.slnx --verify-no-changes --verbosity normal - build: - name: Build - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest] - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - name: Setup Node.js - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5 - with: - node-version: '24.x' - - name: Setup .NET - uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5 - with: - global-json-file: global.json - - name: Test coverage reporting - if: matrix.os == 'ubuntu-latest' - shell: pwsh - run: ./.github/scripts/test-summarize-coverage.ps1 - - name: Build - run: dotnet build -bl - - name: Pack and validate package compatibility - if: matrix.os == 'ubuntu-latest' - run: dotnet pack Orleans.slnx --configuration Debug --no-build --no-restore -bl:package_validation.binlog - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - continue-on-error: true - with: - name: build_log_${{ matrix.os }} - retention-days: 1 - path: | - **/*.binlog - test-redis: - name: Redis provider tests - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - framework: ["net8.0", "net10.0"] - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - name: Start Redis - shell: bash - run: | - set -euo pipefail - - bash .github/scripts/docker-pull-with-retry.sh "$REDIS_IMAGE" - - docker run -d --pull=never --name redis \ - -p 6379:6379 \ - --health-cmd "redis-cli ping" \ - --health-interval 10s \ - --health-timeout 5s \ - --health-retries 5 \ - "$REDIS_IMAGE" - - bash .github/scripts/docker-wait-for-health.sh redis - - name: Test - uses: ./.github/actions/run-tests - with: - artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} - framework: ${{ matrix.framework }} - provider: Redis - env: - ORLEANSREDISCONNECTIONSTRING: "localhost:6379,ssl=False,abortConnect=False" - - name: Clean up Redis - if: always() - run: docker rm -f redis || true - test-cassandra: - name: Cassandra provider tests - runs-on: ubuntu-latest - env: - CASSANDRA_IMAGE: cassandra:${{ matrix.dbversion }} - strategy: - fail-fast: false - matrix: - dbversion: ["4.0", "4.1", "5.0"] - framework: ["net8.0", "net10.0"] - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - name: Pre-pull Cassandra test images - shell: bash - run: | - set -euo pipefail - - bash .github/scripts/docker-pull-with-retry.sh "$TESTCONTAINERS_RYUK_IMAGE" - bash .github/scripts/docker-pull-with-retry.sh "$CASSANDRA_IMAGE" - - name: Test - uses: ./.github/actions/run-tests - with: - artifact-name: test_output_${{ github.job }}_${{ matrix.dbversion }}_${{ matrix.framework }} - framework: ${{ matrix.framework }} - provider: Cassandra - env: - CASSANDRAVERSION: ${{ matrix.dbversion }} - test-postgres: - name: PostgreSQL provider tests - runs-on: ubuntu-latest - env: -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] - POSTGRES_PASSWORD: postgres - strategy: - fail-fast: false - matrix: - framework: ["net8.0", "net10.0"] - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - name: Start PostgreSQL - shell: bash - run: | - set -euo pipefail - - bash .github/scripts/docker-pull-with-retry.sh "$POSTGRES_IMAGE" - - docker run -d --pull=never --name postgres \ - -p 5432:5432 \ - -e POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ - --health-cmd "pg_isready -U postgres" \ - --health-interval 10s \ - --health-timeout 5s \ - --health-retries 5 \ - "$POSTGRES_IMAGE" \ - -c max_connections=200 - - bash .github/scripts/docker-wait-for-health.sh postgres - - name: Test - uses: ./.github/actions/run-tests - with: - artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} - framework: ${{ matrix.framework }} - provider: PostgreSql - env: -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="False positive")] - ORLEANSPOSTGRESCONNECTIONSTRING: "Server=127.0.0.1;Port=5432;Pooling=false;User Id=postgres;Password=postgres;SSL Mode=Disable" - - name: Clean up PostgreSQL - if: always() - run: docker rm -f postgres || true - test-mariadb: - name: MariaDB/MySQL provider tests - runs-on: ubuntu-latest - env: - # The official MariaDB 10.6 image is not published to GHCR. Prefer Google's Docker Hub - # mirror, with Docker Hub as a fallback, and pin both references to the same digest. - MARIADB_IMAGE: mirror.gcr.io/library/mariadb:10.6@sha256:daacc2f260f8ec999daa5e03a017a23a7e6fa3fb982aaf26e8b72f24daf03bc9 - MARIADB_FALLBACK_IMAGE: docker.io/library/mariadb:10.6@sha256:daacc2f260f8ec999daa5e03a017a23a7e6fa3fb982aaf26e8b72f24daf03bc9 -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] - MARIADB_ROOT_PASSWORD: "mariadb" - strategy: - fail-fast: false - matrix: - framework: ["net8.0", "net10.0"] - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - name: Start MariaDB - shell: bash - run: | - set -euo pipefail - - mariadb_image="$MARIADB_IMAGE" - if ! bash .github/scripts/docker-pull-with-retry.sh "$mariadb_image"; then - echo "Pulling MariaDB from Docker Hub after the mirror failed." - mariadb_image="$MARIADB_FALLBACK_IMAGE" - bash .github/scripts/docker-pull-with-retry.sh "$mariadb_image" - fi - - docker run -d --pull=never --name mariadb \ - -p 3306:3306 \ - -e MARIADB_ROOT_PASSWORD="$MARIADB_ROOT_PASSWORD" \ - --health-cmd "mariadb-admin ping -h 127.0.0.1 -uroot -p${MARIADB_ROOT_PASSWORD} --silent" \ - --health-interval 5s \ - --health-timeout 5s \ - --health-retries 24 \ - "$mariadb_image" - - for attempt in $(seq 1 60); do - status="$(docker inspect --format '{{.State.Health.Status}}' mariadb)" - case "$status" in - healthy) - exit 0 - ;; - unhealthy) - docker logs mariadb - exit 1 - ;; - esac - - sleep 2 - done - - echo "MariaDB did not become healthy in time." - docker logs mariadb - exit 1 - - name: Test - uses: ./.github/actions/run-tests - with: - artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} - framework: ${{ matrix.framework }} - provider: MySql - env: -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] - ORLEANSMYSQLCONNECTIONSTRING: "Server=127.0.0.1;Port=3306;UId=root;Pwd=mariadb;" - test-sqlserver: - name: Microsoft SQL Server provider tests - runs-on: ubuntu-latest - env: - ACCEPT_EULA: "Y" - MSSQL_PID: "Developer" -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] - SA_PASSWORD: "yourWeak(!)Password" - strategy: - fail-fast: false - matrix: - framework: ["net8.0", "net10.0"] - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - name: Start SQL Server - shell: bash - run: | - set -euo pipefail - - bash .github/scripts/docker-pull-with-retry.sh "$MSSQL_IMAGE" - - docker run -d --pull=never --name mssql \ - -p 1433:1433 \ - -e ACCEPT_EULA="$ACCEPT_EULA" \ - -e MSSQL_PID="$MSSQL_PID" \ - -e MSSQL_SA_PASSWORD="$SA_PASSWORD" \ - --health-cmd '/opt/mssql-tools18/bin/sqlcmd -C -S localhost -U sa -P "$MSSQL_SA_PASSWORD" -Q "SELECT 1" -b -o /dev/null' \ - --health-interval 2s \ - --health-timeout 5s \ - --health-retries 60 \ - "$MSSQL_IMAGE" - - bash .github/scripts/docker-wait-for-health.sh mssql 120 - - name: Test - uses: ./.github/actions/run-tests - with: - artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} - framework: ${{ matrix.framework }} - provider: SqlServer - env: -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] - ORLEANSMSSQLCONNECTIONSTRING: "Server=127.0.0.1,1433;User Id=SA;Password=yourWeak(!)Password;TrustServerCertificate=True;" - - name: Diagnose SQL Server - if: failure() - shell: bash - run: | - docker inspect --format 'image={{.Config.Image}} imageId={{.Image}} status={{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}} restartCount={{.RestartCount}} oomKilled={{.State.OOMKilled}} exitCode={{.State.ExitCode}} error={{json .State.Error}}' mssql || true - docker inspect --format '{{if .State.Health}}{{range .State.Health.Log}}{{println .Start "exitCode=" .ExitCode .Output}}{{end}}{{end}}' mssql || true - docker logs --timestamps mssql || true - - name: Clean up SQL Server - if: always() - run: docker rm -f mssql || true - test-sqlite: - name: SQLite provider tests - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - framework: ["net8.0", "net10.0"] - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - name: Test - uses: ./.github/actions/run-tests - with: - artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} - framework: ${{ matrix.framework }} - provider: Sqlite - test-azure-storage: - name: Azure Storage provider tests - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - framework: ["net8.0", "net10.0"] - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - name: Start Azurite - run: | - set -euo pipefail - - bash .github/scripts/docker-pull-with-retry.sh "$AZURITE_IMAGE" - - docker run -d --pull=never --name azurite \ - -p 10000:10000 \ - -p 10001:10001 \ - -p 10002:10002 \ - "$AZURITE_IMAGE" \ - azurite --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 --skipApiVersionCheck - - name: Wait for Azurite - run: | - echo "Waiting for Azurite to be ready..." - timeout 60 bash -c 'until nc -z localhost 10000 && nc -z localhost 10001 && nc -z localhost 10002; do sleep 1; done' - echo "Azurite is ready" - - name: Test - uses: ./.github/actions/run-tests - with: - artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} - framework: ${{ matrix.framework }} - provider: AzureStorage - env: -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] -# [SuppressMessage("Microsoft.Security", "CSCAN0090:ConfigFile", Justification="Not a secret")] -# [SuppressMessage("Microsoft.Security", "CSCAN0220:DefaultPasswordContexts", Justification="Not a secret")] - ORLEANSDATACONNECTIONSTRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" - - name: Clean up Azurite - if: always() - run: docker rm -f azurite || true - test-azure-eventhubs: - name: Azure Event Hubs provider tests - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - framework: ["net8.0", "net10.0"] - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - name: Start Azurite - run: | - set -euo pipefail - - bash .github/scripts/docker-pull-with-retry.sh "$AZURITE_IMAGE" - - docker run -d --pull=never --name azurite \ - -p 10000:10000 \ - -p 10001:10001 \ - -p 10002:10002 \ - "$AZURITE_IMAGE" \ - azurite --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 --skipApiVersionCheck - - name: Wait for Azurite - run: | - echo "Waiting for Azurite to be ready..." - timeout 60 bash -c 'until nc -z localhost 10000 && nc -z localhost 10001 && nc -z localhost 10002; do sleep 1; done' - echo "Azurite is ready" - - name: Start Event Hubs emulator - run: | - set -euo pipefail - - bash .github/scripts/docker-pull-with-retry.sh "$EVENTHUBS_EMULATOR_IMAGE" - - docker run -d --pull=never --name eventhubs-emulator \ - -v ${{ github.workspace }}/.github/eventhubs-emulator/Config.json:/Eventhubs_Emulator/ConfigFiles/Config.json \ - -p 5672:5672 \ - -e BLOB_SERVER=host.docker.internal \ - -e METADATA_SERVER=host.docker.internal \ - -e ACCEPT_EULA=Y \ - --add-host=host.docker.internal:host-gateway \ - "$EVENTHUBS_EMULATOR_IMAGE" - - name: Wait for Event Hubs emulator - run: | - echo "Waiting for Event Hubs emulator to be ready..." - timeout 60 bash -c 'until nc -z localhost 5672; do sleep 1; done' - echo "Event Hubs emulator is ready" - - name: Test - uses: ./.github/actions/run-tests - with: - artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} - framework: ${{ matrix.framework }} - provider: EventHub - env: -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] -# [SuppressMessage("Microsoft.Security", "CSCAN0090:ConfigFile", Justification="Not a secret")] -# [SuppressMessage("Microsoft.Security", "CSCAN0220:DefaultPasswordContexts", Justification="Not a secret")] - ORLEANSEVENTHUBCONNECTIONSTRING: "Endpoint=sb://localhost;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=SAS_KEY_VALUE;UseDevelopmentEmulator=true;" -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] - ORLEANSDATACONNECTIONSTRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" - - name: Clean up Event Hubs emulator - if: always() - run: docker rm -f eventhubs-emulator || true - - name: Clean up Azurite - if: always() - run: docker rm -f azurite || true - test-azure-cosmosdb: - name: Azure Cosmos DB provider tests - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - framework: ["net8.0", "net10.0"] - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - name: Start Azure Cosmos DB emulator - run: | - set -euo pipefail - - bash .github/scripts/docker-pull-with-retry.sh "$COSMOSDB_EMULATOR_IMAGE" - - docker run -d --pull=never --name cosmosdb-emulator \ - -p 8081:8081 \ - -e PROTOCOL=https \ - -e ENABLE_EXPLORER=false \ - -e ENABLE_TELEMETRY=false \ - -e LOG_LEVEL=warn \ - "$COSMOSDB_EMULATOR_IMAGE" - - name: Wait for Azure Cosmos DB emulator - run: | - endpoint="https://localhost:8081/" - echo "Waiting for Azure Cosmos DB emulator to be ready..." - for i in {1..60}; do - if curl --silent --show-error --insecure "${endpoint}" > /dev/null; then - echo "Azure Cosmos DB emulator is ready" - echo "ORLEANSCOSMOSDBACCOUNTENDPOINT=${endpoint}" >> "$GITHUB_ENV" - exit 0 - fi - - if [ "$i" -eq 60 ]; then - docker logs --tail 200 cosmosdb-emulator - echo "Azure Cosmos DB emulator did not become ready in time." - exit 1 - fi - - sleep 2 - done - - name: Test - uses: ./.github/actions/run-tests - with: - artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} - framework: ${{ matrix.framework }} - provider: Cosmos - retry: 'true' - env: -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] - #ORLEANSCOSMOSDBACCOUNTENDPOINT: "https://localhost:8081/" -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] - ORLEANSCOSMOSDBACCOUNTKEY: "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==" - - name: Clean up Azure Cosmos DB emulator - if: always() - run: docker rm -f cosmosdb-emulator || true - test-consul: - name: Consul provider tests - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - framework: ["net8.0", "net10.0"] - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - name: Pre-pull Consul test images - shell: bash - run: | - set -euo pipefail - - bash .github/scripts/docker-pull-with-retry.sh "$TESTCONTAINERS_RYUK_IMAGE" - bash .github/scripts/docker-pull-with-retry.sh "$CONSUL_IMAGE" - - name: Test - uses: ./.github/actions/run-tests - with: - artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} - framework: ${{ matrix.framework }} - provider: Consul - test-zookeeper: - name: ZooKeeper provider tests - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - framework: ["net8.0", "net10.0"] - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - name: Start ZooKeeper - shell: bash - run: | - set -euo pipefail - - bash .github/scripts/docker-pull-with-retry.sh "$ZOOKEEPER_IMAGE" - - docker run -d --pull=never --name zookeeper \ - -p 2181:2181 \ - "$ZOOKEEPER_IMAGE" - - echo "Waiting for ZooKeeper to accept TCP connections..." - timeout 60 bash -c 'until nc -z localhost 2181; do sleep 1; done' - echo "ZooKeeper is accepting TCP connections" - - name: Test - uses: ./.github/actions/run-tests - with: - artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} - framework: ${{ matrix.framework }} - provider: ZooKeeper - env: - ORLEANSZOOKEEPERCONNECTIONSTRING: "localhost:2181" - - name: Clean up ZooKeeper - if: always() - run: docker rm -f zookeeper || true - test-dynamodb: - name: AWS DynamoDB provider tests - runs-on: ubuntu-latest - env: -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] - AWS_ACCESS_KEY_ID: root -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] - AWS_SECRET_ACCESS_KEY: pass - AWS_REGION: us-east-1 - strategy: - fail-fast: false - matrix: - framework: ["net8.0", "net10.0"] - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - name: Start DynamoDB Local - shell: bash - run: | - set -euo pipefail - - bash .github/scripts/docker-pull-with-retry.sh "$DYNAMODB_IMAGE" - - docker run -d --pull=never --name dynamodb \ - -p 8000:8000 \ - -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ - -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ - -e AWS_REGION="$AWS_REGION" \ - "$DYNAMODB_IMAGE" - - echo "Waiting for DynamoDB Local to accept TCP connections..." - timeout 60 bash -c 'until nc -z localhost 8000; do sleep 1; done' - echo "DynamoDB Local is accepting TCP connections" - - name: Test - uses: ./.github/actions/run-tests - with: - artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} - framework: ${{ matrix.framework }} - provider: DynamoDB - env: - ORLEANSDYNAMODBSERVICE: "http://127.0.0.1:8000" -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] - ORLEANSDYNAMODBACCESSKEY: "root" -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] - ORLEANSDYNAMODBSECRETKEY: "pass" - - name: Clean up DynamoDB Local - if: always() - run: docker rm -f dynamodb || true - test-s3: - name: AWS S3 provider tests - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - framework: ["net8.0", "net10.0"] - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - name: Pre-pull S3 test images - shell: bash - run: | - set -euo pipefail - - bash .github/scripts/docker-pull-with-retry.sh "$TESTCONTAINERS_RYUK_IMAGE" - bash .github/scripts/docker-pull-with-retry.sh "$SEAWEEDFS_IMAGE" - - name: Test - uses: ./.github/actions/run-tests - with: - artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} - framework: ${{ matrix.framework }} - provider: S3 - test-sqs: - name: AWS SQS provider tests - runs-on: ubuntu-latest - env: -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] - AWS_ACCESS_KEY_ID: root -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] - AWS_SECRET_ACCESS_KEY: pass - AWS_REGION: us-east-1 - strategy: - fail-fast: false - matrix: - framework: ["net8.0", "net10.0"] - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - name: Start DynamoDB Local - shell: bash - run: | - set -euo pipefail - - bash .github/scripts/docker-pull-with-retry.sh "$DYNAMODB_IMAGE" - - docker run -d --pull=never --name dynamodb \ - -p 8000:8000 \ - -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ - -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ - -e AWS_REGION="$AWS_REGION" \ - "$DYNAMODB_IMAGE" - - name: Start ElasticMQ - shell: bash - run: | - set -euo pipefail - - bash .github/scripts/docker-pull-with-retry.sh "$ELASTICMQ_IMAGE" - - docker run -d --pull=never --name elasticmq \ - -p 9324:9324 \ - "$ELASTICMQ_IMAGE" - - name: Wait for ElasticMQ and DynamoDB - run: | - echo "Waiting for ElasticMQ and DynamoDB Local to be ready..." - timeout 60 bash -c 'until nc -z localhost 9324 && nc -z localhost 8000; do sleep 1; done' - echo "ElasticMQ and DynamoDB Local are ready" - - name: Test - uses: ./.github/actions/run-tests - with: - artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} - framework: ${{ matrix.framework }} - provider: SQS - env: - ORLEANSSQSCONNECTIONSTRING: "Service=http://127.0.0.1:9324" - ORLEANSDYNAMODBSERVICE: "http://127.0.0.1:8000" -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] - ORLEANSDYNAMODBACCESSKEY: "root" -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] - ORLEANSDYNAMODBSECRETKEY: "pass" - - name: Clean up ElasticMQ - if: always() - run: docker rm -f elasticmq || true - - name: Clean up DynamoDB Local - if: always() - run: docker rm -f dynamodb || true - test-nats: - name: NATS stream provider tests - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - framework: ["net8.0", "net10.0"] -# services: -# nats: -# image: nats:latest -# ports: -# - 4222:4222 -# - 8222:8222 -# env: -# HTTP_PORT: 8222 - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - name: Start NATS - shell: bash - run: | - set -euo pipefail - - bash .github/scripts/docker-pull-with-retry.sh "$NATS_IMAGE" - - docker run -d --pull=never --name nats \ - -p 4222:4222 \ - -p 8222:8222 \ - "$NATS_IMAGE" \ - --js --http_port=8222 - - name: Wait for NATS - run: | - echo "Waiting for NATS to be ready..." - if ! timeout 60 bash -c 'until docker logs nats 2>&1 | grep -q "Server is ready"; do sleep 1; done'; then - docker logs nats - exit 1 - fi - echo "NATS is ready" - - name: Test - uses: ./.github/actions/run-tests - with: - artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} - framework: ${{ matrix.framework }} - provider: NATS - - name: Clean up container - if: always() - run: docker rm -f nats || true - test-kinesis: - name: AWS Kinesis provider tests - runs-on: ubuntu-latest - env: -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] - AWS_ACCESS_KEY_ID: test -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] - AWS_SECRET_ACCESS_KEY: test - AWS_REGION: us-east-1 - strategy: - fail-fast: false - matrix: - framework: ["net8.0", "net10.0"] - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - name: Start Ministack - shell: bash - run: | - set -euo pipefail - - bash .github/scripts/docker-pull-with-retry.sh "$MINISTACK_IMAGE" - - docker run -d --pull=never --name ministack \ - -p 4566:4566 \ - -e MINISTACK_REGION="$AWS_REGION" \ - "$MINISTACK_IMAGE" - - name: Start DynamoDB Local - shell: bash - run: | - set -euo pipefail - - bash .github/scripts/docker-pull-with-retry.sh "$DYNAMODB_IMAGE" - - docker run -d --pull=never --name dynamodb \ - -p 8000:8000 \ - -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ - -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ - -e AWS_REGION="$AWS_REGION" \ - "$DYNAMODB_IMAGE" - - name: Wait for Ministack and DynamoDB Local - run: | - echo "Waiting for Ministack and DynamoDB Local to be ready..." - timeout 60 bash -c 'until curl --fail --silent http://localhost:4566/_ministack/health > /dev/null && nc -z localhost 8000; do sleep 1; done' - echo "Ministack and DynamoDB Local are ready" - - name: Test - uses: ./.github/actions/run-tests - with: - artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} - framework: ${{ matrix.framework }} - provider: Kinesis - env: - ORLEANSKINESISCONNECTIONSTRING: "http://localhost:4566;test;test;us-east-1" - ORLEANSDYNAMODBSERVICE: "http://localhost:8000" -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] - ORLEANSDYNAMODBACCESSKEY: "test" -# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] - ORLEANSDYNAMODBSECRETKEY: "test" - - name: Clean up Ministack - if: always() - run: docker rm -f ministack || true - - name: Clean up DynamoDB Local - if: always() - run: docker rm -f dynamodb || true - test: - name: Test - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - suite: [BVT, SlowBVT, Functional] - os: [ubuntu-latest, windows-latest] - framework: ["net8.0", "net10.0"] - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - name: Setup Node.js - if: matrix.suite == 'BVT' - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5 - with: - node-version: '24.x' - - name: Test - uses: ./.github/actions/run-tests - with: - artifact-name: test_output_${{ github.job }}_${{ matrix.suite }}_${{ matrix.os }}_${{ matrix.framework }} - build-external-assets: ${{ matrix.suite == 'BVT' && 'true' || 'false' }} - filter-query: /[(Provider=None)&(Suite=${{ matrix.suite }})&(Area!=CodeGen)] - framework: ${{ matrix.framework }} - test-codegenerator: - name: Test Code Generator - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest] - framework: ["net8.0", "net10.0"] - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - name: Test Code Generator - uses: ./.github/actions/run-tests - with: - artifact-name: test_output_${{ github.job }}_${{ matrix.os }}_${{ matrix.framework }} - filter-query: /[(Provider=None)&(Area=CodeGen)&((Suite=BVT)|(Suite=SlowBVT)|(Suite=Functional))] - framework: ${{ matrix.framework }} - test-google: - name: Google Cloud providers tests - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - framework: ["net8.0", "net10.0"] - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - name: Setup Node.js - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5 - with: - node-version: '24.x' - - name: Setup Java - uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5 - with: - distribution: temurin - java-version: '21' - - name: Setup Google Emulators - working-directory: test/Extensions/Google/Emulators - run: | - set -euo pipefail - - npm install --global firebase-tools@15.24.0 - firebase setup:emulators:storage - firebase setup:emulators:firestore - firebase setup:emulators:pubsub - firebase emulators:start --only firestore,pubsub,storage --project orleans-test > firebase-emulators.log 2>&1 & - emulator_pid=$! - - for attempt in {1..60}; do - if ! kill -0 "$emulator_pid" 2>/dev/null; then - cat firebase-emulators.log - exit 1 - fi - - ready=true - for port in 8080 8085 9199; do - if ! (echo > /dev/tcp/127.0.0.1/$port) 2>/dev/null; then - ready=false - break - fi - done - - if $ready; then - exit 0 - fi - - sleep 1 - done - - cat firebase-emulators.log - exit 1 - - name: Test - uses: ./.github/actions/run-tests - with: - artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} - framework: ${{ matrix.framework }} - provider: GoogleCloud - ci: - name: CI - if: always() - needs: - - build - - test-redis - - test-cassandra - - test-postgres - - test-mariadb - - test-sqlserver - - test-sqlite - - test-azure-storage - - test-azure-eventhubs - - test-azure-cosmosdb - - test-consul - - test-zookeeper - - test-dynamodb - - test-sqs - - test-nats - - test-kinesis - - test - - test-codegenerator - - test-google - runs-on: ubuntu-latest - steps: - - name: Verify all CI jobs succeeded - env: - JOB_RESULTS: ${{ toJSON(needs) }} - run: jq --exit-status 'all(.[]; .result == "success")' <<< "$JOB_RESULTS" + measure: + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + persist-credentials: false + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + repository: dotnet/orleans + ref: 6739589254b746a8790cf53524e6abe372bb53d4 + path: .dissemination-runtime-original + persist-credentials: false + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + repository: ReubenBond/orleans + ref: 3fc0a4610cd2eb5feef27e958fcca59658aaae47 + path: .dissemination-runtime-candidate + persist-credentials: false + - uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5 + with: + global-json-file: global.json + - name: Check script boundaries + shell: pwsh + run: ./.github/scripts/test-dissemination-performance.ps1 + - name: Build and measure bounded all-path workloads + shell: pwsh + run: | + ./.github/scripts/dissemination-performance.ps1 ` + -CandidateRepository ReubenBond/orleans ` + -CandidateRef 3fc0a4610cd2eb5feef27e958fcca59658aaae47 ` + -Sizes '4,8' -Iterations 3 -Repetitions 2 -RuntimePaths All ` + -Scenarios 'stable,churn,partition' -SiloProcessorCount 0 -GCConserveMemory 0 + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + if: always() + with: + name: dissemination-performance-results + retention-days: 30 + path: | + Artifacts/DisseminationPerformance/results + Artifacts/DisseminationPerformance/*.binlog + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + if: always() + with: + name: dissemination-performance-binaries + retention-days: 7 + path: Artifacts/DisseminationPerformance/bin From 80e0cf4c227c79717279643202cd132c648d5897 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 16 Sep 2026 09:11:11 -0700 Subject: [PATCH 03/12] ci: restore standard workflow after manual dispatch --- .github/workflows/ci.yml | 943 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 895 insertions(+), 48 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c943bdda4a..20a90248ade 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,56 +1,903 @@ -name: Dissemination performance bootstrap +name: .NET CI on: workflow_dispatch: + push: + branches: + - main + pull_request: + branches: + - main + merge_group: + types: + - checks_requested permissions: contents: read env: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 DOTNET_NOLOGO: true + DOTNET_COVERAGE_VERSION: 18.5.2 + # Docker images started by workflow steps are centralized so pulls can be retried and mirrors can be swapped in one place. + AZURITE_IMAGE: mcr.microsoft.com/azure-storage/azurite:3.35.0@sha256:647c63a91102a9d8e8000aab803436e1fc85fbb285e7ce830a82ee5d6661cf37 + EVENTHUBS_EMULATOR_IMAGE: mcr.microsoft.com/azure-messaging/eventhubs-emulator:2.2.0@sha256:2c8e0d4dd93a5fc078df2721eeb3e211442d555d61293ac3972df931c6d9333a + COSMOSDB_EMULATOR_IMAGE: mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:vnext-preview@sha256:bf3bd3cbe3fae2005a0745f77b01d1fc8cd04112193f3ee32289ee15ee9ae5fa + CONSUL_IMAGE: public.ecr.aws/hashicorp/consul:1.19 + DYNAMODB_IMAGE: amazon/dynamodb-local:latest + ELASTICMQ_IMAGE: softwaremill/elasticmq-native:latest + # Keep in sync with SeaweedFSTestContainer + # so the fixtures use the pre-pulled image. + SEAWEEDFS_IMAGE: ghcr.io/chrislusf/seaweedfs:4.46@sha256:08d516132314207d10c8e37cbffc1f32b147d870169688734cc61c6231625b62 + MINISTACK_IMAGE: ministackorg/ministack:1.4.11@sha256:8b05715b9effc23d998d4a4e0c0a0c4af7f9435f0ff076a30cf27a77326f14b1 + MSSQL_IMAGE: mcr.microsoft.com/mssql/server:2022-CU26-ubuntu-22.04@sha256:ba4c8329f48fb8f02e1416be6a930ebfd71268caee78aa985f3af4315e457c89 + NATS_IMAGE: nats:latest + POSTGRES_IMAGE: postgres + REDIS_IMAGE: redis + TESTCONTAINERS_RYUK_IMAGE: testcontainers/ryuk:0.14.0 + ZOOKEEPER_IMAGE: zookeeper:3.9 jobs: - measure: - runs-on: ubuntu-latest - timeout-minutes: 90 - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - with: - persist-credentials: false - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - with: - repository: dotnet/orleans - ref: 6739589254b746a8790cf53524e6abe372bb53d4 - path: .dissemination-runtime-original - persist-credentials: false - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - with: - repository: ReubenBond/orleans - ref: 3fc0a4610cd2eb5feef27e958fcca59658aaae47 - path: .dissemination-runtime-candidate - persist-credentials: false - - uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5 - with: - global-json-file: global.json - - name: Check script boundaries - shell: pwsh - run: ./.github/scripts/test-dissemination-performance.ps1 - - name: Build and measure bounded all-path workloads - shell: pwsh - run: | - ./.github/scripts/dissemination-performance.ps1 ` - -CandidateRepository ReubenBond/orleans ` - -CandidateRef 3fc0a4610cd2eb5feef27e958fcca59658aaae47 ` - -Sizes '4,8' -Iterations 3 -Repetitions 2 -RuntimePaths All ` - -Scenarios 'stable,churn,partition' -SiloProcessorCount 0 -GCConserveMemory 0 - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - if: always() - with: - name: dissemination-performance-results - retention-days: 30 - path: | - Artifacts/DisseminationPerformance/results - Artifacts/DisseminationPerformance/*.binlog - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - if: always() - with: - name: dissemination-performance-binaries - retention-days: 7 - path: Artifacts/DisseminationPerformance/bin + whitespace: + name: Whitespace formatting + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + fetch-depth: 0 + - name: Setup .NET + uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5 + with: + global-json-file: global.json + - name: Verify .NET formatter + run: dotnet format --version + - name: Check solution + run: dotnet format whitespace Orleans.slnx --verify-no-changes --verbosity normal + build: + name: Build + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Setup Node.js + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5 + with: + node-version: '24.x' + - name: Setup .NET + uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5 + with: + global-json-file: global.json + - name: Test coverage reporting + if: matrix.os == 'ubuntu-latest' + shell: pwsh + run: ./.github/scripts/test-summarize-coverage.ps1 + - name: Build + run: dotnet build -bl + - name: Pack and validate package compatibility + if: matrix.os == 'ubuntu-latest' + run: dotnet pack Orleans.slnx --configuration Debug --no-build --no-restore -bl:package_validation.binlog + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + continue-on-error: true + with: + name: build_log_${{ matrix.os }} + retention-days: 1 + path: | + **/*.binlog + test-redis: + name: Redis provider tests + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + framework: ["net8.0", "net10.0"] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Start Redis + shell: bash + run: | + set -euo pipefail + + bash .github/scripts/docker-pull-with-retry.sh "$REDIS_IMAGE" + + docker run -d --pull=never --name redis \ + -p 6379:6379 \ + --health-cmd "redis-cli ping" \ + --health-interval 10s \ + --health-timeout 5s \ + --health-retries 5 \ + "$REDIS_IMAGE" + + bash .github/scripts/docker-wait-for-health.sh redis + - name: Test + uses: ./.github/actions/run-tests + with: + artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} + framework: ${{ matrix.framework }} + provider: Redis + env: + ORLEANSREDISCONNECTIONSTRING: "localhost:6379,ssl=False,abortConnect=False" + - name: Clean up Redis + if: always() + run: docker rm -f redis || true + test-cassandra: + name: Cassandra provider tests + runs-on: ubuntu-latest + env: + CASSANDRA_IMAGE: cassandra:${{ matrix.dbversion }} + strategy: + fail-fast: false + matrix: + dbversion: ["4.0", "4.1", "5.0"] + framework: ["net8.0", "net10.0"] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Pre-pull Cassandra test images + shell: bash + run: | + set -euo pipefail + + bash .github/scripts/docker-pull-with-retry.sh "$TESTCONTAINERS_RYUK_IMAGE" + bash .github/scripts/docker-pull-with-retry.sh "$CASSANDRA_IMAGE" + - name: Test + uses: ./.github/actions/run-tests + with: + artifact-name: test_output_${{ github.job }}_${{ matrix.dbversion }}_${{ matrix.framework }} + framework: ${{ matrix.framework }} + provider: Cassandra + env: + CASSANDRAVERSION: ${{ matrix.dbversion }} + test-postgres: + name: PostgreSQL provider tests + runs-on: ubuntu-latest + env: +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] + POSTGRES_PASSWORD: postgres + strategy: + fail-fast: false + matrix: + framework: ["net8.0", "net10.0"] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Start PostgreSQL + shell: bash + run: | + set -euo pipefail + + bash .github/scripts/docker-pull-with-retry.sh "$POSTGRES_IMAGE" + + docker run -d --pull=never --name postgres \ + -p 5432:5432 \ + -e POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ + --health-cmd "pg_isready -U postgres" \ + --health-interval 10s \ + --health-timeout 5s \ + --health-retries 5 \ + "$POSTGRES_IMAGE" \ + -c max_connections=200 + + bash .github/scripts/docker-wait-for-health.sh postgres + - name: Test + uses: ./.github/actions/run-tests + with: + artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} + framework: ${{ matrix.framework }} + provider: PostgreSql + env: +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="False positive")] + ORLEANSPOSTGRESCONNECTIONSTRING: "Server=127.0.0.1;Port=5432;Pooling=false;User Id=postgres;Password=postgres;SSL Mode=Disable" + - name: Clean up PostgreSQL + if: always() + run: docker rm -f postgres || true + test-mariadb: + name: MariaDB/MySQL provider tests + runs-on: ubuntu-latest + env: + # The official MariaDB 10.6 image is not published to GHCR. Prefer Google's Docker Hub + # mirror, with Docker Hub as a fallback, and pin both references to the same digest. + MARIADB_IMAGE: mirror.gcr.io/library/mariadb:10.6@sha256:daacc2f260f8ec999daa5e03a017a23a7e6fa3fb982aaf26e8b72f24daf03bc9 + MARIADB_FALLBACK_IMAGE: docker.io/library/mariadb:10.6@sha256:daacc2f260f8ec999daa5e03a017a23a7e6fa3fb982aaf26e8b72f24daf03bc9 +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] + MARIADB_ROOT_PASSWORD: "mariadb" + strategy: + fail-fast: false + matrix: + framework: ["net8.0", "net10.0"] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Start MariaDB + shell: bash + run: | + set -euo pipefail + + mariadb_image="$MARIADB_IMAGE" + if ! bash .github/scripts/docker-pull-with-retry.sh "$mariadb_image"; then + echo "Pulling MariaDB from Docker Hub after the mirror failed." + mariadb_image="$MARIADB_FALLBACK_IMAGE" + bash .github/scripts/docker-pull-with-retry.sh "$mariadb_image" + fi + + docker run -d --pull=never --name mariadb \ + -p 3306:3306 \ + -e MARIADB_ROOT_PASSWORD="$MARIADB_ROOT_PASSWORD" \ + --health-cmd "mariadb-admin ping -h 127.0.0.1 -uroot -p${MARIADB_ROOT_PASSWORD} --silent" \ + --health-interval 5s \ + --health-timeout 5s \ + --health-retries 24 \ + "$mariadb_image" + + for attempt in $(seq 1 60); do + status="$(docker inspect --format '{{.State.Health.Status}}' mariadb)" + case "$status" in + healthy) + exit 0 + ;; + unhealthy) + docker logs mariadb + exit 1 + ;; + esac + + sleep 2 + done + + echo "MariaDB did not become healthy in time." + docker logs mariadb + exit 1 + - name: Test + uses: ./.github/actions/run-tests + with: + artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} + framework: ${{ matrix.framework }} + provider: MySql + env: +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] + ORLEANSMYSQLCONNECTIONSTRING: "Server=127.0.0.1;Port=3306;UId=root;Pwd=mariadb;" + test-sqlserver: + name: Microsoft SQL Server provider tests + runs-on: ubuntu-latest + env: + ACCEPT_EULA: "Y" + MSSQL_PID: "Developer" +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] + SA_PASSWORD: "yourWeak(!)Password" + strategy: + fail-fast: false + matrix: + framework: ["net8.0", "net10.0"] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Start SQL Server + shell: bash + run: | + set -euo pipefail + + bash .github/scripts/docker-pull-with-retry.sh "$MSSQL_IMAGE" + + docker run -d --pull=never --name mssql \ + -p 1433:1433 \ + -e ACCEPT_EULA="$ACCEPT_EULA" \ + -e MSSQL_PID="$MSSQL_PID" \ + -e MSSQL_SA_PASSWORD="$SA_PASSWORD" \ + --health-cmd '/opt/mssql-tools18/bin/sqlcmd -C -S localhost -U sa -P "$MSSQL_SA_PASSWORD" -Q "SELECT 1" -b -o /dev/null' \ + --health-interval 2s \ + --health-timeout 5s \ + --health-retries 60 \ + "$MSSQL_IMAGE" + + bash .github/scripts/docker-wait-for-health.sh mssql 120 + - name: Test + uses: ./.github/actions/run-tests + with: + artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} + framework: ${{ matrix.framework }} + provider: SqlServer + env: +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] + ORLEANSMSSQLCONNECTIONSTRING: "Server=127.0.0.1,1433;User Id=SA;Password=yourWeak(!)Password;TrustServerCertificate=True;" + - name: Diagnose SQL Server + if: failure() + shell: bash + run: | + docker inspect --format 'image={{.Config.Image}} imageId={{.Image}} status={{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}} restartCount={{.RestartCount}} oomKilled={{.State.OOMKilled}} exitCode={{.State.ExitCode}} error={{json .State.Error}}' mssql || true + docker inspect --format '{{if .State.Health}}{{range .State.Health.Log}}{{println .Start "exitCode=" .ExitCode .Output}}{{end}}{{end}}' mssql || true + docker logs --timestamps mssql || true + - name: Clean up SQL Server + if: always() + run: docker rm -f mssql || true + test-sqlite: + name: SQLite provider tests + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + framework: ["net8.0", "net10.0"] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Test + uses: ./.github/actions/run-tests + with: + artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} + framework: ${{ matrix.framework }} + provider: Sqlite + test-azure-storage: + name: Azure Storage provider tests + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + framework: ["net8.0", "net10.0"] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Start Azurite + run: | + set -euo pipefail + + bash .github/scripts/docker-pull-with-retry.sh "$AZURITE_IMAGE" + + docker run -d --pull=never --name azurite \ + -p 10000:10000 \ + -p 10001:10001 \ + -p 10002:10002 \ + "$AZURITE_IMAGE" \ + azurite --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 --skipApiVersionCheck + - name: Wait for Azurite + run: | + echo "Waiting for Azurite to be ready..." + timeout 60 bash -c 'until nc -z localhost 10000 && nc -z localhost 10001 && nc -z localhost 10002; do sleep 1; done' + echo "Azurite is ready" + - name: Test + uses: ./.github/actions/run-tests + with: + artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} + framework: ${{ matrix.framework }} + provider: AzureStorage + env: +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] +# [SuppressMessage("Microsoft.Security", "CSCAN0090:ConfigFile", Justification="Not a secret")] +# [SuppressMessage("Microsoft.Security", "CSCAN0220:DefaultPasswordContexts", Justification="Not a secret")] + ORLEANSDATACONNECTIONSTRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" + - name: Clean up Azurite + if: always() + run: docker rm -f azurite || true + test-azure-eventhubs: + name: Azure Event Hubs provider tests + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + framework: ["net8.0", "net10.0"] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Start Azurite + run: | + set -euo pipefail + + bash .github/scripts/docker-pull-with-retry.sh "$AZURITE_IMAGE" + + docker run -d --pull=never --name azurite \ + -p 10000:10000 \ + -p 10001:10001 \ + -p 10002:10002 \ + "$AZURITE_IMAGE" \ + azurite --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 --skipApiVersionCheck + - name: Wait for Azurite + run: | + echo "Waiting for Azurite to be ready..." + timeout 60 bash -c 'until nc -z localhost 10000 && nc -z localhost 10001 && nc -z localhost 10002; do sleep 1; done' + echo "Azurite is ready" + - name: Start Event Hubs emulator + run: | + set -euo pipefail + + bash .github/scripts/docker-pull-with-retry.sh "$EVENTHUBS_EMULATOR_IMAGE" + + docker run -d --pull=never --name eventhubs-emulator \ + -v ${{ github.workspace }}/.github/eventhubs-emulator/Config.json:/Eventhubs_Emulator/ConfigFiles/Config.json \ + -p 5672:5672 \ + -e BLOB_SERVER=host.docker.internal \ + -e METADATA_SERVER=host.docker.internal \ + -e ACCEPT_EULA=Y \ + --add-host=host.docker.internal:host-gateway \ + "$EVENTHUBS_EMULATOR_IMAGE" + - name: Wait for Event Hubs emulator + run: | + echo "Waiting for Event Hubs emulator to be ready..." + timeout 60 bash -c 'until nc -z localhost 5672; do sleep 1; done' + echo "Event Hubs emulator is ready" + - name: Test + uses: ./.github/actions/run-tests + with: + artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} + framework: ${{ matrix.framework }} + provider: EventHub + env: +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] +# [SuppressMessage("Microsoft.Security", "CSCAN0090:ConfigFile", Justification="Not a secret")] +# [SuppressMessage("Microsoft.Security", "CSCAN0220:DefaultPasswordContexts", Justification="Not a secret")] + ORLEANSEVENTHUBCONNECTIONSTRING: "Endpoint=sb://localhost;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=SAS_KEY_VALUE;UseDevelopmentEmulator=true;" +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] + ORLEANSDATACONNECTIONSTRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" + - name: Clean up Event Hubs emulator + if: always() + run: docker rm -f eventhubs-emulator || true + - name: Clean up Azurite + if: always() + run: docker rm -f azurite || true + test-azure-cosmosdb: + name: Azure Cosmos DB provider tests + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + framework: ["net8.0", "net10.0"] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Start Azure Cosmos DB emulator + run: | + set -euo pipefail + + bash .github/scripts/docker-pull-with-retry.sh "$COSMOSDB_EMULATOR_IMAGE" + + docker run -d --pull=never --name cosmosdb-emulator \ + -p 8081:8081 \ + -e PROTOCOL=https \ + -e ENABLE_EXPLORER=false \ + -e ENABLE_TELEMETRY=false \ + -e LOG_LEVEL=warn \ + "$COSMOSDB_EMULATOR_IMAGE" + - name: Wait for Azure Cosmos DB emulator + run: | + endpoint="https://localhost:8081/" + echo "Waiting for Azure Cosmos DB emulator to be ready..." + for i in {1..60}; do + if curl --silent --show-error --insecure "${endpoint}" > /dev/null; then + echo "Azure Cosmos DB emulator is ready" + echo "ORLEANSCOSMOSDBACCOUNTENDPOINT=${endpoint}" >> "$GITHUB_ENV" + exit 0 + fi + + if [ "$i" -eq 60 ]; then + docker logs --tail 200 cosmosdb-emulator + echo "Azure Cosmos DB emulator did not become ready in time." + exit 1 + fi + + sleep 2 + done + - name: Test + uses: ./.github/actions/run-tests + with: + artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} + framework: ${{ matrix.framework }} + provider: Cosmos + retry: 'true' + env: +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] + #ORLEANSCOSMOSDBACCOUNTENDPOINT: "https://localhost:8081/" +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] + ORLEANSCOSMOSDBACCOUNTKEY: "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==" + - name: Clean up Azure Cosmos DB emulator + if: always() + run: docker rm -f cosmosdb-emulator || true + test-consul: + name: Consul provider tests + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + framework: ["net8.0", "net10.0"] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Pre-pull Consul test images + shell: bash + run: | + set -euo pipefail + + bash .github/scripts/docker-pull-with-retry.sh "$TESTCONTAINERS_RYUK_IMAGE" + bash .github/scripts/docker-pull-with-retry.sh "$CONSUL_IMAGE" + - name: Test + uses: ./.github/actions/run-tests + with: + artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} + framework: ${{ matrix.framework }} + provider: Consul + test-zookeeper: + name: ZooKeeper provider tests + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + framework: ["net8.0", "net10.0"] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Start ZooKeeper + shell: bash + run: | + set -euo pipefail + + bash .github/scripts/docker-pull-with-retry.sh "$ZOOKEEPER_IMAGE" + + docker run -d --pull=never --name zookeeper \ + -p 2181:2181 \ + "$ZOOKEEPER_IMAGE" + + echo "Waiting for ZooKeeper to accept TCP connections..." + timeout 60 bash -c 'until nc -z localhost 2181; do sleep 1; done' + echo "ZooKeeper is accepting TCP connections" + - name: Test + uses: ./.github/actions/run-tests + with: + artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} + framework: ${{ matrix.framework }} + provider: ZooKeeper + env: + ORLEANSZOOKEEPERCONNECTIONSTRING: "localhost:2181" + - name: Clean up ZooKeeper + if: always() + run: docker rm -f zookeeper || true + test-dynamodb: + name: AWS DynamoDB provider tests + runs-on: ubuntu-latest + env: +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] + AWS_ACCESS_KEY_ID: root +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] + AWS_SECRET_ACCESS_KEY: pass + AWS_REGION: us-east-1 + strategy: + fail-fast: false + matrix: + framework: ["net8.0", "net10.0"] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Start DynamoDB Local + shell: bash + run: | + set -euo pipefail + + bash .github/scripts/docker-pull-with-retry.sh "$DYNAMODB_IMAGE" + + docker run -d --pull=never --name dynamodb \ + -p 8000:8000 \ + -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ + -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ + -e AWS_REGION="$AWS_REGION" \ + "$DYNAMODB_IMAGE" + + echo "Waiting for DynamoDB Local to accept TCP connections..." + timeout 60 bash -c 'until nc -z localhost 8000; do sleep 1; done' + echo "DynamoDB Local is accepting TCP connections" + - name: Test + uses: ./.github/actions/run-tests + with: + artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} + framework: ${{ matrix.framework }} + provider: DynamoDB + env: + ORLEANSDYNAMODBSERVICE: "http://127.0.0.1:8000" +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] + ORLEANSDYNAMODBACCESSKEY: "root" +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] + ORLEANSDYNAMODBSECRETKEY: "pass" + - name: Clean up DynamoDB Local + if: always() + run: docker rm -f dynamodb || true + test-s3: + name: AWS S3 provider tests + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + framework: ["net8.0", "net10.0"] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Pre-pull S3 test images + shell: bash + run: | + set -euo pipefail + + bash .github/scripts/docker-pull-with-retry.sh "$TESTCONTAINERS_RYUK_IMAGE" + bash .github/scripts/docker-pull-with-retry.sh "$SEAWEEDFS_IMAGE" + - name: Test + uses: ./.github/actions/run-tests + with: + artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} + framework: ${{ matrix.framework }} + provider: S3 + test-sqs: + name: AWS SQS provider tests + runs-on: ubuntu-latest + env: +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] + AWS_ACCESS_KEY_ID: root +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] + AWS_SECRET_ACCESS_KEY: pass + AWS_REGION: us-east-1 + strategy: + fail-fast: false + matrix: + framework: ["net8.0", "net10.0"] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Start DynamoDB Local + shell: bash + run: | + set -euo pipefail + + bash .github/scripts/docker-pull-with-retry.sh "$DYNAMODB_IMAGE" + + docker run -d --pull=never --name dynamodb \ + -p 8000:8000 \ + -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ + -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ + -e AWS_REGION="$AWS_REGION" \ + "$DYNAMODB_IMAGE" + - name: Start ElasticMQ + shell: bash + run: | + set -euo pipefail + + bash .github/scripts/docker-pull-with-retry.sh "$ELASTICMQ_IMAGE" + + docker run -d --pull=never --name elasticmq \ + -p 9324:9324 \ + "$ELASTICMQ_IMAGE" + - name: Wait for ElasticMQ and DynamoDB + run: | + echo "Waiting for ElasticMQ and DynamoDB Local to be ready..." + timeout 60 bash -c 'until nc -z localhost 9324 && nc -z localhost 8000; do sleep 1; done' + echo "ElasticMQ and DynamoDB Local are ready" + - name: Test + uses: ./.github/actions/run-tests + with: + artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} + framework: ${{ matrix.framework }} + provider: SQS + env: + ORLEANSSQSCONNECTIONSTRING: "Service=http://127.0.0.1:9324" + ORLEANSDYNAMODBSERVICE: "http://127.0.0.1:8000" +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] + ORLEANSDYNAMODBACCESSKEY: "root" +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] + ORLEANSDYNAMODBSECRETKEY: "pass" + - name: Clean up ElasticMQ + if: always() + run: docker rm -f elasticmq || true + - name: Clean up DynamoDB Local + if: always() + run: docker rm -f dynamodb || true + test-nats: + name: NATS stream provider tests + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + framework: ["net8.0", "net10.0"] +# services: +# nats: +# image: nats:latest +# ports: +# - 4222:4222 +# - 8222:8222 +# env: +# HTTP_PORT: 8222 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Start NATS + shell: bash + run: | + set -euo pipefail + + bash .github/scripts/docker-pull-with-retry.sh "$NATS_IMAGE" + + docker run -d --pull=never --name nats \ + -p 4222:4222 \ + -p 8222:8222 \ + "$NATS_IMAGE" \ + --js --http_port=8222 + - name: Wait for NATS + run: | + echo "Waiting for NATS to be ready..." + if ! timeout 60 bash -c 'until docker logs nats 2>&1 | grep -q "Server is ready"; do sleep 1; done'; then + docker logs nats + exit 1 + fi + echo "NATS is ready" + - name: Test + uses: ./.github/actions/run-tests + with: + artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} + framework: ${{ matrix.framework }} + provider: NATS + - name: Clean up container + if: always() + run: docker rm -f nats || true + test-kinesis: + name: AWS Kinesis provider tests + runs-on: ubuntu-latest + env: +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] + AWS_ACCESS_KEY_ID: test +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] + AWS_SECRET_ACCESS_KEY: test + AWS_REGION: us-east-1 + strategy: + fail-fast: false + matrix: + framework: ["net8.0", "net10.0"] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Start Ministack + shell: bash + run: | + set -euo pipefail + + bash .github/scripts/docker-pull-with-retry.sh "$MINISTACK_IMAGE" + + docker run -d --pull=never --name ministack \ + -p 4566:4566 \ + -e MINISTACK_REGION="$AWS_REGION" \ + "$MINISTACK_IMAGE" + - name: Start DynamoDB Local + shell: bash + run: | + set -euo pipefail + + bash .github/scripts/docker-pull-with-retry.sh "$DYNAMODB_IMAGE" + + docker run -d --pull=never --name dynamodb \ + -p 8000:8000 \ + -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ + -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ + -e AWS_REGION="$AWS_REGION" \ + "$DYNAMODB_IMAGE" + - name: Wait for Ministack and DynamoDB Local + run: | + echo "Waiting for Ministack and DynamoDB Local to be ready..." + timeout 60 bash -c 'until curl --fail --silent http://localhost:4566/_ministack/health > /dev/null && nc -z localhost 8000; do sleep 1; done' + echo "Ministack and DynamoDB Local are ready" + - name: Test + uses: ./.github/actions/run-tests + with: + artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} + framework: ${{ matrix.framework }} + provider: Kinesis + env: + ORLEANSKINESISCONNECTIONSTRING: "http://localhost:4566;test;test;us-east-1" + ORLEANSDYNAMODBSERVICE: "http://localhost:8000" +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] + ORLEANSDYNAMODBACCESSKEY: "test" +# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")] + ORLEANSDYNAMODBSECRETKEY: "test" + - name: Clean up Ministack + if: always() + run: docker rm -f ministack || true + - name: Clean up DynamoDB Local + if: always() + run: docker rm -f dynamodb || true + test: + name: Test + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + suite: [BVT, SlowBVT, Functional] + os: [ubuntu-latest, windows-latest] + framework: ["net8.0", "net10.0"] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Setup Node.js + if: matrix.suite == 'BVT' + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5 + with: + node-version: '24.x' + - name: Test + uses: ./.github/actions/run-tests + with: + artifact-name: test_output_${{ github.job }}_${{ matrix.suite }}_${{ matrix.os }}_${{ matrix.framework }} + build-external-assets: ${{ matrix.suite == 'BVT' && 'true' || 'false' }} + filter-query: /[(Provider=None)&(Suite=${{ matrix.suite }})&(Area!=CodeGen)] + framework: ${{ matrix.framework }} + test-codegenerator: + name: Test Code Generator + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + framework: ["net8.0", "net10.0"] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Test Code Generator + uses: ./.github/actions/run-tests + with: + artifact-name: test_output_${{ github.job }}_${{ matrix.os }}_${{ matrix.framework }} + filter-query: /[(Provider=None)&(Area=CodeGen)&((Suite=BVT)|(Suite=SlowBVT)|(Suite=Functional))] + framework: ${{ matrix.framework }} + test-google: + name: Google Cloud providers tests + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + framework: ["net8.0", "net10.0"] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Setup Node.js + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5 + with: + node-version: '24.x' + - name: Setup Java + uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5 + with: + distribution: temurin + java-version: '21' + - name: Setup Google Emulators + working-directory: test/Extensions/Google/Emulators + run: | + set -euo pipefail + + npm install --global firebase-tools@15.24.0 + firebase setup:emulators:storage + firebase setup:emulators:firestore + firebase setup:emulators:pubsub + firebase emulators:start --only firestore,pubsub,storage --project orleans-test > firebase-emulators.log 2>&1 & + emulator_pid=$! + + for attempt in {1..60}; do + if ! kill -0 "$emulator_pid" 2>/dev/null; then + cat firebase-emulators.log + exit 1 + fi + + ready=true + for port in 8080 8085 9199; do + if ! (echo > /dev/tcp/127.0.0.1/$port) 2>/dev/null; then + ready=false + break + fi + done + + if $ready; then + exit 0 + fi + + sleep 1 + done + + cat firebase-emulators.log + exit 1 + - name: Test + uses: ./.github/actions/run-tests + with: + artifact-name: test_output_${{ github.job }}_${{ matrix.framework }} + framework: ${{ matrix.framework }} + provider: GoogleCloud + ci: + name: CI + if: always() + needs: + - build + - test-redis + - test-cassandra + - test-postgres + - test-mariadb + - test-sqlserver + - test-sqlite + - test-azure-storage + - test-azure-eventhubs + - test-azure-cosmosdb + - test-consul + - test-zookeeper + - test-dynamodb + - test-sqs + - test-nats + - test-kinesis + - test + - test-codegenerator + - test-google + runs-on: ubuntu-latest + steps: + - name: Verify all CI jobs succeeded + env: + JOB_RESULTS: ${{ toJSON(needs) }} + run: jq --exit-status 'all(.[]; .result == "success")' <<< "$JOB_RESULTS" From 0a0fb005e67dabdacd51c1b94c7c12d5668882ce Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 16 Sep 2026 09:56:59 -0700 Subject: [PATCH 04/12] fix: align performance harness paths and membership cleanup --- .github/scripts/dissemination-performance.ps1 | 25 ++-- .../test-dissemination-performance.ps1 | 79 ++++++++++++ .github/workflows/ci.yml | 5 + .../ProviderTests/FileMembershipTableTests.cs | 116 ++++++++++++++++++ .../README.md | 25 +++- .../Silo/FileMembershipTable.cs | 4 +- .../methodology.json | 2 +- .../Orleans.Runtime.Tests.csproj | 6 + 8 files changed, 242 insertions(+), 20 deletions(-) create mode 100644 test/Dissemination.PerformanceHarness/ProviderTests/FileMembershipTableTests.cs diff --git a/.github/scripts/dissemination-performance.ps1 b/.github/scripts/dissemination-performance.ps1 index 004061b11a0..6c85dbf2113 100644 --- a/.github/scripts/dissemination-performance.ps1 +++ b/.github/scripts/dissemination-performance.ps1 @@ -7,7 +7,7 @@ param( [ValidatePattern('^[A-Za-z0-9][A-Za-z0-9._/-]{0,199}$')] [string] $CandidateRef, [ValidatePattern('^Artifacts[\\/][A-Za-z0-9][A-Za-z0-9._-]{0,79}$')] - [string] $ArtifactsDirectory = 'Artifacts\DisseminationPerformance', + [string] $ArtifactsDirectory = (Join-Path 'Artifacts' 'DisseminationPerformance'), [ValidatePattern('^\d+(,\d+){0,5}$')] [string] $Sizes = '4,8', [ValidateRange(3, 200)] @@ -28,8 +28,8 @@ param( $ErrorActionPreference = 'Stop' Set-StrictMode -Version Latest $baselineSha = '6739589254b746a8790cf53524e6abe372bb53d4' -$root = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..\..')) -$harnessSource = Join-Path $root 'test\Dissemination.PerformanceHarness' +$root = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..' '..')) +$harnessSource = Join-Path $root 'test' 'Dissemination.PerformanceHarness' $sizesArray = @($Sizes.Split(',') | ForEach-Object { [int]$_ }) if (@($sizesArray | Select-Object -Unique).Count -ne $sizesArray.Count -or @($sizesArray | Where-Object { $_ -lt 3 -or $_ -gt 128 }).Count -gt 0) { @@ -84,7 +84,7 @@ function Assert-Checkout([string] $Directory, [string] $Repository) { } function Copy-Worker([string] $Checkout) { - $destination = Join-Path $Checkout 'test\Dissemination.PerformanceHarness' + $destination = Join-Path $Checkout 'test' 'Dissemination.PerformanceHarness' if (Test-Path -LiteralPath $destination) { throw "Refusing to overwrite an existing runtime harness: $destination" } @@ -98,7 +98,7 @@ function Copy-Worker([string] $Checkout) { } function Publish-Silo([string] $Checkout, [string] $Runtime, [string] $Revision) { - $project = Join-Path $Checkout 'test\Dissemination.PerformanceHarness\Silo\Dissemination.Silo.csproj' + $project = Join-Path $Checkout 'test' 'Dissemination.PerformanceHarness' 'Silo' 'Dissemination.Silo.csproj' $output = Join-Path $binaries $Runtime Push-Location $Checkout try { @@ -133,7 +133,8 @@ function Publish-Silo([string] $Checkout, [string] $Runtime, [string] $Revision) } | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath (Join-Path $output 'manifest.json') } -$artifacts = [System.IO.Path]::GetFullPath((Join-Path $root $ArtifactsDirectory)) +$artifactName = ($ArtifactsDirectory -split '[\\/]')[1] +$artifacts = [System.IO.Path]::GetFullPath((Join-Path $root 'Artifacts' $artifactName)) Assert-NoLinks $artifacts if (Test-Path -LiteralPath $artifacts) { foreach ($item in Get-ChildItem -LiteralPath $artifacts -Recurse -Force) { @@ -161,10 +162,10 @@ if (!$SkipBuild) { if ($CandidateRef -match '^[0-9a-fA-F]{40}$' -and $candidateSha -ne $CandidateRef) { throw "Candidate checkout does not match requested commit $CandidateRef." } - if (!(Test-Path -LiteralPath (Join-Path $candidate 'src\Orleans.Runtime\Dissemination'))) { + if (!(Test-Path -LiteralPath (Join-Path $candidate 'src' 'Orleans.Runtime' 'Dissemination'))) { throw 'Select a candidate revision containing the dissemination runtime APIs.' } - if (Test-Path -LiteralPath (Join-Path $baseline 'src\Orleans.Runtime\Dissemination')) { + if (Test-Path -LiteralPath (Join-Path $baseline 'src' 'Orleans.Runtime' 'Dissemination')) { throw 'Original runtime unexpectedly contains dissemination.' } New-Item -ItemType Directory -Force $binaries | Out-Null @@ -173,7 +174,7 @@ if (!$SkipBuild) { Publish-Silo $baseline 'Old' $baselineSha Publish-Silo $candidate 'New' $candidateSha Invoke-DotNet @( - 'publish', (Join-Path $harnessSource 'Tests\Dissemination.PerformanceHarness.Tests.csproj'), + 'publish', (Join-Path $harnessSource 'Tests' 'Dissemination.PerformanceHarness.Tests.csproj'), '--configuration', 'Release', '--framework', 'net10.0', '--output', (Join-Path $binaries 'Runner'), '--verbosity', 'minimal', "-bl:$(Join-Path $artifacts 'build-runner.binlog')" @@ -195,18 +196,18 @@ if ($selection.CandidateRepository -ne $CandidateRepository -or $selection.Candi throw 'Requested revisions must match the downloaded binary selection.json.' } foreach ($runtime in @('Old', 'New')) { - $manifest = Get-Content -LiteralPath (Join-Path $binaries "$runtime\manifest.json") -Raw | ConvertFrom-Json + $manifest = Get-Content -LiteralPath (Join-Path $binaries $runtime 'manifest.json') -Raw | ConvertFrom-Json $expected = if ($runtime -eq 'Old') { $baselineSha } else { $selection.CandidateCommit } if ($manifest.SourceRevision -ne $expected -or $manifest.Runtime -ne $runtime) { throw "Binary manifest does not match selected $runtime revision." } } -$runner = Join-Path $binaries 'Runner\Dissemination.PerformanceHarness.Tests.dll' +$runner = Join-Path $binaries 'Runner' 'Dissemination.PerformanceHarness.Tests.dll' if (!(Test-Path -LiteralPath $runner)) { throw "Published runner not found: $runner" } -$results = Join-Path $artifacts "results\$([DateTime]::UtcNow.ToString('yyyyMMddTHHmmss'))-$([Guid]::NewGuid().ToString('N'))" +$results = Join-Path $artifacts 'results' "$([DateTime]::UtcNow.ToString('yyyyMMddTHHmmss'))-$([Guid]::NewGuid().ToString('N'))" New-Item -ItemType Directory -Force $results | Out-Null Copy-Item -LiteralPath (Join-Path $harnessSource 'methodology.json') -Destination $results Copy-Item -LiteralPath (Join-Path $binaries 'selection.json') -Destination $results diff --git a/.github/scripts/test-dissemination-performance.ps1 b/.github/scripts/test-dissemination-performance.ps1 index 059b8891352..1feb11e02c4 100644 --- a/.github/scripts/test-dissemination-performance.ps1 +++ b/.github/scripts/test-dissemination-performance.ps1 @@ -48,3 +48,82 @@ foreach ($case in $cases) { } } Write-Output "Passed $($cases.Count) script-boundary checks." + +$root = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..' '..')) +$artifactName = "DisseminationPathCheck-$([Guid]::NewGuid().ToString('N'))" +$testArtifacts = Join-Path $root 'Artifacts' $artifactName +$binaries = Join-Path $testArtifacts 'bin' +$runner = Join-Path $binaries 'Runner' 'Dissemination.PerformanceHarness.Tests.dll' +$calls = [System.Collections.Generic.List[object]]::new() +$environment = @{} +foreach ($entry in Get-ChildItem Env:ORLEANS_DISSEMINATION_*) { + $environment[$entry.Name] = $entry.Value +} + +function dotnet { + param([Parameter(ValueFromRemainingArguments)][string[]] $Arguments) + if ($Arguments[0] -ne $runner) { + throw "Unexpected runner path: $($Arguments[0])" + } + $calls.Add($Arguments) + $global:LASTEXITCODE = 0 +} + +try { + $selection = @{ + CandidateRepository = 'ReubenBond/orleans' + CandidateRef = '3fc0a4610cd2eb5feef27e958fcca59658aaae47' + CandidateCommit = '3fc0a4610cd2eb5feef27e958fcca59658aaae47' + BaselineCommit = '6739589254b746a8790cf53524e6abe372bb53d4' + } + foreach ($runtime in @('Old', 'New')) { + $directory = Join-Path $binaries $runtime + New-Item -ItemType Directory -Path $directory -Force | Out-Null + @{ + Runtime = $runtime + SourceRevision = if ($runtime -eq 'Old') { $selection.BaselineCommit } else { $selection.CandidateCommit } + } | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $directory 'manifest.json') + } + New-Item -ItemType Directory -Path (Split-Path -Parent $runner) | Out-Null + New-Item -ItemType File -Path $runner | Out-Null + $selection | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $binaries 'selection.json') + + foreach ($separator in @('/', '\')) { + & $script -CandidateRepository $selection.CandidateRepository -CandidateRef $selection.CandidateRef ` + -ArtifactsDirectory "Artifacts$separator$artifactName" -Sizes '3' -Iterations 3 -Repetitions 1 -SkipBuild + } + if ($calls.Count -ne 4) { + throw "Expected measurement and scaling invocations for both path spellings; got $($calls.Count)." + } + for ($index = 0; $index -lt $calls.Count; $index++) { + $expectedClass = if ($index % 2 -eq 0) { 'MeasurementTests' } else { 'ScalingTests' } + if ($calls[$index][2] -ne "Orleans.Dissemination.PerformanceHarness.$expectedClass") { + throw "Unexpected test selection in invocation $index." + } + } + $runs = @(Get-ChildItem -LiteralPath (Join-Path $testArtifacts 'results') -Directory) + if ($runs.Count -ne 2) { + throw "Expected two results directories under the same artifact root; got $($runs.Count)." + } + foreach ($run in $runs) { + foreach ($filename in @('invocation.json', 'selection.json', 'methodology.json', 'host-resources.json')) { + $null = Get-Content -LiteralPath (Join-Path $run.FullName $filename) -Raw | ConvertFrom-Json + } + $invocation = Get-Content -LiteralPath (Join-Path $run.FullName 'invocation.json') -Raw | ConvertFrom-Json + if ($invocation.ResultsDirectory -ne $run.FullName) { + throw 'The recorded results path differs from the created directory.' + } + } + Write-Output 'Passed 2 SkipBuild path checks with worker execution stubbed.' +} +finally { + foreach ($entry in Get-ChildItem Env:ORLEANS_DISSEMINATION_*) { + [Environment]::SetEnvironmentVariable($entry.Name, $null) + } + foreach ($entry in $environment.GetEnumerator()) { + [Environment]::SetEnvironmentVariable($entry.Key, $entry.Value) + } + if (Test-Path -LiteralPath $testArtifacts) { + Remove-Item -LiteralPath $testArtifacts -Recurse -Force + } +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20a90248ade..f5054c5ff38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,8 +70,13 @@ jobs: if: matrix.os == 'ubuntu-latest' shell: pwsh run: ./.github/scripts/test-summarize-coverage.ps1 + - name: Test performance harness script paths + shell: pwsh + run: ./.github/scripts/test-dissemination-performance.ps1 - name: Build run: dotnet build -bl + - name: Test performance harness membership provider + run: dotnet test --project test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj --framework net10.0 --no-build --filter-class "Orleans.Dissemination.PerformanceHarness.FileMembershipTableTests" --minimum-expected-tests 10 - name: Pack and validate package compatibility if: matrix.os == 'ubuntu-latest' run: dotnet pack Orleans.slnx --configuration Debug --no-build --no-restore -bl:package_validation.binlog diff --git a/test/Dissemination.PerformanceHarness/ProviderTests/FileMembershipTableTests.cs b/test/Dissemination.PerformanceHarness/ProviderTests/FileMembershipTableTests.cs new file mode 100644 index 00000000000..4bc06adb19f --- /dev/null +++ b/test/Dissemination.PerformanceHarness/ProviderTests/FileMembershipTableTests.cs @@ -0,0 +1,116 @@ +using System.Net; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Dissemination.PerformanceHarness; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Runtime")] +[TestCategory("BVT")] +public sealed class FileMembershipTableTests : IDisposable +{ + private static readonly DateTime Cutoff = new(2026, 1, 15, 12, 0, 0, DateTimeKind.Utc); + private readonly string _directory = Path.Combine(Path.GetTempPath(), $"orleans-dissemination-cleanup-{Guid.NewGuid():N}"); + + [Fact] + public async Task CleanupRemovesEveryExpiredNonActiveStatus() + { + var table = CreateTable(); + await table.InitializeMembershipTableAsync(true, TestContext.Current.CancellationToken); + foreach (var status in Enum.GetValues()) + { + var entry = CreateEntry(status, (int)status + 1); + var data = await table.ReadAllAsync(TestContext.Current.CancellationToken); + Assert.True(await table.InsertRowAsync(entry, data.Version.Next(), TestContext.Current.CancellationToken)); + } + + var before = await table.ReadAllAsync(TestContext.Current.CancellationToken); + var expected = Assert.Single(before.Members, row => row.Item1.Status == SiloStatus.Active); + var reopened = CreateTable(); + await reopened.CleanupDefunctSiloEntriesAsync(Cutoff, TestContext.Current.CancellationToken); + var after = await reopened.ReadAllAsync(TestContext.Current.CancellationToken); + + var remaining = Assert.Single(after.Members); + Assert.Equal(expected.Item1.SiloAddress, remaining.Item1.SiloAddress); + Assert.Equal(SiloStatus.Active, remaining.Item1.Status); + Assert.Equal(expected.Item2, remaining.Item2); + Assert.Equal(before.Version.Version, after.Version.Version); + Assert.Equal(before.Version.VersionEtag, after.Version.VersionEtag); + } + + [Theory] + [InlineData("start", -1)] + [InlineData("start", 0)] + [InlineData("start", 1)] + [InlineData("heartbeat", -1)] + [InlineData("heartbeat", 0)] + [InlineData("heartbeat", 1)] + [InlineData("suspect", -1)] + [InlineData("suspect", 0)] + [InlineData("suspect", 1)] + public async Task CleanupUsesEffectiveUpdateTimeAndExclusiveCutoff(string timestamp, int offset) + { + var table = CreateTable(); + await table.InitializeMembershipTableAsync(true, TestContext.Current.CancellationToken); + var entry = CreateEntry(SiloStatus.Dead, 1); + var latest = Cutoff.AddTicks(offset); + switch (timestamp) + { + case "start": + entry.StartTime = latest; + break; + case "heartbeat": + entry.IAmAliveTime = latest; + break; + case "suspect": + entry.SuspectTimes = + [ + Tuple.Create(CreateEntry(SiloStatus.Active, 2).SiloAddress, latest), + Tuple.Create(CreateEntry(SiloStatus.Active, 3).SiloAddress, Cutoff.AddDays(-2)), + ]; + break; + } + + var data = await table.ReadAllAsync(TestContext.Current.CancellationToken); + Assert.True(await table.InsertRowAsync(entry, data.Version.Next(), TestContext.Current.CancellationToken)); + var reopened = CreateTable(); + var persisted = Assert.Single((await reopened.ReadAllAsync(TestContext.Current.CancellationToken)).Members); + Assert.Equal(latest, persisted.Item1.EffectiveUpdateTime); + + await reopened.CleanupDefunctSiloEntriesAsync(Cutoff, TestContext.Current.CancellationToken); + var after = await reopened.ReadAllAsync(TestContext.Current.CancellationToken); + if (offset < 0) + { + Assert.Empty(after.Members); + } + else + { + var remaining = Assert.Single(after.Members); + Assert.Equal(entry.SiloAddress, remaining.Item1.SiloAddress); + Assert.Equal(latest, remaining.Item1.EffectiveUpdateTime); + Assert.Equal(persisted.Item2, remaining.Item2); + } + } + + private FileMembershipTable CreateTable() => + new(new("cleanup", "cleanup", _directory, 0, Environment.ProcessId, Enabled: false)); + + private static MembershipEntry CreateEntry(SiloStatus status, int port) => new() + { + SiloAddress = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 10000 + port), 1), + HostName = "localhost", + SiloName = $"cleanup-{port}", + Status = status, + StartTime = Cutoff.AddDays(-1), + IAmAliveTime = Cutoff.AddDays(-1), + }; + + public void Dispose() + { + if (Directory.Exists(_directory)) + { + Directory.Delete(_directory, recursive: true); + } + } +} diff --git a/test/Dissemination.PerformanceHarness/README.md b/test/Dissemination.PerformanceHarness/README.md index 5d97b2032af..c193f9dede2 100644 --- a/test/Dissemination.PerformanceHarness/README.md +++ b/test/Dissemination.PerformanceHarness/README.md @@ -106,8 +106,9 @@ the exact repository/ref recorded in `bin\selection.json`: ``` `-ArtifactsDirectory` accepts a single named directory under `Artifacts`; paths -containing existing links/reparse points are rejected. Builds require fresh binary -output; each replay creates its own results directory. +accept either slash spelling and are assembled from platform-native components. +Existing links/reparse points are rejected. Builds require fresh binary output; +each replay creates its own results directory. Run only the lightweight controller measurement cases locally: @@ -117,6 +118,20 @@ dotnet test --project test\Dissemination.PerformanceHarness\Tests\Dissemination. --framework net10.0 --filter-class '*MeasurementTests' --minimum-expected-tests 15 ``` -The harness projects are selected explicitly by the workflow and remain outside -the solution's ordinary test runs. The scaling test requires the script's explicit -manual-run marker and published binary selection. +Ordinary Linux/Windows CI runs the script checks with worker execution stubbed. +`ProviderTests\FileMembershipTableTests.cs` is linked into `Orleans.Runtime.Tests` +alongside the file-backed provider and its protocol. These BVT cases exercise +persisted membership cleanup: all expired non-active statuses are removed, while +active rows and rows updated at or after the cutoff retain their ETags. Cleanup uses +the selected runtime's `EffectiveUpdateTime`, including startup, heartbeat and +suspect-vote timestamps. + +```powershell +dotnet test --project test\Orleans.Runtime.Tests\Orleans.Runtime.Tests.csproj ` + --framework net10.0 --filter-class 'Orleans.Dissemination.PerformanceHarness.FileMembershipTableTests' ` + --minimum-expected-tests 10 +``` + +The scaling test runs through the script's explicit manual-run marker and published +binary selection. The measurement controller retains its independent build using +shared protocol and instrumentation sources. diff --git a/test/Dissemination.PerformanceHarness/Silo/FileMembershipTable.cs b/test/Dissemination.PerformanceHarness/Silo/FileMembershipTable.cs index 0e6e88c20b4..5dcd6dc425b 100644 --- a/test/Dissemination.PerformanceHarness/Silo/FileMembershipTable.cs +++ b/test/Dissemination.PerformanceHarness/Silo/FileMembershipTable.cs @@ -75,8 +75,8 @@ public Task DeleteMembershipTableEntriesAsync(string clusterId, CancellationToke public Task CleanupDefunctSiloEntriesAsync(DateTimeOffset beforeDate, CancellationToken cancellationToken) => Access(state => { foreach (var key in state.Rows.Where(pair => - pair.Value.Entry.Status == (int)SiloStatus.Dead - && pair.Value.Entry.IAmAliveTime < beforeDate.UtcDateTime).Select(pair => pair.Key).ToArray()) + pair.Value.Entry.Status != (int)SiloStatus.Active + && pair.Value.Entry.ToEntry().EffectiveUpdateTime < beforeDate.UtcDateTime).Select(pair => pair.Key).ToArray()) { state.Rows.Remove(key); } diff --git a/test/Dissemination.PerformanceHarness/methodology.json b/test/Dissemination.PerformanceHarness/methodology.json index 73971d5035a..5b04eccdb2d 100644 --- a/test/Dissemination.PerformanceHarness/methodology.json +++ b/test/Dissemination.PerformanceHarness/methodology.json @@ -36,7 +36,7 @@ "churn": "Gracefully stop and restart one process at its original endpoint every other round. Confirm membership and supported enabled peers before the next offered round. Replacement startup, shutdown, discovery and fallback costs remain in the measured window.", "partition": "Abort real inbound/outbound silo connections and reject new admissions at one endpoint. Publish once on each remaining node. Recovery drains middleware and close tasks at both endpoints, including unidentified handshakes, then confirms bidirectional acknowledged control RPCs before a single publication per active node. Teardown/readiness/retries remain inside recovery time and total cost, preserving the #11243 barrier.", "window": "Identical two-second socket-sampling margins; costs include background runtime, control RPCs and instrumentation. The post-window full-GC retained-memory probe is separate from CPU/allocation costs.", - "membership": "Exclusive-file-lease JSON provider with atomic replacement, ETags and incarnation-aware rows. Liveness detection is disabled so the transport partition retains stable membership.", + "membership": "Exclusive-file-lease JSON provider with atomic replacement, ETags and incarnation-aware rows. Cleanup removes non-active rows whose runtime EffectiveUpdateTime precedes the cutoff, accounting for startup, heartbeat and suspect votes. Liveness detection is disabled so the transport partition retains stable membership.", "metrics": [ "Request plus one-way Orleans RPC counts; all messages including responses reported separately", "Load broadcast requests and logical value transmissions reported separately, with requests per round and per second", diff --git a/test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj b/test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj index 7c50a8e91a8..2a8e7d91efc 100644 --- a/test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj +++ b/test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj @@ -9,6 +9,12 @@ true + + + + + + From 0340d0021bed27af1a15036e7ab08ee28c42bc30 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 16 Sep 2026 10:12:09 -0700 Subject: [PATCH 05/12] ci: keep provider regressions in canonical BVT discovery --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5054c5ff38..933b0111aff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,8 +75,6 @@ jobs: run: ./.github/scripts/test-dissemination-performance.ps1 - name: Build run: dotnet build -bl - - name: Test performance harness membership provider - run: dotnet test --project test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj --framework net10.0 --no-build --filter-class "Orleans.Dissemination.PerformanceHarness.FileMembershipTableTests" --minimum-expected-tests 10 - name: Pack and validate package compatibility if: matrix.os == 'ubuntu-latest' run: dotnet pack Orleans.slnx --configuration Debug --no-build --no-restore -bl:package_validation.binlog From fcd7ba2654e0ad104065260bba596037e72db2c5 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 16 Sep 2026 10:17:16 -0700 Subject: [PATCH 06/12] fix: invalidate stale membership writes after clearing rows --- .../ProviderTests/FileMembershipTableTests.cs | 57 +++++++++++++++++++ .../README.md | 5 +- .../Silo/FileMembershipTable.cs | 1 + 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/test/Dissemination.PerformanceHarness/ProviderTests/FileMembershipTableTests.cs b/test/Dissemination.PerformanceHarness/ProviderTests/FileMembershipTableTests.cs index 4bc06adb19f..4365ce7ab21 100644 --- a/test/Dissemination.PerformanceHarness/ProviderTests/FileMembershipTableTests.cs +++ b/test/Dissemination.PerformanceHarness/ProviderTests/FileMembershipTableTests.cs @@ -13,6 +13,63 @@ public sealed class FileMembershipTableTests : IDisposable private static readonly DateTime Cutoff = new(2026, 1, 15, 12, 0, 0, DateTimeKind.Utc); private readonly string _directory = Path.Combine(Path.GetTempPath(), $"orleans-dissemination-cleanup-{Guid.NewGuid():N}"); + [Fact] + public async Task DeleteInvalidatesThePreviousTableEtag() + { + var table = CreateTable(); + var cancellationToken = TestContext.Current.CancellationToken; + await table.InitializeMembershipTableAsync(true, cancellationToken); + var entry = CreateEntry(SiloStatus.Active, 1); + var initial = await table.ReadAllAsync(cancellationToken); + Assert.True(await table.InsertRowAsync(entry, initial.Version.Next(), cancellationToken)); + var before = await table.ReadAllAsync(cancellationToken); + var previousRow = Assert.Single(before.Members); + + await table.DeleteMembershipTableEntriesAsync("cleanup", cancellationToken); + var reopened = CreateTable(); + var deleted = await reopened.ReadAllAsync(cancellationToken); + Assert.Empty(deleted.Members); + Assert.Equal(before.Version.Version + 1, deleted.Version.Version); + Assert.NotEqual(before.Version.VersionEtag, deleted.Version.VersionEtag); + Assert.False(await reopened.InsertRowAsync(entry, before.Version.Next(), cancellationToken)); + Assert.False(await reopened.UpdateRowAsync(entry, previousRow.Item2, before.Version.Next(), cancellationToken)); + var afterRejectedWrites = await reopened.ReadAllAsync(cancellationToken); + Assert.Empty(afterRejectedWrites.Members); + Assert.Equal(deleted.Version, afterRejectedWrites.Version); + + Assert.True(await reopened.InsertRowAsync(entry, deleted.Version.Next(), cancellationToken)); + var inserted = await reopened.ReadAllAsync(cancellationToken); + var currentRow = Assert.Single(inserted.Members); + Assert.Equal(entry.SiloAddress, currentRow.Item1.SiloAddress); + Assert.False(await reopened.UpdateRowAsync(entry, currentRow.Item2, before.Version.Next(), cancellationToken)); + Assert.Equal(inserted.Version, (await reopened.ReadAllAsync(cancellationToken)).Version); + } + + [Fact] + public async Task HeartbeatPreservesTableVersionAndOtherColumns() + { + var table = CreateTable(); + var cancellationToken = TestContext.Current.CancellationToken; + await table.InitializeMembershipTableAsync(true, cancellationToken); + var entry = CreateEntry(SiloStatus.Active, 1); + var initial = await table.ReadAllAsync(cancellationToken); + Assert.True(await table.InsertRowAsync(entry, initial.Version.Next(), cancellationToken)); + var before = await table.ReadAllAsync(cancellationToken); + var previousRow = Assert.Single(before.Members); + + var update = CreateEntry(SiloStatus.Dead, 1); + update.HostName = "ignored"; + update.IAmAliveTime = Cutoff; + await table.UpdateIAmAliveAsync(update, cancellationToken); + var reopened = CreateTable(); + var after = await reopened.ReadAllAsync(cancellationToken); + var currentRow = Assert.Single(after.Members); + Assert.Equal(before.Version, after.Version); + Assert.Equal(previousRow.Item2, currentRow.Item2); + var expected = FileMembershipTable.EntryData.From(previousRow.Item1) with { IAmAliveTime = Cutoff }; + Assert.Equal(StateComparison.Serialize(expected), StateComparison.Serialize(FileMembershipTable.EntryData.From(currentRow.Item1))); + } + [Fact] public async Task CleanupRemovesEveryExpiredNonActiveStatus() { diff --git a/test/Dissemination.PerformanceHarness/README.md b/test/Dissemination.PerformanceHarness/README.md index c193f9dede2..5d4e4be20d2 100644 --- a/test/Dissemination.PerformanceHarness/README.md +++ b/test/Dissemination.PerformanceHarness/README.md @@ -125,11 +125,14 @@ persisted membership cleanup: all expired non-active statuses are removed, while active rows and rows updated at or after the cutoff retain their ETags. Cleanup uses the selected runtime's `EffectiveUpdateTime`, including startup, heartbeat and suspect-vote timestamps. +Clearing membership advances the table ETag so writes from an earlier snapshot +are rejected. Heartbeat dirty writes preserve the table version and update only +the liveness timestamp. ```powershell dotnet test --project test\Orleans.Runtime.Tests\Orleans.Runtime.Tests.csproj ` --framework net10.0 --filter-class 'Orleans.Dissemination.PerformanceHarness.FileMembershipTableTests' ` - --minimum-expected-tests 10 + --minimum-expected-tests 12 ``` The scaling test runs through the script's explicit manual-run marker and published diff --git a/test/Dissemination.PerformanceHarness/Silo/FileMembershipTable.cs b/test/Dissemination.PerformanceHarness/Silo/FileMembershipTable.cs index 5dcd6dc425b..cfa1b5dba5f 100644 --- a/test/Dissemination.PerformanceHarness/Silo/FileMembershipTable.cs +++ b/test/Dissemination.PerformanceHarness/Silo/FileMembershipTable.cs @@ -69,6 +69,7 @@ public Task DeleteMembershipTableEntriesAsync(string clusterId, CancellationToke state.Rows.Clear(); state.Version++; + state.Etag = Guid.NewGuid().ToString("N"); return true; }, write: true, cancellationToken); From a2f057e78b87f16f406b796eef7344458805e71f Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 16 Sep 2026 10:35:40 -0700 Subject: [PATCH 07/12] docs: clarify dissemination performance opt-in labels --- test/Dissemination.PerformanceHarness/README.md | 6 ++++-- .../Tests/MeasurementTests.cs | 2 ++ .../Tests/ProcessCluster.cs | 2 +- .../Tests/ScalingComparison.cs | 1 + test/Dissemination.PerformanceHarness/Tests/ScalingTests.cs | 2 +- test/Dissemination.PerformanceHarness/methodology.json | 2 +- 6 files changed, 10 insertions(+), 5 deletions(-) diff --git a/test/Dissemination.PerformanceHarness/README.md b/test/Dissemination.PerformanceHarness/README.md index 5d4e4be20d2..5d1df314c68 100644 --- a/test/Dissemination.PerformanceHarness/README.md +++ b/test/Dissemination.PerformanceHarness/README.md @@ -57,8 +57,10 @@ Readiness time and RPC costs stay inside recovery and total cost windows. Automatic load publication is paused by disposing its timer on the publisher scheduler. The timer field retains its disposed reference for original-runtime -shutdown. Enabled runs use the candidate's production cadence after peer support -confirmation; replacement startup and support confirmation remain inside churn costs. +shutdown. Enabled runs explicitly opt into the subsystem and both namespaces while +retaining the candidate's production dissemination tuning defaults. Offered rounds +begin after peer support confirmation; replacement startup and support confirmation +remain inside churn costs. The Linux/Windows memory guard plans 128 MiB per silo plus 1 GiB controller headroom. Commands, startup, shutdown and processes have deadlines; the workflow has a diff --git a/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.cs b/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.cs index b0e1aa0e532..23e12171c1d 100644 --- a/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.cs +++ b/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.cs @@ -61,6 +61,8 @@ public void ScalingComparisonNormalizesEqualOfferedWorkAndReportsLatencyDistribu Assert.Equal(40, pair.Disabled.P95ConvergenceMilliseconds); Assert.Equal(30, pair.Enabled.P95ConvergenceMilliseconds); Assert.Contains("99.00 -> 10.00", result.ToMarkdown(), StringComparison.Ordinal); + Assert.Contains("Enabled mode explicitly opts into the subsystem and both namespaces", result.ToMarkdown(), StringComparison.Ordinal); + Assert.Contains("production dissemination tuning defaults", result.ToMarkdown(), StringComparison.Ordinal); } [Theory] diff --git a/test/Dissemination.PerformanceHarness/Tests/ProcessCluster.cs b/test/Dissemination.PerformanceHarness/Tests/ProcessCluster.cs index 0ac07f17c4d..07cac076baa 100644 --- a/test/Dissemination.PerformanceHarness/Tests/ProcessCluster.cs +++ b/test/Dissemination.PerformanceHarness/Tests/ProcessCluster.cs @@ -61,7 +61,7 @@ public async Task Start(string runtime, bool enabled, int? port = n Assert.NotEqual(Baseline, manifest.SourceRevision); } - var name = $"{runtime}-{_sequence++}-{(enabled ? "enabled" : "default")}"; + var name = $"{runtime}-{_sequence++}-{(enabled ? "enabled" : "disabled")}"; var node = new SiloProcess( binaryDirectory, new(name, Path.GetFileName(Directory), _membershipDirectory, port ?? AvailablePort(), Environment.ProcessId, diff --git a/test/Dissemination.PerformanceHarness/Tests/ScalingComparison.cs b/test/Dissemination.PerformanceHarness/Tests/ScalingComparison.cs index 2482816057c..3be17bf690c 100644 --- a/test/Dissemination.PerformanceHarness/Tests/ScalingComparison.cs +++ b/test/Dissemination.PerformanceHarness/Tests/ScalingComparison.cs @@ -61,6 +61,7 @@ public string ToMarkdown() { var result = new StringBuilder(); result.AppendLine("### Dissemination disabled vs enabled"); + result.AppendLine("Enabled mode explicitly opts into the subsystem and both namespaces, retaining the candidate's production dissemination tuning defaults."); result.AppendLine("Each cell is disabled -> enabled. Costs per offered publication include background and control traffic. Single-host, resource-shared measurements; raw totals, environment and per-node data are in the artifacts."); result.AppendLine(); result.AppendLine("| Silos / scenario / repetition | RPCs / publication | KiB / publication | Allocated KiB / publication | CPU ms / publication | Median ms | P95 ms |"); diff --git a/test/Dissemination.PerformanceHarness/Tests/ScalingTests.cs b/test/Dissemination.PerformanceHarness/Tests/ScalingTests.cs index 66a9ec25d75..d8ddf98d171 100644 --- a/test/Dissemination.PerformanceHarness/Tests/ScalingTests.cs +++ b/test/Dissemination.PerformanceHarness/Tests/ScalingTests.cs @@ -301,7 +301,7 @@ async Task PublishRound() Excludes = "controller process CPU/allocation, storage disk byte accounting, cross-machine latency, NIC/TCP wire overhead, production storage/placement/grain work", PartitionComparison = "retire partitioned connections at both endpoints, then confirm healed connections with bidirectional acknowledged control RPCs before the same single publication round on all paths; teardown and readiness remain inside partition recovery time and total cost", OriginalBaselineAttribution = "original-to-current disabled deltas include all intervening source changes, including lifecycle prerequisites and load notification/tombstone/dedup changes; only current off/on isolates enabling the subsystem on the same binary", - Interpretation = "separate original-legacy, current-explicitly-disabled and current-explicitly-enabled-supported measurements", + Interpretation = "separate original-legacy, current-explicitly-disabled and current-explicitly-enabled-supported measurements; enabled mode explicitly opts into the subsystem and both namespaces, retaining candidate production dissemination tuning defaults", }, }; await cluster.Save("window-start.json", start); diff --git a/test/Dissemination.PerformanceHarness/methodology.json b/test/Dissemination.PerformanceHarness/methodology.json index 5b04eccdb2d..235e0739d05 100644 --- a/test/Dissemination.PerformanceHarness/methodology.json +++ b/test/Dissemination.PerformanceHarness/methodology.json @@ -29,7 +29,7 @@ "measurement": { "test": "Scaling_SelectedRuntimePathsConverge", "paths": ["OriginalLegacy", "CurrentDisabled", "CurrentEnabledSupported"], - "enablement": "Candidate off/on explicitly sets the subsystem and both namespace flags. Overlay, repair, coalescing and namespace cadence use the selected runtime's settings.", + "enablement": "Enabled mode explicitly opts into the subsystem and both namespaces; disabled mode explicitly opts out. Both retain the candidate's production dissemination tuning defaults for overlay, repair, coalescing and namespace cadence.", "warmup": "Two production publication rounds and complete enabled-peer support confirmation precede the cost window. Preparation time, CPU and allocation totals are reported separately.", "publication": "Dispose the automatic DeploymentLoadPublisher timer on its scheduler while retaining the field reference for original-runtime lifecycle shutdown. Invoke the production PublishStatistics method once per active node, then require exact load dictionary inventory and values using IncludeFields=true.", "stable": "Repeat equal closed-loop publication/convergence rounds.", From e265fadd452756281a00f93e9b16559f66a66985 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 16 Sep 2026 13:41:21 -0700 Subject: [PATCH 08/12] fix: diagnose performance harness runtime boundaries --- .github/scripts/dissemination-performance.ps1 | 18 +++- .../test-dissemination-performance.ps1 | 88 ++++++++++++++++++- .../README.md | 14 ++- .../Shared/PublisherApi.cs | 33 +++++++ .../Silo/Dissemination.Silo.csproj | 1 + .../Silo/Program.cs | 7 +- ...semination.PerformanceHarness.Tests.csproj | 3 +- .../Tests/MeasurementTests.PublisherApi.cs | 81 +++++++++++++++++ .../Tests/MeasurementTests.cs | 2 +- 9 files changed, 233 insertions(+), 14 deletions(-) create mode 100644 test/Dissemination.PerformanceHarness/Shared/PublisherApi.cs create mode 100644 test/Dissemination.PerformanceHarness/Tests/MeasurementTests.PublisherApi.cs diff --git a/.github/scripts/dissemination-performance.ps1 b/.github/scripts/dissemination-performance.ps1 index 6c85dbf2113..3beac0351b9 100644 --- a/.github/scripts/dissemination-performance.ps1 +++ b/.github/scripts/dissemination-performance.ps1 @@ -40,15 +40,25 @@ if (@($Scenarios.Split(',') | Select-Object -Unique).Count -ne $Scenarios.Split( } function Assert-NoLinks([string] $Path) { - $current = $Path - while ($current -and $current -ne $root) { + $current = [System.IO.Path]::GetFullPath($Path) + $relative = [System.IO.Path]::GetRelativePath($root, $current) + if ([System.IO.Path]::IsPathRooted($relative) -or $relative -eq '..' -or + $relative.StartsWith("..$([System.IO.Path]::DirectorySeparatorChar)", [StringComparison]::Ordinal)) { + throw "Path is outside the repository boundary: $Path" + } + $comparison = if ($IsWindows) { [StringComparison]::OrdinalIgnoreCase } else { [StringComparison]::Ordinal } + while (![string]::Equals($current, $root, $comparison)) { if (Test-Path -LiteralPath $current) { $item = Get-Item -LiteralPath $current -Force if ($item.Attributes -band [System.IO.FileAttributes]::ReparsePoint) { throw "Artifact and runtime paths must use ordinary directories: $current" } } - $current = Split-Path -Parent $current + $parent = Split-Path -Parent $current + if (!$parent -or [string]::Equals($parent, $current, $comparison)) { + throw "Path is outside the repository boundary: $Path" + } + $current = $parent } } @@ -271,7 +281,7 @@ Push-Location $root try { Invoke-DotNet @( $runner, '--filter-class', 'Orleans.Dissemination.PerformanceHarness.MeasurementTests', - '--minimum-expected-tests', '15', '--report-trx', + '--minimum-expected-tests', '21', '--report-trx', '--results-directory', (Join-Path $results 'instrument-checks') ) Invoke-DotNet @( diff --git a/.github/scripts/test-dissemination-performance.ps1 b/.github/scripts/test-dissemination-performance.ps1 index 1feb11e02c4..82a136a6cba 100644 --- a/.github/scripts/test-dissemination-performance.ps1 +++ b/.github/scripts/test-dissemination-performance.ps1 @@ -2,7 +2,7 @@ $ErrorActionPreference = 'Stop' Set-StrictMode -Version Latest $script = Join-Path $PSScriptRoot 'dissemination-performance.ps1' $errors = $null -$null = [System.Management.Automation.Language.Parser]::ParseFile($script, [ref]$null, [ref]$errors) +$scriptAst = [System.Management.Automation.Language.Parser]::ParseFile($script, [ref]$null, [ref]$errors) if ($errors) { throw ($errors | Out-String) } @@ -70,6 +70,92 @@ function dotnet { } try { + & { + foreach ($name in @('Assert-NoLinks', 'Copy-Worker')) { + $definition = $scriptAst.Find({ + param($node) + $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq $name + }, $false) + . ([scriptblock]::Create($definition.Extent.Text)) + } + + Assert-NoLinks $root + Assert-NoLinks (Join-Path $testArtifacts 'not-created') + foreach ($outside in @((Split-Path -Parent $root), [System.IO.Path]::GetPathRoot($root), "$root-sibling")) { + $rejected = $false + try { + Assert-NoLinks $outside + } + catch { + if ($_.Exception.Message -notlike 'Path is outside the repository boundary:*') { + throw + } + $rejected = $true + } + if (!$rejected) { + throw "Expected repository-boundary rejection for $outside." + } + } + Write-Output 'Passed 5 repository-boundary checks.' + + $harnessSource = Join-Path $root 'test' 'Dissemination.PerformanceHarness' + $checkout = Join-Path $testArtifacts 'runtime-checkout' + $existingHarness = Join-Path $checkout 'test' 'Dissemination.IntegrationHarness' + New-Item -ItemType Directory -Path $existingHarness -Force | Out-Null + $sentinel = Join-Path $existingHarness 'preserved.txt' + Set-Content -LiteralPath $sentinel -Value 'Existing candidate harness remains intact.' + Copy-Worker $checkout + $copied = Join-Path $checkout 'test' 'Dissemination.PerformanceHarness' + if (@(Get-ChildItem -LiteralPath $copied -Directory).Count -ne 2) { + throw 'Copy-Worker must copy exactly the Silo and Shared directories.' + } + foreach ($part in @('Silo', 'Shared')) { + $expected = @(Get-ChildItem -LiteralPath (Join-Path $harnessSource $part) -File | + Where-Object { $_.Extension -in '.cs', '.csproj' }) + $actual = @(Get-ChildItem -LiteralPath (Join-Path $copied $part) -File) + if ($expected.Count -ne $actual.Count) { + throw "Copied $part inventory differs from the tooling source." + } + foreach ($file in $expected) { + $copy = Join-Path $copied $part $file.Name + if ((Get-FileHash -LiteralPath $file.FullName).Hash -ne (Get-FileHash -LiteralPath $copy).Hash) { + throw "Copied file differs: $copy" + } + } + } + $rejected = $false + try { + Copy-Worker $checkout + } + catch { + if ($_.Exception.Message -notlike 'Refusing to overwrite an existing runtime harness:*') { + throw + } + $rejected = $true + } + if (!$rejected -or (Get-Content -LiteralPath $sentinel) -ne 'Existing candidate harness remains intact.') { + throw 'Copy-Worker must preserve both existing harness directories.' + } + Write-Output 'Passed worker-copy inventory, coexistence and overwrite checks.' + } + + $methodologyPath = Join-Path $root 'test' 'Dissemination.PerformanceHarness' 'methodology.json' + $methodology = Get-Content -LiteralPath $methodologyPath -Raw | ConvertFrom-Json + $scriptText = Get-Content -LiteralPath $script -Raw + $baselineMatch = [regex]::Match($scriptText, '(?m)^\$baselineSha = ''([0-9a-f]{40})''\r?$') + if (!$baselineMatch.Success -or $baselineMatch.Groups[1].Value -ne $methodology.baseline.commit) { + throw 'Script and methodology baseline pins differ.' + } + $baselineSha = $baselineMatch.Groups[1].Value + $controller = Get-Content -LiteralPath (Join-Path $root 'test' 'Dissemination.PerformanceHarness' 'Tests' 'ProcessCluster.cs') -Raw + $workflow = Get-Content -LiteralPath (Join-Path $PSScriptRoot '..' 'workflows' 'dissemination-performance.yml') -Raw + $readme = Get-Content -LiteralPath (Join-Path $root 'test' 'Dissemination.PerformanceHarness' 'README.md') -Raw + if (!$controller.Contains("public const string Baseline = `"$baselineSha`";") -or + $workflow -notmatch "repository: dotnet/orleans\s+ref: $baselineSha\b" -or !$readme.Contains($baselineSha)) { + throw 'Controller, workflow and documentation must retain the same pinned original runtime.' + } + Write-Output 'Passed baseline-pin consistency checks.' + $selection = @{ CandidateRepository = 'ReubenBond/orleans' CandidateRef = '3fc0a4610cd2eb5feef27e958fcca59658aaae47' diff --git a/test/Dissemination.PerformanceHarness/README.md b/test/Dissemination.PerformanceHarness/README.md index 5d1df314c68..e8942c95706 100644 --- a/test/Dissemination.PerformanceHarness/README.md +++ b/test/Dissemination.PerformanceHarness/README.md @@ -27,14 +27,21 @@ The workflow runs only through `workflow_dispatch`. GitHub requires a workflow to be registered on the repository's default branch before it can be dispatched. Select trusted public runtime revisions: their build code executes on the runner with read-only repository permissions and checkout credentials disabled. +The operator who can dispatch this workflow authorizes the exact repository/ref to +execute. Inspect that revision's build inputs and dependencies before dispatch; +selected code has the hosted runner's network access. Results record both the +requested ref and resolved commit for audit. CI creates separate original and candidate checkouts, verifies their origins and clean state, and copies only `Silo` and `Shared` support sources into each checkout. Each worker builds against that checkout's runtime projects and package settings. -The controller builds against the tooling checkout's shared protocol and instrumentation. +The controller builds against the tooling checkout's shared protocol, instrumentation +and reflection-boundary helpers. The tooling branch can therefore be based on main while the selected candidate provides the dissemination APIs. `NewRuntime.cs` describes the expected internal API shape; update this adapter deliberately when testing a candidate with changed APIs. +The shared publisher adapter validates the publication method's signature and the +disposable timer field, reporting the incompatible member and selected assembly. Published manifests record DLL hashes, versions and source commits. Every worker checks the loaded Orleans assemblies and exact process identity. Candidate off/on @@ -117,10 +124,11 @@ Run only the lightweight controller measurement cases locally: ```powershell .\.github\scripts\test-dissemination-performance.ps1 dotnet test --project test\Dissemination.PerformanceHarness\Tests\Dissemination.PerformanceHarness.Tests.csproj ` - --framework net10.0 --filter-class '*MeasurementTests' --minimum-expected-tests 15 + --framework net10.0 --filter-class '*MeasurementTests' --minimum-expected-tests 21 ``` -Ordinary Linux/Windows CI runs the script checks with worker execution stubbed. +Ordinary Linux/Windows CI runs script checks for repository containment, worker-copy +coexistence, baseline-pin consistency and replay paths with worker execution stubbed. `ProviderTests\FileMembershipTableTests.cs` is linked into `Orleans.Runtime.Tests` alongside the file-backed provider and its protocol. These BVT cases exercise persisted membership cleanup: all expired non-active statuses are removed, while diff --git a/test/Dissemination.PerformanceHarness/Shared/PublisherApi.cs b/test/Dissemination.PerformanceHarness/Shared/PublisherApi.cs new file mode 100644 index 00000000000..0ac9ce653a7 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Shared/PublisherApi.cs @@ -0,0 +1,33 @@ +using System.Reflection; + +namespace Orleans.Dissemination.PerformanceHarness; + +internal static class PublisherApi +{ + public static MethodInfo GetPublishMethod(Type publisherType) + { + var method = publisherType.GetMethod( + "PublishStatistics", BindingFlags.Instance | BindingFlags.NonPublic, [typeof(CancellationToken)]); + if (method is null || method.ReturnType != typeof(Task)) + { + throw Incompatible(publisherType, "Task PublishStatistics(CancellationToken)"); + } + + return method; + } + + public static FieldInfo GetTimerField(Type publisherType) + { + var field = publisherType.GetField("_publishTimer", BindingFlags.Instance | BindingFlags.NonPublic); + if (field is null || !typeof(IDisposable).IsAssignableFrom(field.FieldType)) + { + throw Incompatible(publisherType, "IDisposable _publishTimer"); + } + + return field; + } + + private static InvalidOperationException Incompatible(Type publisherType, string member) => + new($"Selected runtime '{publisherType.Assembly.FullName}' must provide {publisherType.FullName}.{member}. " + + "Update the performance harness runtime adapter for this candidate."); +} diff --git a/test/Dissemination.PerformanceHarness/Silo/Dissemination.Silo.csproj b/test/Dissemination.PerformanceHarness/Silo/Dissemination.Silo.csproj index 027816ec065..8ae421a7724 100644 --- a/test/Dissemination.PerformanceHarness/Silo/Dissemination.Silo.csproj +++ b/test/Dissemination.PerformanceHarness/Silo/Dissemination.Silo.csproj @@ -11,6 +11,7 @@ + diff --git a/test/Dissemination.PerformanceHarness/Silo/Program.cs b/test/Dissemination.PerformanceHarness/Silo/Program.cs index e320f1f9185..c6fe17ad534 100644 --- a/test/Dissemination.PerformanceHarness/Silo/Program.cs +++ b/test/Dissemination.PerformanceHarness/Silo/Program.cs @@ -19,8 +19,8 @@ namespace Orleans.Dissemination.PerformanceHarness; internal static class Program { - private static readonly MethodInfo PublishMethod = typeof(DeploymentLoadPublisher).GetMethod( - "PublishStatistics", BindingFlags.Instance | BindingFlags.NonPublic)!; + private static readonly MethodInfo PublishMethod = PublisherApi.GetPublishMethod(typeof(DeploymentLoadPublisher)); + private static readonly FieldInfo PublishTimerField = PublisherApi.GetTimerField(typeof(DeploymentLoadPublisher)); public static async Task Main(string[] args) { @@ -100,8 +100,7 @@ public static async Task Main(string[] args) // Commands own publication cadence; lifecycle shutdown retains its disposable timer reference. await publisher.RunOrQueueTask(() => { - var field = typeof(DeploymentLoadPublisher).GetField("_publishTimer", BindingFlags.Instance | BindingFlags.NonPublic)!; - ((IDisposable)field.GetValue(publisher)!).Dispose(); + ((IDisposable)PublishTimerField.GetValue(publisher)!).Dispose(); return Task.CompletedTask; }); await Write(new Response(0, Snapshot(host.Services), null)); diff --git a/test/Dissemination.PerformanceHarness/Tests/Dissemination.PerformanceHarness.Tests.csproj b/test/Dissemination.PerformanceHarness/Tests/Dissemination.PerformanceHarness.Tests.csproj index 39ef47e709d..1abb4a5f086 100644 --- a/test/Dissemination.PerformanceHarness/Tests/Dissemination.PerformanceHarness.Tests.csproj +++ b/test/Dissemination.PerformanceHarness/Tests/Dissemination.PerformanceHarness.Tests.csproj @@ -4,8 +4,9 @@ enable - + + diff --git a/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.PublisherApi.cs b/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.PublisherApi.cs new file mode 100644 index 00000000000..08a20654c3e --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.PublisherApi.cs @@ -0,0 +1,81 @@ +using Xunit; + +namespace Orleans.Dissemination.PerformanceHarness; + +public sealed partial class MeasurementTests +{ + [Theory] + [InlineData(typeof(object))] + [InlineData(typeof(WrongPublishParameters))] + [InlineData(typeof(WrongPublishResult))] + public void PublisherApiReportsMissingOrIncompatiblePublication(Type publisherType) + { + var error = Assert.Throws(() => PublisherApi.GetPublishMethod(publisherType)); + Assert.Contains(publisherType.FullName!, error.Message, StringComparison.Ordinal); + Assert.Contains("Task PublishStatistics(CancellationToken)", error.Message, StringComparison.Ordinal); + Assert.Contains("Update the performance harness runtime adapter", error.Message, StringComparison.Ordinal); + } + + [Theory] + [InlineData(typeof(object))] + [InlineData(typeof(WrongTimerType))] + public void PublisherApiReportsMissingOrIncompatibleTimer(Type publisherType) + { + var error = Assert.Throws(() => PublisherApi.GetTimerField(publisherType)); + Assert.Contains(publisherType.FullName!, error.Message, StringComparison.Ordinal); + Assert.Contains("IDisposable _publishTimer", error.Message, StringComparison.Ordinal); + Assert.Contains("Update the performance harness runtime adapter", error.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task PublisherApiPreservesPublicationTokenAndDisposedTimerReference() + { + var publisher = new CompatiblePublisher(); + using var cancellation = new CancellationTokenSource(); + var publishMethod = PublisherApi.GetPublishMethod(typeof(CompatiblePublisher)); + var timerField = PublisherApi.GetTimerField(typeof(CompatiblePublisher)); + var timer = Assert.IsType(timerField.GetValue(publisher)); + timer.Dispose(); + await (Task)publishMethod.Invoke(publisher, [cancellation.Token])!; + + Assert.True(timer.Disposed); + Assert.Same(timer, timerField.GetValue(publisher)); + Assert.Equal(cancellation.Token, publisher.LastToken); + } + + private sealed class CompatiblePublisher + { + private readonly IDisposable _publishTimer = new PublisherTimer(); + public CancellationToken LastToken { get; private set; } + + internal Task PublishStatistics(CancellationToken cancellationToken) + { + Assert.True(((PublisherTimer)_publishTimer).Disposed); + LastToken = cancellationToken; + return Task.CompletedTask; + } + } + + private sealed class PublisherTimer : IDisposable + { + public bool Disposed { get; private set; } + public void Dispose() => Disposed = true; + } + + private sealed class WrongPublishParameters + { + internal Task PublishStatistics() => Task.FromResult(this); + } + + private sealed class WrongPublishResult + { + internal ValueTask PublishStatistics(CancellationToken cancellationToken) => + new(Task.FromResult((this, cancellationToken))); + } + + private sealed class WrongTimerType + { + private readonly int _publishTimer = 1; + public int Timer => _publishTimer; + } +} diff --git a/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.cs b/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.cs index 23e12171c1d..71c8f48b820 100644 --- a/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.cs +++ b/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.cs @@ -9,7 +9,7 @@ namespace Orleans.Dissemination.PerformanceHarness; -public sealed class MeasurementTests +public sealed partial class MeasurementTests { [Theory] [InlineData(2)] From 33fffa2965fc52a6395d222f3e21e42da63ea8eb Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 16 Sep 2026 14:41:08 -0700 Subject: [PATCH 09/12] fix: reject linked worker copy destinations --- .github/scripts/dissemination-performance.ps1 | 1 + .../test-dissemination-performance.ps1 | 53 +++++++++++++++++++ .../README.md | 2 + 3 files changed, 56 insertions(+) diff --git a/.github/scripts/dissemination-performance.ps1 b/.github/scripts/dissemination-performance.ps1 index 3beac0351b9..e532a152246 100644 --- a/.github/scripts/dissemination-performance.ps1 +++ b/.github/scripts/dissemination-performance.ps1 @@ -95,6 +95,7 @@ function Assert-Checkout([string] $Directory, [string] $Repository) { function Copy-Worker([string] $Checkout) { $destination = Join-Path $Checkout 'test' 'Dissemination.PerformanceHarness' + Assert-NoLinks $destination if (Test-Path -LiteralPath $destination) { throw "Refusing to overwrite an existing runtime harness: $destination" } diff --git a/.github/scripts/test-dissemination-performance.ps1 b/.github/scripts/test-dissemination-performance.ps1 index 82a136a6cba..ef69b10f6d7 100644 --- a/.github/scripts/test-dissemination-performance.ps1 +++ b/.github/scripts/test-dissemination-performance.ps1 @@ -137,6 +137,59 @@ try { throw 'Copy-Worker must preserve both existing harness directories.' } Write-Output 'Passed worker-copy inventory, coexistence and overwrite checks.' + + foreach ($linkedPart in @('test', 'harness')) { + foreach ($dangling in @($false, $true)) { + $caseName = "$linkedPart-$dangling" + $linkedCheckout = Join-Path $testArtifacts "linked-checkout-$caseName" + $target = Join-Path $testArtifacts "outside-checkout-$caseName" + New-Item -ItemType Directory -Path $linkedCheckout, $target | Out-Null + $link = Join-Path $linkedCheckout 'test' + if ($linkedPart -eq 'harness') { + New-Item -ItemType Directory -Path $link | Out-Null + $link = Join-Path $link 'Dissemination.PerformanceHarness' + } + $linkType = if ($IsWindows) { 'Junction' } else { 'SymbolicLink' } + New-Item -ItemType $linkType -Path $link -Target $target | Out-Null + try { + if ($dangling) { + [System.IO.Directory]::Delete($target) + } + else { + Set-Content -LiteralPath (Join-Path $target 'preserved.txt') -Value 'Untouched link target.' + } + $rejected = $false + try { + Copy-Worker $linkedCheckout + } + catch { + if ($_.Exception.Message -ne "Artifact and runtime paths must use ordinary directories: $link") { + throw + } + $rejected = $true + } + if (!$rejected) { + throw "Copy-Worker accepted a linked destination parent: $caseName" + } + if ($dangling) { + if (Test-Path -LiteralPath $target) { + throw "Copy-Worker created the dangling link target: $caseName" + } + } + else { + $files = @(Get-ChildItem -LiteralPath $target -Force) + if ($files.Count -ne 1 -or $files[0].Name -ne 'preserved.txt' -or + (Get-Content -LiteralPath $files[0].FullName) -ne 'Untouched link target.') { + throw "Copy-Worker modified the link target: $caseName" + } + } + } + finally { + Remove-Item -LiteralPath $link -Force + } + } + } + Write-Output 'Passed 4 worker-copy linked-parent checks with untouched targets.' } $methodologyPath = Join-Path $root 'test' 'Dissemination.PerformanceHarness' 'methodology.json' diff --git a/test/Dissemination.PerformanceHarness/README.md b/test/Dissemination.PerformanceHarness/README.md index e8942c95706..ac503c7be96 100644 --- a/test/Dissemination.PerformanceHarness/README.md +++ b/test/Dissemination.PerformanceHarness/README.md @@ -34,6 +34,8 @@ requested ref and resolved commit for audit. CI creates separate original and candidate checkouts, verifies their origins and clean state, and copies only `Silo` and `Shared` support sources into each checkout. +Before creating worker files, the copy step validates every existing destination +parent through the checkout, rejecting symlinks and reparse points within it. Each worker builds against that checkout's runtime projects and package settings. The controller builds against the tooling checkout's shared protocol, instrumentation and reflection-boundary helpers. From 7f0ec09918643696dd29cc1cdadd93120f217dc7 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 16 Sep 2026 15:27:38 -0700 Subject: [PATCH 10/12] feat: add bounded open-loop dissemination workloads --- .github/scripts/dissemination-performance.ps1 | 10 +- .../test-dissemination-performance.ps1 | 27 +- .../workflows/dissemination-performance.yml | 11 +- .../README.md | 48 +++- .../Shared/AggregationFanout.cs | 32 +++ .../Shared/OpenLoopProducer.cs | 148 ++++++++++ .../Shared/Protocol.cs | 5 +- .../Silo/Dissemination.Silo.csproj | 2 + .../Silo/NewRuntime.cs | 8 +- .../Silo/Program.cs | 34 ++- ...semination.PerformanceHarness.Tests.csproj | 3 + .../MeasurementTests.AggregationFanout.cs | 42 +++ .../Tests/MeasurementTests.OpenLoop.cs | 265 ++++++++++++++++++ .../Tests/OpenLoopMeasurements.cs | 105 +++++++ .../Tests/ProcessCluster.OpenLoop.cs | 94 +++++++ .../Tests/ProcessCluster.cs | 7 +- .../Tests/ScalingComparison.cs | 48 +++- .../Tests/ScalingTests.cs | 36 ++- .../methodology.json | 10 +- .../Orleans.Runtime.Tests.csproj | 1 + 20 files changed, 902 insertions(+), 34 deletions(-) create mode 100644 test/Dissemination.PerformanceHarness/Shared/AggregationFanout.cs create mode 100644 test/Dissemination.PerformanceHarness/Shared/OpenLoopProducer.cs create mode 100644 test/Dissemination.PerformanceHarness/Tests/MeasurementTests.AggregationFanout.cs create mode 100644 test/Dissemination.PerformanceHarness/Tests/MeasurementTests.OpenLoop.cs create mode 100644 test/Dissemination.PerformanceHarness/Tests/OpenLoopMeasurements.cs create mode 100644 test/Dissemination.PerformanceHarness/Tests/ProcessCluster.OpenLoop.cs diff --git a/.github/scripts/dissemination-performance.ps1 b/.github/scripts/dissemination-performance.ps1 index e532a152246..579fd593fb8 100644 --- a/.github/scripts/dissemination-performance.ps1 +++ b/.github/scripts/dissemination-performance.ps1 @@ -18,6 +18,8 @@ param( [string] $RuntimePaths = 'All', [ValidatePattern('^(stable|churn|partition)(,(stable|churn|partition)){0,2}$')] [string] $Scenarios = 'stable,churn,partition', + [ValidateSet('ClosedLoop', 'OpenLoopSynchronized', 'OpenLoopStaggered')] + [string] $Workload = 'ClosedLoop', [ValidateRange(0, 64)] [int] $SiloProcessorCount = 0, [ValidateRange(0, 9)] @@ -38,6 +40,10 @@ if (@($sizesArray | Select-Object -Unique).Count -ne $sizesArray.Count -or if (@($Scenarios.Split(',') | Select-Object -Unique).Count -ne $Scenarios.Split(',').Count) { throw 'Select each scenario at most once.' } +if ($Workload -ne 'ClosedLoop' -and ($Scenarios -ne 'stable' -or $Iterations -gt 30 -or + @($sizesArray | Where-Object { $_ -gt 32 }).Count -gt 0)) { + throw 'Open-loop workloads require stable only, 3..32 silo processes and 3..30 seconds (Iterations).' +} function Assert-NoLinks([string] $Path) { $current = [System.IO.Path]::GetFullPath($Path) @@ -262,6 +268,7 @@ $env:ORLEANS_DISSEMINATION_ITERATIONS = $Iterations.ToString([System.Globalizati $env:ORLEANS_DISSEMINATION_REPETITIONS = $Repetitions.ToString([System.Globalization.CultureInfo]::InvariantCulture) $env:ORLEANS_DISSEMINATION_RUNTIME_PATHS = $RuntimePaths $env:ORLEANS_DISSEMINATION_SCENARIOS = $Scenarios +$env:ORLEANS_DISSEMINATION_WORKLOAD = $Workload $env:ORLEANS_DISSEMINATION_SILO_PROCESSOR_COUNT = $SiloProcessorCount.ToString([System.Globalization.CultureInfo]::InvariantCulture) $env:ORLEANS_DISSEMINATION_GC_CONSERVE_MEMORY = $GCConserveMemory.ToString([System.Globalization.CultureInfo]::InvariantCulture) @{ @@ -272,6 +279,7 @@ $env:ORLEANS_DISSEMINATION_GC_CONSERVE_MEMORY = $GCConserveMemory.ToString([Syst Repetitions = $Repetitions RuntimePaths = $RuntimePaths Scenarios = $Scenarios + Workload = $Workload SiloProcessorCount = $SiloProcessorCount GCConserveMemory = $GCConserveMemory StartUtc = [DateTimeOffset]::UtcNow @@ -282,7 +290,7 @@ Push-Location $root try { Invoke-DotNet @( $runner, '--filter-class', 'Orleans.Dissemination.PerformanceHarness.MeasurementTests', - '--minimum-expected-tests', '21', '--report-trx', + '--minimum-expected-tests', '50', '--report-trx', '--results-directory', (Join-Path $results 'instrument-checks') ) Invoke-DotNet @( diff --git a/.github/scripts/test-dissemination-performance.ps1 b/.github/scripts/test-dissemination-performance.ps1 index ef69b10f6d7..97e284f3504 100644 --- a/.github/scripts/test-dissemination-performance.ps1 +++ b/.github/scripts/test-dissemination-performance.ps1 @@ -25,6 +25,10 @@ $cases = @( @{ Name = 'ref option'; Parameter = 'CandidateRef'; Value = '--upload-pack=command'; Error = '*cannot validate argument*' } @{ Name = 'processor bound'; Parameter = 'SiloProcessorCount'; Value = 65; Error = '*cannot validate argument*' } @{ Name = 'GC bound'; Parameter = 'GCConserveMemory'; Value = 10; Error = '*cannot validate argument*' } + @{ Name = 'unknown workload'; Parameter = 'Workload'; Value = 'unknown'; Error = '*cannot validate argument*' } + @{ Name = 'open-loop stable only'; Parameter = 'Workload'; Value = 'OpenLoopSynchronized'; Error = 'Open-loop workloads require*' } + @{ Name = 'open-loop duration bound'; Parameter = 'Iterations'; Value = 31; Additional = @{ Workload = 'OpenLoopStaggered'; Scenarios = 'stable' }; Error = 'Open-loop workloads require*' } + @{ Name = 'open-loop process bound'; Parameter = 'Sizes'; Value = '4,33'; Additional = @{ Workload = 'OpenLoopSynchronized'; Scenarios = 'stable' }; Error = 'Open-loop workloads require*' } ) foreach ($case in $cases) { $arguments = @{ @@ -33,6 +37,11 @@ foreach ($case in $cases) { SkipBuild = $true } $arguments[$case.Parameter] = $case.Value + if ($case.ContainsKey('Additional')) { + foreach ($entry in $case.Additional.GetEnumerator()) { + $arguments[$entry.Key] = $entry.Value + } + } $rejected = $false try { & $script @arguments @@ -231,8 +240,16 @@ try { & $script -CandidateRepository $selection.CandidateRepository -CandidateRef $selection.CandidateRef ` -ArtifactsDirectory "Artifacts$separator$artifactName" -Sizes '3' -Iterations 3 -Repetitions 1 -SkipBuild } - if ($calls.Count -ne 4) { - throw "Expected measurement and scaling invocations for both path spellings; got $($calls.Count)." + foreach ($workload in @('OpenLoopSynchronized', 'OpenLoopStaggered')) { + & $script -CandidateRepository $selection.CandidateRepository -CandidateRef $selection.CandidateRef ` + -ArtifactsDirectory (Join-Path 'Artifacts' $artifactName) -Sizes '3' -Iterations 30 -Repetitions 1 ` + -Scenarios stable -Workload $workload -SkipBuild + if ($env:ORLEANS_DISSEMINATION_WORKLOAD -ne $workload -or $env:ORLEANS_DISSEMINATION_ITERATIONS -ne '30') { + throw "Open-loop invocation lost its workload selection: $workload" + } + } + if ($calls.Count -ne 8) { + throw "Expected measurement and scaling invocations for two path spellings and two open-loop patterns; got $($calls.Count)." } for ($index = 0; $index -lt $calls.Count; $index++) { $expectedClass = if ($index % 2 -eq 0) { 'MeasurementTests' } else { 'ScalingTests' } @@ -241,8 +258,8 @@ try { } } $runs = @(Get-ChildItem -LiteralPath (Join-Path $testArtifacts 'results') -Directory) - if ($runs.Count -ne 2) { - throw "Expected two results directories under the same artifact root; got $($runs.Count)." + if ($runs.Count -ne 4) { + throw "Expected four results directories under the same artifact root; got $($runs.Count)." } foreach ($run in $runs) { foreach ($filename in @('invocation.json', 'selection.json', 'methodology.json', 'host-resources.json')) { @@ -253,7 +270,7 @@ try { throw 'The recorded results path differs from the created directory.' } } - Write-Output 'Passed 2 SkipBuild path checks with worker execution stubbed.' + Write-Output 'Passed 4 SkipBuild path/workload checks with worker execution stubbed.' } finally { foreach ($entry in Get-ChildItem Env:ORLEANS_DISSEMINATION_*) { diff --git a/.github/workflows/dissemination-performance.yml b/.github/workflows/dissemination-performance.yml index 1819728efd8..ebc569c2a82 100644 --- a/.github/workflows/dissemination-performance.yml +++ b/.github/workflows/dissemination-performance.yml @@ -18,7 +18,7 @@ on: type: string default: '4,8' iterations: - description: 'Publication rounds per scenario (3..200)' + description: 'Closed-loop rounds (3..200), or open-loop 1 Hz duration in seconds (3..30)' type: number default: 3 repetitions: @@ -34,6 +34,11 @@ on: description: 'Distinct comma-separated scenarios: stable,churn,partition' type: string default: 'stable,churn,partition' + workload: + description: 'Open-loop requires stable only and at most 32 processes' + type: choice + options: [ClosedLoop, OpenLoopSynchronized, OpenLoopStaggered] + default: ClosedLoop silo_processor_count: description: 'Logical processors per silo (0 = inherit runner, maximum 64)' type: number @@ -94,6 +99,7 @@ jobs: PERFORMANCE_REPETITIONS: ${{ inputs.repetitions }} PERFORMANCE_PATHS: ${{ inputs.runtime_paths }} PERFORMANCE_SCENARIOS: ${{ inputs.scenarios }} + PERFORMANCE_WORKLOAD: ${{ inputs.workload }} PERFORMANCE_PROCESSORS: ${{ inputs.silo_processor_count }} PERFORMANCE_GC: ${{ inputs.gc_conserve_memory }} run: | @@ -101,7 +107,8 @@ jobs: -CandidateRepository $env:CANDIDATE_REPOSITORY -CandidateRef $env:CANDIDATE_REF ` -Sizes $env:PERFORMANCE_SIZES -Iterations ([int]$env:PERFORMANCE_ITERATIONS) ` -Repetitions ([int]$env:PERFORMANCE_REPETITIONS) -RuntimePaths $env:PERFORMANCE_PATHS ` - -Scenarios $env:PERFORMANCE_SCENARIOS -SiloProcessorCount ([int]$env:PERFORMANCE_PROCESSORS) ` + -Scenarios $env:PERFORMANCE_SCENARIOS -Workload $env:PERFORMANCE_WORKLOAD ` + -SiloProcessorCount ([int]$env:PERFORMANCE_PROCESSORS) ` -GCConserveMemory ([int]$env:PERFORMANCE_GC) - name: Preserve raw measurements and build diagnostics if: always() diff --git a/test/Dissemination.PerformanceHarness/README.md b/test/Dissemination.PerformanceHarness/README.md index ac503c7be96..960802e07c8 100644 --- a/test/Dissemination.PerformanceHarness/README.md +++ b/test/Dissemination.PerformanceHarness/README.md @@ -20,6 +20,7 @@ gh workflow run dissemination-performance.yml --repo ReubenBond/orleans ` -f candidate_ref=3fc0a4610cd2eb5feef27e958fcca59658aaae47 ` -f sizes=4,8 -f iterations=3 -f repetitions=2 ` -f runtime_paths=All -f scenarios=stable,churn,partition ` + -f workload=ClosedLoop ` -f silo_processor_count=0 -f gc_conserve_memory=0 ``` @@ -44,6 +45,11 @@ provides the dissemination APIs. `NewRuntime.cs` describes the expected internal shape; update this adapter deliberately when testing a candidate with changed APIs. The shared publisher adapter validates the publication method's signature and the disposable timer field, reporting the incompatible member and selected assembly. +Aggregation-tree reports use the optional public `Overlay.AggregationFanOutFactor` +property when present, clamped to the candidate's effective range +`1..max(1, memberCount)`. Older candidates such as `3fc0a461` use their membership +fanout formula. `FanoutSource` names the detected capability; an incompatible +property shape raises an adapter diagnostic. Membership routing retains its selector. Published manifests record DLL hashes, versions and source commits. Every worker checks the loaded Orleans assemblies and exact process identity. Candidate off/on @@ -57,6 +63,39 @@ paths across `stable`, `churn` and `partition`. Sizes accept one to six distinct counts from 3 through 128; rounds accept 3 through 200; repetitions accept 1 through 5. Successive repetitions reverse the complete execution order, including off/on. +`workload=ClosedLoop` preserves the publication/convergence barrier after each round. +For fixed-rate stable measurement, choose `OpenLoopSynchronized` or +`OpenLoopStaggered`, `scenarios=stable`, 3 through 32 processes, and `iterations` +from 3 through 30 (duration in seconds at one publication per producer per second). +Synchronized producers share each one-second boundary; staggered producers use +evenly spaced phases within that second. Each worker arms a local monotonic schedule +at a common future UTC epoch, then calls the production publisher on its scheduler. +Controller polling observes state independently of these publication deadlines. + +```powershell +gh workflow run dissemination-performance.yml --repo ReubenBond/orleans ` + --ref -f candidate_repository=ReubenBond/orleans ` + -f candidate_ref= -f sizes=4,8,32 -f iterations=10 ` + -f repetitions=2 -f runtime_paths=Current -f scenarios=stable ` + -f workload=OpenLoopStaggered -f silo_processor_count=0 -f gc_conserve_memory=0 +``` + +Arming must finish within the five-second lead. A producer fails on a missed period +or a publication extending past its one-second slot, preserving partial timestamps, +missed-period and overrun counts. Each accepted run offers exactly +`processes * durationSeconds` publications. Producers retain the configured rate +without catch-up bursts. The controller has a separate ten-second completion and +latest-state drain after the scheduled end. It verifies active process identities, +exact latest dictionary inventory and values, and that observed versions and values +were actually published. + +Open-loop artifacts preserve planned/actual start and completion timestamps, +actual inter-publication periods, first-observed peer ages, and each producer's +final value reaching every process. Ages and final-latest latency are **polling +upper bounds**, using same-host UTC timestamps and 100 ms controller polling. +Intermediate values may coalesce; observed and unobserved publication/observer pairs +are reported separately. Exact latest-state convergence remains required. + Every round invokes the actual load publisher once per active node and checks exact state inventory and field values at every node. Churn restarts a process at the same endpoint. Partition recovery drains connections at both endpoints, including @@ -91,9 +130,14 @@ bytes, CPU, allocations, retained memory, publication/convergence/recovery laten and per-node topology. Load broadcast requests and logical value transmissions are separate counters. Reports compare equal binary, environment, process count and offered work, with costs per publication and median/P95 convergence. +Open-loop comparisons use median/P95 final-latest polling upper bounds instead of +per-round closed-loop latency. Raw `open-loop-{plans,producers,observations,summary}.json` +files preserve cadence and observation evidence. Total costs include arming, the +fixed-rate window, final-state drain, and the common socket-sampling margin. These are shared-runner, single-host loopback measurements. The workload is -closed-loop, with a convergence barrier after each round. Silo costs include control +closed-loop by default, with a convergence barrier after each round. The optional +open-loop mode fixes the offered producer rate at 1 Hz. Silo costs include control and instrumentation work; controller costs, storage disk bytes, cross-machine latency, TCP/IP wire overhead and application-grain workloads need separate experiments. The 100 ms polling cadence and one-second socket sampling introduce boundary @@ -126,7 +170,7 @@ Run only the lightweight controller measurement cases locally: ```powershell .\.github\scripts\test-dissemination-performance.ps1 dotnet test --project test\Dissemination.PerformanceHarness\Tests\Dissemination.PerformanceHarness.Tests.csproj ` - --framework net10.0 --filter-class '*MeasurementTests' --minimum-expected-tests 21 + --framework net10.0 --filter-class '*MeasurementTests' --minimum-expected-tests 50 ``` Ordinary Linux/Windows CI runs script checks for repository containment, worker-copy diff --git a/test/Dissemination.PerformanceHarness/Shared/AggregationFanout.cs b/test/Dissemination.PerformanceHarness/Shared/AggregationFanout.cs new file mode 100644 index 00000000000..fb74a3024b2 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Shared/AggregationFanout.cs @@ -0,0 +1,32 @@ +using System.Reflection; + +namespace Orleans.Dissemination.PerformanceHarness; + +internal sealed record AggregationFanout(int Value, string Source) +{ + public static AggregationFanout Read(object overlay, bool aggregationTree, int memberCount, Func legacyFanout) + { + if (!aggregationTree) + { + return new(legacyFanout(), "MembershipFanout"); + } + + var type = overlay.GetType(); + var property = type.GetProperty("AggregationFanOutFactor", BindingFlags.Instance | BindingFlags.Public); + if (property is null) + { + return new(legacyFanout(), "LegacyAggregationMembershipFanout"); + } + + if (property.PropertyType != typeof(int) || property.GetIndexParameters().Length != 0 + || property.GetMethod is not { IsPublic: true } + || property.GetValue(overlay) is not int value) + { + throw new InvalidOperationException( + $"Selected runtime '{type.Assembly.FullName}' must expose a public int {type.FullName}.AggregationFanOutFactor. " + + "Update the performance harness runtime adapter for this candidate."); + } + + return new(Math.Clamp(value, 1, Math.Max(1, memberCount)), "Overlay.AggregationFanOutFactor"); + } +} diff --git a/test/Dissemination.PerformanceHarness/Shared/OpenLoopProducer.cs b/test/Dissemination.PerformanceHarness/Shared/OpenLoopProducer.cs new file mode 100644 index 00000000000..5f4eda4d3ac --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Shared/OpenLoopProducer.cs @@ -0,0 +1,148 @@ +namespace Orleans.Dissemination.PerformanceHarness; + +internal sealed record OpenLoopPlan(DateTimeOffset StartUtc, int DurationSeconds, int ProducerIndex, int ProducerCount, string Workload) +{ + public void Validate() + { + if (DurationSeconds is < 3 or > 30 || ProducerCount is < 3 or > 32 + || ProducerIndex < 0 || ProducerIndex >= ProducerCount + || Workload is not ("OpenLoopSynchronized" or "OpenLoopStaggered")) + { + throw new InvalidOperationException("Open-loop requires 3..32 producers, 3..30 seconds, and a synchronized or staggered pattern."); + } + } + + public TimeSpan Offset(int sequence) => + TimeSpan.FromSeconds(sequence) + (Workload == "OpenLoopStaggered" + ? TimeSpan.FromTicks(TimeSpan.TicksPerSecond * ProducerIndex / ProducerCount) + : TimeSpan.Zero); +} + +internal sealed record ProducedValue(DateTimeOffset StartedAtUtc, DateTimeOffset CompletedAtUtc, long Version, string Value); +internal sealed record OpenLoopPublication(int Sequence, DateTimeOffset ScheduledAtUtc, ProducedValue Result); +internal sealed record OpenLoopReport(OpenLoopPlan Plan, OpenLoopPublication[] Publications, int MissedPeriods, int Overruns); +internal sealed record OpenLoopProgress(int PublishedCount, bool Completed, int MissedPeriods, int Overruns, string? Error); + +internal sealed class OpenLoopProducer(TimeProvider timeProvider) : IAsyncDisposable +{ + private CancellationTokenSource? _cancellation; + private Task? _completion; + private int _published; + private int _missedPeriods; + private int _overruns; + private OpenLoopPlan? _plan; + private readonly List _publications = []; + private readonly object _lock = new(); + + public Task Completion => _completion + ?? throw new InvalidOperationException("The open-loop producer has not been armed."); + + public OpenLoopProgress? Progress => _completion is { } task + ? new(Volatile.Read(ref _published), task.IsCompletedSuccessfully, Volatile.Read(ref _missedPeriods), + Volatile.Read(ref _overruns), task.Exception?.GetBaseException().ToString()) + : null; + + public OpenLoopReport Report + { + get + { + lock (_lock) + { + return new(_plan ?? throw new InvalidOperationException("The open-loop producer has not been armed."), + _publications.ToArray(), _missedPeriods, _overruns); + } + } + } + + public void Start(OpenLoopPlan plan, Func> publish, CancellationToken cancellationToken) + { + if (_completion is not null) + { + throw new InvalidOperationException("The open-loop producer has already been armed."); + } + + plan.Validate(); + var lead = plan.StartUtc - timeProvider.GetUtcNow(); + if (lead <= TimeSpan.Zero || lead > TimeSpan.FromSeconds(10)) + { + throw new InvalidOperationException("Open-loop start must be in the next 10 seconds; all workers must acknowledge arming before it."); + } + + _cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + _plan = plan; + _completion = Run(plan, lead, publish, _cancellation.Token); + _ = _completion.ContinueWith( + static task => _ = task.Exception, CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); + } + + private async Task Run( + OpenLoopPlan plan, TimeSpan lead, Func> publish, CancellationToken cancellationToken) + { + var origin = timeProvider.GetTimestamp(); + for (var sequence = 0; sequence < plan.DurationSeconds; sequence++) + { + var due = lead + plan.Offset(sequence); + await DelayUntil(due); + var elapsed = timeProvider.GetElapsedTime(origin); + if (elapsed - due >= TimeSpan.FromSeconds(1)) + { + Volatile.Write(ref _missedPeriods, (int)(elapsed - due).TotalSeconds); + throw new InvalidOperationException($"Open-loop producer {plan.ProducerIndex} missed publication slot {sequence}."); + } + + var result = await publish(cancellationToken); + lock (_lock) + { + if (_publications.Count > 0 && result.Version <= _publications[^1].Result.Version) + { + throw new InvalidOperationException("The production publisher did not create a strictly newer sample."); + } + _publications.Add(new(sequence, plan.StartUtc + plan.Offset(sequence), result)); + Volatile.Write(ref _published, _publications.Count); + } + if (timeProvider.GetElapsedTime(origin) - due >= TimeSpan.FromSeconds(1)) + { + Volatile.Write(ref _overruns, 1); + throw new InvalidOperationException($"Open-loop producer {plan.ProducerIndex} publication {sequence} exceeded its 1 Hz slot."); + } + } + + await DelayUntil(lead + TimeSpan.FromSeconds(plan.DurationSeconds)); + return Report; + + async Task DelayUntil(TimeSpan due) + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + var remaining = due - timeProvider.GetElapsedTime(origin); + if (remaining <= TimeSpan.Zero) + { + return; + } + await Task.Delay(TimeSpan.FromMilliseconds(Math.Ceiling(remaining.TotalMilliseconds)), timeProvider, cancellationToken); + } + } + } + + public async ValueTask DisposeAsync() + { + if (_cancellation is { } cancellation) + { + await cancellation.CancelAsync(); + try + { + await Completion.WaitAsync(TimeSpan.FromSeconds(5)); + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + } + finally + { + cancellation.Dispose(); + _cancellation = null; + } + } + } +} diff --git a/test/Dissemination.PerformanceHarness/Shared/Protocol.cs b/test/Dissemination.PerformanceHarness/Shared/Protocol.cs index 96840b57006..4ea011d22d5 100644 --- a/test/Dissemination.PerformanceHarness/Shared/Protocol.cs +++ b/test/Dissemination.PerformanceHarness/Shared/Protocol.cs @@ -12,7 +12,7 @@ internal sealed record NodeConfiguration( int SiloProcessorCount = 0, int GCConserveMemory = 0); -internal sealed record Command(int Id, string Operation, string? Peer = null); +internal sealed record Command(int Id, string Operation, string? Peer = null, OpenLoopPlan? OpenLoop = null); internal sealed record Response(int Id, NodeSnapshot? Snapshot, string? Error); @@ -64,6 +64,7 @@ internal sealed record NodeSnapshot public required string[] ForwardingTargets { get; init; } public string[] TopologyMembers { get; init; } = []; public int Fanout { get; init; } + public string? FanoutSource { get; init; } public bool AggregationTree { get; init; } public int AntiEntropyPeerCount { get; init; } public double AntiEntropyIntervalMilliseconds { get; init; } @@ -83,4 +84,6 @@ internal sealed record NodeSnapshot public bool Partitioned { get; init; } public int? RemoteProcessId { get; init; } public string? ProbeError { get; init; } + public OpenLoopProgress? OpenLoopProgress { get; init; } + public OpenLoopReport? OpenLoopReport { get; init; } } diff --git a/test/Dissemination.PerformanceHarness/Silo/Dissemination.Silo.csproj b/test/Dissemination.PerformanceHarness/Silo/Dissemination.Silo.csproj index 8ae421a7724..7a4162a987b 100644 --- a/test/Dissemination.PerformanceHarness/Silo/Dissemination.Silo.csproj +++ b/test/Dissemination.PerformanceHarness/Silo/Dissemination.Silo.csproj @@ -12,6 +12,8 @@ + + diff --git a/test/Dissemination.PerformanceHarness/Silo/NewRuntime.cs b/test/Dissemination.PerformanceHarness/Silo/NewRuntime.cs index 4e19c9b1f84..03412525df3 100644 --- a/test/Dissemination.PerformanceHarness/Silo/NewRuntime.cs +++ b/test/Dissemination.PerformanceHarness/Silo/NewRuntime.cs @@ -28,6 +28,9 @@ public static NodeSnapshot Decorate(IServiceProvider services, NodeSnapshot snap var ns = services.GetRequiredService(); var topology = services.GetRequiredService().CurrentSnapshots.ActiveMembers; + var aggregationTree = ns.RoutingMode == DisseminationRoutingMode.AggregationTree; + var fanout = AggregationFanout.Read( + options.Overlay, aggregationTree, topology.Members.Length, () => options.Overlay.GetFanOutFactor(topology.Members.Length)); return snapshot with { Enabled = options.Enabled, @@ -37,8 +40,9 @@ public static NodeSnapshot Decorate(IServiceProvider services, NodeSnapshot snap OriginatorTargets = topology.GetOriginatorTargets(ns.RoutingMode).Select(address => address.ToParsableString()).ToArray(), ForwardingTargets = topology.GetForwardingTargets(ns.RoutingMode).Select(address => address.ToParsableString()).ToArray(), TopologyMembers = topology.Members.Select(address => address.ToParsableString()).ToArray(), - Fanout = options.Overlay.GetFanOutFactor(topology.Members.Length), - AggregationTree = ns.RoutingMode == DisseminationRoutingMode.AggregationTree, + Fanout = fanout.Value, + FanoutSource = fanout.Source, + AggregationTree = aggregationTree, AntiEntropyPeerCount = options.Overlay.AntiEntropyPeerCount, AntiEntropyIntervalMilliseconds = options.Overlay.AntiEntropyInterval.TotalMilliseconds, }; diff --git a/test/Dissemination.PerformanceHarness/Silo/Program.cs b/test/Dissemination.PerformanceHarness/Silo/Program.cs index c6fe17ad534..0b68c90a763 100644 --- a/test/Dissemination.PerformanceHarness/Silo/Program.cs +++ b/test/Dissemination.PerformanceHarness/Silo/Program.cs @@ -39,6 +39,7 @@ public static async Task Main(string[] args) #endif using var lifetime = new CancellationTokenSource(TimeSpan.FromMinutes(45)); using var observation = new Observation(); + var openLoop = new OpenLoopProducer(TimeProvider.System); var membership = new FileMembershipTable(configuration); using var host = TestClusterHostFactory.CreateSiloHost( configuration.Name, @@ -47,6 +48,7 @@ public static async Task Main(string[] args) { services.AddSingleton(configuration); services.AddSingleton(observation); + services.AddSingleton(openLoop); services.AddSingleton(membership); services.AddSingleton(membership); services.AddSingleton(); @@ -112,7 +114,7 @@ await publisher.RunOrQueueTask(() => operation.CancelAfter(TimeSpan.FromSeconds(60)); try { - var result = await Execute(host.Services, command, operation.Token); + var result = await Execute(host.Services, command, operation.Token, lifetime.Token); await Write(new Response(command.Id, result, null)); if (command.Operation == "stop") { @@ -126,6 +128,7 @@ await publisher.RunOrQueueTask(() => } using var shutdown = new CancellationTokenSource(TimeSpan.FromSeconds(20)); + await openLoop.DisposeAsync(); await host.StopAsync(shutdown.Token).WaitAsync(shutdown.Token); // Drain the last one-second OS socket-counter bucket before recording shutdown cost. await Task.Delay(TimeSpan.FromMilliseconds(1100)); @@ -135,11 +138,13 @@ await publisher.RunOrQueueTask(() => finally { await lifetime.CancelAsync(); + await openLoop.DisposeAsync(); await monitor; } } - private static async Task Execute(IServiceProvider services, Command command, CancellationToken cancellationToken) + private static async Task Execute( + IServiceProvider services, Command command, CancellationToken cancellationToken, CancellationToken lifetimeToken) { var manager = services.GetRequiredService(); var publisher = services.GetRequiredService(); @@ -158,6 +163,22 @@ await publisher.RunOrQueueTask(() => (Task)PublishMethod.Invoke(publisher, [canc .WaitAsync(cancellationToken); break; + case "open-loop-arm": + services.GetRequiredService().Start( + command.OpenLoop ?? throw new InvalidOperationException("Missing open-loop plan."), + token => publisher.RunOrQueueTaskResult(async () => + { + token.ThrowIfCancellationRequested(); + var startedAt = DateTimeOffset.UtcNow; + await (Task)PublishMethod.Invoke(publisher, [token])!; + var local = services.GetRequiredService().SiloAddress; + var value = publisher.PeriodicStatistics[local]; + return new ProducedValue(startedAt, DateTimeOffset.UtcNow, value.DateTime.Ticks, StateComparison.Serialize(value)); + }).Unwrap(), lifetimeToken); + break; + case "open-loop-finish": + var report = await services.GetRequiredService().Completion.WaitAsync(cancellationToken); + return Snapshot(services) with { OpenLoopReport = report }; case "partition": services.GetRequiredService().Partition(); break; @@ -233,6 +254,9 @@ internal static NodeSnapshot Snapshot(IServiceProvider services, bool detailed = var membership = manager.CurrentSnapshot; using var process = Process.GetCurrentProcess(); var socket = observation.SocketSnapshot; + var load = publisher.PeriodicStatistics.ToArray(); + var openLoop = services.GetRequiredService(); + var progress = openLoop.Progress; var result = new NodeSnapshot { CapturedAtUtc = DateTimeOffset.UtcNow, @@ -244,9 +268,9 @@ internal static NodeSnapshot Snapshot(IServiceProvider services, bool detailed = pair => JsonSerializer.Serialize(FileMembershipTable.EntryData.From(pair.Value))), StringComparer.Ordinal), ActiveMembers = membership.Entries.Values.Where(entry => entry.Status == SiloStatus.Active) .Select(entry => entry.SiloAddress.ToParsableString()).Order(StringComparer.Ordinal).ToArray(), - Load = new(publisher.PeriodicStatistics.ToDictionary( + Load = new(load.ToDictionary( pair => pair.Key.ToParsableString(), pair => StateComparison.Serialize(pair.Value)), StringComparer.Ordinal), - LoadVersions = new(publisher.PeriodicStatistics.ToDictionary( + LoadVersions = new(load.ToDictionary( pair => pair.Key.ToParsableString(), pair => pair.Value.DateTime.Ticks), StringComparer.Ordinal), UnconfirmedPeers = [], OriginatorTargets = [], @@ -265,6 +289,8 @@ internal static NodeSnapshot Snapshot(IServiceProvider services, bool detailed = ProcessorCount = Environment.ProcessorCount, ServerGC = System.Runtime.GCSettings.IsServerGC, Partitioned = observation.Partitioned, + OpenLoopProgress = progress, + OpenLoopReport = progress?.Error is not null ? openLoop.Report : null, }; #if NEW_RUNTIME result = NewRuntime.Decorate(services, result); diff --git a/test/Dissemination.PerformanceHarness/Tests/Dissemination.PerformanceHarness.Tests.csproj b/test/Dissemination.PerformanceHarness/Tests/Dissemination.PerformanceHarness.Tests.csproj index 1abb4a5f086..2f57d3b8d7a 100644 --- a/test/Dissemination.PerformanceHarness/Tests/Dissemination.PerformanceHarness.Tests.csproj +++ b/test/Dissemination.PerformanceHarness/Tests/Dissemination.PerformanceHarness.Tests.csproj @@ -7,6 +7,9 @@ + + + diff --git a/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.AggregationFanout.cs b/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.AggregationFanout.cs new file mode 100644 index 00000000000..7466d6ffd2a --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.AggregationFanout.cs @@ -0,0 +1,42 @@ +using Xunit; + +namespace Orleans.Dissemination.PerformanceHarness; + +public sealed partial class MeasurementTests +{ + [Theory] + [InlineData(0, 1)] + [InlineData(4, 4)] + [InlineData(32, 8)] + public void AggregationFanoutReportsTheCandidateRoutingClamp(int members, int expected) + { + var result = AggregationFanout.Read(new ModernOverlay(), true, members, + () => throw new InvalidOperationException("A modern aggregation tree must use its own fanout.")); + Assert.Equal(new(expected, "Overlay.AggregationFanOutFactor"), result); + } + + [Fact] + public void AggregationFanoutIdentifiesLegacyCapabilityAndPreservesMembershipRouting() + { + Assert.Equal(new(7, "LegacyAggregationMembershipFanout"), AggregationFanout.Read(new object(), true, 32, () => 7)); + Assert.Equal(new(5, "MembershipFanout"), AggregationFanout.Read(new ModernOverlay(), false, 32, () => 5)); + } + + [Fact] + public void AggregationFanoutRejectsIncompatibleCandidateCapability() + { + var exception = Assert.Throws(() => AggregationFanout.Read(new IncompatibleOverlay(), true, 32, () => 7)); + Assert.Contains("public int", exception.Message, StringComparison.Ordinal); + Assert.Contains("AggregationFanOutFactor", exception.Message, StringComparison.Ordinal); + } + + private sealed class ModernOverlay + { + public int AggregationFanOutFactor => 8; + } + + private sealed class IncompatibleOverlay + { + public string AggregationFanOutFactor => "8"; + } +} diff --git a/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.OpenLoop.cs b/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.OpenLoop.cs new file mode 100644 index 00000000000..a8be7cf825f --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.OpenLoop.cs @@ -0,0 +1,265 @@ +using System.Threading.Channels; +using Microsoft.Extensions.Time.Testing; +using Xunit; + +namespace Orleans.Dissemination.PerformanceHarness; + +public sealed partial class MeasurementTests +{ + [Theory] + [InlineData("OpenLoopSynchronized", 0)] + [InlineData("OpenLoopStaggered", 250)] + public async Task OpenLoopKeepsOneHertzDeadlinesIndependentOfPublicationDuration(string workload, int phaseMilliseconds) + { + var clock = new RecordingClock(); + var epoch = clock.GetUtcNow().AddSeconds(1); + var producer = new OpenLoopProducer(clock); + await using var cleanup = producer; + producer.Start(new(epoch, 3, 1, 4, workload), async token => + { + var start = clock.GetUtcNow(); + await Task.Delay(TimeSpan.FromMilliseconds(200), clock, token); + return new(start, clock.GetUtcNow(), start.UtcTicks, start.ToString("O")); + }, TestContext.Current.CancellationToken); + + clock.Advance(await clock.NextDelay()); + for (var index = 0; index < 3; index++) + { + var publicationDelay = await clock.NextDelay(); + Assert.Equal(TimeSpan.FromMilliseconds(200), publicationDelay); + clock.Advance(publicationDelay); + var nextDeadline = await clock.NextDelay(); + Assert.Equal(TimeSpan.FromMilliseconds(index < 2 ? 800 : 800 - phaseMilliseconds), nextDeadline); + clock.Advance(nextDeadline); + } + var result = await producer.Completion.WaitAsync(TestContext.Current.CancellationToken); + + Assert.Equal(3, result.Publications.Length); + Assert.Equal(0, result.MissedPeriods); + Assert.Equal(0, result.Overruns); + for (var index = 0; index < 3; index++) + { + var sample = result.Publications[index]; + Assert.Equal(epoch.AddSeconds(index).AddMilliseconds(phaseMilliseconds), sample.ScheduledAtUtc); + Assert.Equal(sample.ScheduledAtUtc, sample.Result.StartedAtUtc); + Assert.Equal(TimeSpan.FromMilliseconds(200), sample.Result.CompletedAtUtc - sample.Result.StartedAtUtc); + } + Assert.Equal(new OpenLoopProgress(3, true, 0, 0, null), producer.Progress); + } + + [Fact] + public void StaggeredScheduleEvenlyDistributesEveryProducerAcrossOneSecond() + { + for (var count = 3; count <= 32; count++) + { + var offsets = Enumerable.Range(0, count) + .Select(index => new OpenLoopPlan(default, 3, index, count, "OpenLoopStaggered").Offset(0)).ToArray(); + for (var index = 0; index < count; index++) + { + Assert.Equal(TimeSpan.TicksPerSecond * index / count, offsets[index].Ticks); + Assert.Equal(TimeSpan.Zero, new OpenLoopPlan(default, 3, index, count, "OpenLoopSynchronized").Offset(0)); + } + } + } + + [Theory] + [InlineData(2, 4, 0, "OpenLoopSynchronized")] + [InlineData(31, 4, 0, "OpenLoopSynchronized")] + [InlineData(3, 2, 0, "OpenLoopSynchronized")] + [InlineData(3, 33, 0, "OpenLoopSynchronized")] + [InlineData(3, 4, -1, "OpenLoopSynchronized")] + [InlineData(3, 4, 4, "OpenLoopSynchronized")] + [InlineData(3, 4, 0, "ClosedLoop")] + public void OpenLoopRejectsUnboundedOrInvalidPlans(int seconds, int count, int index, string workload) => + Assert.Throws(() => new OpenLoopPlan(default, seconds, index, count, workload).Validate()); + + [Fact] + public async Task OpenLoopReportsMissedPeriodsWithoutCatchUp() + { + var clock = new RecordingClock(); + var producer = new OpenLoopProducer(clock); + producer.Start(new(clock.GetUtcNow().AddSeconds(1), 3, 0, 3, "OpenLoopSynchronized"), + _ => throw new InvalidOperationException("A missed slot must not publish."), TestContext.Current.CancellationToken); + Assert.Equal(TimeSpan.FromSeconds(1), await clock.NextDelay()); + clock.Advance(TimeSpan.FromSeconds(2)); + var error = await Assert.ThrowsAsync(() => producer.Completion); + Assert.Contains("missed publication slot 0", error.Message, StringComparison.Ordinal); + Assert.Equal(1, producer.Progress!.MissedPeriods); + Assert.Equal(0, producer.Progress.PublishedCount); + Assert.Empty(producer.Report.Publications); + await Assert.ThrowsAsync(() => producer.DisposeAsync().AsTask()); + } + + [Fact] + public async Task OpenLoopReportsOverrunAndPreservesTheCompletedPublication() + { + var clock = new RecordingClock(); + var producer = new OpenLoopProducer(clock); + producer.Start(new(clock.GetUtcNow().AddSeconds(1), 3, 0, 3, "OpenLoopSynchronized"), async token => + { + var start = clock.GetUtcNow(); + await Task.Delay(TimeSpan.FromMilliseconds(1500), clock, token); + return new(start, clock.GetUtcNow(), start.UtcTicks, "overrun"); + }, TestContext.Current.CancellationToken); + clock.Advance(await clock.NextDelay()); + clock.Advance(await clock.NextDelay()); + var error = await Assert.ThrowsAsync(() => producer.Completion); + Assert.Contains("exceeded its 1 Hz slot", error.Message, StringComparison.Ordinal); + Assert.Equal(1, producer.Progress!.Overruns); + var sample = Assert.Single(producer.Report.Publications); + Assert.Equal(TimeSpan.FromMilliseconds(1500), sample.Result.CompletedAtUtc - sample.Result.StartedAtUtc); + await Assert.ThrowsAsync(() => producer.DisposeAsync().AsTask()); + } + + [Fact] + public async Task OpenLoopCancellationStopsArmedPublication() + { + var clock = new RecordingClock(); + var producer = new OpenLoopProducer(clock); + using var cancellation = new CancellationTokenSource(); + producer.Start(new(clock.GetUtcNow().AddSeconds(1), 3, 0, 3, "OpenLoopSynchronized"), + _ => throw new InvalidOperationException("Canceled work must not publish."), cancellation.Token); + await clock.NextDelay(); + await cancellation.CancelAsync(); + await Assert.ThrowsAnyAsync(() => producer.Completion); + Assert.Equal(0, producer.Progress!.PublishedCount); + await producer.DisposeAsync(); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(11)] + public void OpenLoopRejectsElapsedOrUnboundedStartEpoch(int leadSeconds) + { + var clock = new RecordingClock(); + var producer = new OpenLoopProducer(clock); + Assert.Throws(() => + producer.Start(new(clock.GetUtcNow().AddSeconds(leadSeconds), 3, 0, 3, "OpenLoopSynchronized"), + _ => throw new InvalidOperationException("Invalid plans must not publish."), TestContext.Current.CancellationToken)); + Assert.Null(producer.Progress); + } + + [Fact] + public async Task OpenLoopArmingIsSingleUse() + { + var clock = new RecordingClock(); + var producer = new OpenLoopProducer(clock); + var plan = new OpenLoopPlan(clock.GetUtcNow().AddSeconds(1), 3, 0, 3, "OpenLoopSynchronized"); + producer.Start(plan, _ => throw new InvalidOperationException("The producer is canceled before its start."), TestContext.Current.CancellationToken); + Assert.Throws(() => producer.Start(plan, _ => throw new InvalidOperationException(), TestContext.Current.CancellationToken)); + await producer.DisposeAsync(); + Assert.Equal(0, producer.Progress!.PublishedCount); + } + + [Fact] + public void OpenLoopReportsPollingUpperBoundsAndUnobservedIntermediateValues() + { + var reports = CreateOpenLoopReports(); + var measurements = CreateOpenLoopMeasurements(); + ObserveOpenLoop(measurements, reports, sequence: 2); + var result = measurements.Complete(reports); + + Assert.Equal(1, result.ProducerRateHz); + Assert.Equal(9, result.ObservedPublicationPairs); + Assert.Equal(18, result.UnobservedPublicationPairs); + Assert.Equal(6, result.FirstObservedAgeUpperBoundMilliseconds.Length); + Assert.Equal(new double[] { 102, 102, 102 }, result.FinalLatestLatencyUpperBoundMilliseconds); + Assert.Equal(6, result.ActualPublicationPeriodsMilliseconds.Length); + Assert.All(result.ActualPublicationPeriodsMilliseconds, value => Assert.Equal(1000, value)); + Assert.All(result.PublicationMilliseconds, value => Assert.Equal(10, value)); + Assert.Equal(0, result.MaximumStartLatenessMilliseconds); + } + + [Theory] + [InlineData("value")] + [InlineData("version")] + [InlineData("latest")] + [InlineData("process")] + [InlineData("overrun")] + public void OpenLoopRejectsMismatchedStateMissingLatestOrInvalidProducer(string mismatch) + { + var reports = CreateOpenLoopReports(); + var measurements = CreateOpenLoopMeasurements(); + ObserveOpenLoop(measurements, reports, mismatch == "latest" ? 1 : 2, mismatch); + if (mismatch == "process") + { + reports[0] = reports[0] with { ProcessId = 999 }; + } + else if (mismatch == "overrun") + { + reports[0] = reports[0] with { Report = reports[0].Report with { Overruns = 1 } }; + } + Assert.Throws(() => measurements.Complete(reports)); + } + + [Fact] + public void ScalingComparisonSeparatesOpenLoopLatencyAndRejectsMixedWorkloads() + { + var measurements = CreateOpenLoopMeasurements(); + ObserveOpenLoop(measurements, CreateOpenLoopReports(), 2); + var summary = measurements.Complete(CreateOpenLoopReports()); + var sample = new ScalingSample("CurrentDisabled", 3, 3, "stable", 0, 3, 9, + new("candidate", "same"), System.Text.Json.JsonSerializer.SerializeToElement(new { ProcessorCount = 4 }), + 90, 900, 9000, 90, 1000, [], "OpenLoopSynchronized", summary); + var enabled = sample with { RuntimePath = "CurrentEnabledSupported" }; + var pair = Assert.Single(ScalingComparison.Create(System.Text.Json.JsonSerializer.Serialize(new[] { sample, enabled })).Pairs); + Assert.Equal("OpenLoopSynchronized", pair.Workload); + Assert.Equal(102, pair.Disabled.MedianConvergenceMilliseconds); + Assert.Equal(10, pair.Disabled.RpcsPerPublication); + enabled = enabled with { Workload = "OpenLoopStaggered" }; + Assert.Throws(() => ScalingComparison.Create(System.Text.Json.JsonSerializer.Serialize(new[] { sample, enabled }))); + } + + private static OpenLoopNodeReport[] CreateOpenLoopReports() + { + var epoch = new DateTimeOffset(2030, 1, 1, 0, 0, 0, TimeSpan.Zero); + return Enumerable.Range(0, 3).Select(index => new OpenLoopNodeReport(10 + index, $"node-{index}", + new(new(epoch, 3, index, 3, "OpenLoopSynchronized"), + Enumerable.Range(0, 3).Select(sequence => + { + var start = epoch.AddSeconds(sequence); + return new OpenLoopPublication(sequence, start, new(start, start.AddMilliseconds(10), start.UtcTicks, $"{index}/{sequence}")); + }).ToArray(), 0, 0))).ToArray(); + } + + private static OpenLoopMeasurements CreateOpenLoopMeasurements() => + new(Enumerable.Range(0, 3).ToDictionary(index => 10 + index, index => $"node-{index}"), + Enumerable.Range(0, 3).ToDictionary(index => $"node-{index}", _ => 0L), 3); + + private static void ObserveOpenLoop(OpenLoopMeasurements measurements, OpenLoopNodeReport[] reports, int sequence, string? mismatch = null) + { + var values = reports.ToDictionary(report => report.Address, report => report.Report.Publications[sequence].Result); + var versions = values.ToDictionary(pair => pair.Key, pair => pair.Value.Version); + var load = values.ToDictionary(pair => pair.Key, pair => pair.Value.Value); + if (mismatch == "value") + { + load["node-0"] = "wrong"; + } + else if (mismatch == "version") + { + versions["node-0"]++; + } + for (var index = 0; index < 3; index++) + { + measurements.Observe(10 + index, values["node-0"].StartedAtUtc.AddMilliseconds(100 + index), versions, load); + } + } + + private sealed class RecordingClock : TimeProvider + { + private readonly FakeTimeProvider _inner = new(new DateTimeOffset(2030, 1, 1, 0, 0, 0, TimeSpan.Zero)); + private readonly Channel _delays = Channel.CreateUnbounded(); + public override DateTimeOffset GetUtcNow() => _inner.GetUtcNow(); + public override long GetTimestamp() => _inner.GetTimestamp(); + public override long TimestampFrequency => _inner.TimestampFrequency; + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + var timer = _inner.CreateTimer(callback, state, dueTime, period); + Assert.True(_delays.Writer.TryWrite(dueTime)); + return timer; + } + public ValueTask NextDelay() => _delays.Reader.ReadAsync(TestContext.Current.CancellationToken); + public void Advance(TimeSpan delta) => _inner.Advance(delta); + } +} diff --git a/test/Dissemination.PerformanceHarness/Tests/OpenLoopMeasurements.cs b/test/Dissemination.PerformanceHarness/Tests/OpenLoopMeasurements.cs new file mode 100644 index 00000000000..7801b571be3 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Tests/OpenLoopMeasurements.cs @@ -0,0 +1,105 @@ +namespace Orleans.Dissemination.PerformanceHarness; + +internal sealed record OpenLoopSummary( + int DurationSeconds, int ProducerRateHz, int ObservedPublicationPairs, int UnobservedPublicationPairs, + int MissedPeriods, int Overruns, double MaximumStartLatenessMilliseconds, + double[] ActualPublicationPeriodsMilliseconds, double[] PublicationMilliseconds, + double[] FirstObservedAgeUpperBoundMilliseconds, double[] FinalLatestLatencyUpperBoundMilliseconds); + +internal sealed record OpenLoopObservation(int ObserverProcessId, string Source, long Version, DateTimeOffset ObservedAtUtc, string Value); +internal sealed record OpenLoopNodeReport(int ProcessId, string Address, OpenLoopReport Report); + +internal sealed class OpenLoopMeasurements(Dictionary nodes, Dictionary initialVersions, int seconds) +{ + private readonly Dictionary<(int Observer, string Source, long Version), OpenLoopObservation> _observations = []; + + public OpenLoopObservation[] Observations => _observations.Values.ToArray(); + + public void Observe(int processId, DateTimeOffset capturedAtUtc, IReadOnlyDictionary versions, IReadOnlyDictionary load) + { + if (!nodes.ContainsKey(processId) || versions.Count != nodes.Count || load.Count != nodes.Count + || versions.Keys.Any(address => !initialVersions.ContainsKey(address))) + { + throw new InvalidOperationException("Open-loop observation has an unexpected process or load inventory."); + } + + foreach (var (address, version) in versions) + { + if (!load.TryGetValue(address, out var value)) + { + throw new InvalidOperationException($"Missing observed load value for {address}."); + } + if (version > initialVersions[address]) + { + _observations.TryAdd((processId, address, version), new(processId, address, version, capturedAtUtc, value)); + } + } + if (_observations.Count > nodes.Count * nodes.Count * seconds) + { + throw new InvalidOperationException("Observed more publications than the bounded open-loop workload offered."); + } + } + + public OpenLoopSummary Complete(OpenLoopNodeReport[] reports) + { + if (reports.Length != nodes.Count || reports.Select(report => report.ProcessId).Distinct().Count() != nodes.Count + || reports.Any(report => !nodes.TryGetValue(report.ProcessId, out var address) || address != report.Address + || report.Report.Publications.Length != seconds || report.Report.MissedPeriods != 0 || report.Report.Overruns != 0)) + { + throw new InvalidOperationException("Open-loop completion has missing producers, missed periods or overruns."); + } + + var publications = reports.SelectMany(report => report.Report.Publications + .Select(publication => (report.Address, Publication: publication))) + .ToDictionary(item => (item.Address, item.Publication.Result.Version), item => item.Publication); + var ages = new List(); + foreach (var observation in _observations.Values) + { + if (!publications.TryGetValue((observation.Source, observation.Version), out var publication) + || observation.Value != publication.Result.Value) + { + throw new InvalidOperationException($"Observed an unoffered or mismatched value from {observation.Source}."); + } + + var age = (observation.ObservedAtUtc - publication.Result.StartedAtUtc).TotalMilliseconds; + if (age < 0 || age > (seconds + 10) * 1000) + { + throw new InvalidOperationException("Open-loop observation timestamps exceed the bounded same-host latency window."); + } + if (nodes[observation.ObserverProcessId] != observation.Source) + { + ages.Add(age); + } + } + + var latestLatencies = new List(); + foreach (var report in reports) + { + var latest = report.Report.Publications[^1].Result; + var latencies = new List(); + foreach (var processId in nodes.Keys) + { + if (!_observations.TryGetValue((processId, report.Address, latest.Version), out var observation)) + { + throw new InvalidOperationException($"Process {processId} has not observed the latest value from {report.Address}."); + } + latencies.Add((observation.ObservedAtUtc - latest.StartedAtUtc).TotalMilliseconds); + } + latestLatencies.Add(latencies.Max()); + } + + var all = reports.SelectMany(report => report.Report.Publications).ToArray(); + var lateness = all.Select(value => (value.Result.StartedAtUtc - value.ScheduledAtUtc).TotalMilliseconds).ToArray(); + var durations = all.Select(value => (value.Result.CompletedAtUtc - value.Result.StartedAtUtc).TotalMilliseconds).ToArray(); + if (lateness.Any(value => value < 0 || value >= 1000) || durations.Any(value => value < 0 || value >= 1000)) + { + throw new InvalidOperationException("Actual open-loop publication timing exceeded its 1 Hz slot."); + } + + var periods = reports.SelectMany(report => report.Report.Publications.Skip(1) + .Zip(report.Report.Publications, (current, previous) => (current.Result.StartedAtUtc - previous.Result.StartedAtUtc).TotalMilliseconds)) + .ToArray(); + return new(seconds, 1, _observations.Count, nodes.Count * nodes.Count * seconds - _observations.Count, + 0, 0, lateness.Max(), periods, durations, ages.ToArray(), latestLatencies.ToArray()); + } +} diff --git a/test/Dissemination.PerformanceHarness/Tests/ProcessCluster.OpenLoop.cs b/test/Dissemination.PerformanceHarness/Tests/ProcessCluster.OpenLoop.cs new file mode 100644 index 00000000000..0748ef53b6e --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Tests/ProcessCluster.OpenLoop.cs @@ -0,0 +1,94 @@ +using Xunit; + +namespace Orleans.Dissemination.PerformanceHarness; + +internal sealed partial class ProcessCluster +{ + public async Task MeasureOpenLoop(string workload, int seconds) + { + var nodes = Active; + var owned = nodes.ToDictionary(node => node.Last.Identity.ProcessId, node => node.Last.Address); + var initialVersions = nodes.ToDictionary(node => node.Last.Address, node => node.Last.LoadVersions[node.Last.Address]); + var measurement = new OpenLoopMeasurements(owned, initialVersions, seconds); + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(5 + seconds + 10)); + var cancellationToken = deadline.Token; + var startUtc = DateTimeOffset.UtcNow.AddSeconds(5); + var plans = nodes.Select((_, index) => new OpenLoopPlan(startUtc, seconds, index, nodes.Length, workload)).ToArray(); + await Save("open-loop-plans.json", plans); + try + { + var armed = await Task.WhenAll(nodes.Select((node, index) => + node.Send(cancellationToken, "open-loop-arm", openLoop: plans[index]))); + Assert.True(DateTimeOffset.UtcNow < startUtc, "All open-loop workers must acknowledge arming before the common start epoch."); + Assert.All(armed, snapshot => + { + Assert.NotNull(snapshot.OpenLoopProgress); + Assert.Equal(0, snapshot.OpenLoopProgress.PublishedCount); + Assert.Null(snapshot.OpenLoopProgress.Error); + }); + + while (true) + { + var snapshots = await Sample(); + if (snapshots.All(snapshot => snapshot.OpenLoopProgress!.Completed)) + { + break; + } + await Task.Delay(100, cancellationToken); + } + + var completed = await Task.WhenAll(nodes.Select(node => node.Send(cancellationToken, "open-loop-finish"))); + var reports = completed.Select(snapshot => new OpenLoopNodeReport(snapshot.Identity.ProcessId, snapshot.Address, + snapshot.OpenLoopReport ?? throw new InvalidOperationException("Worker completion is missing its open-loop report."))).ToArray(); + await Save("open-loop-producers.json", reports); + var expected = reports.ToDictionary(report => report.Address, report => report.Report.Publications[^1].Result.Value); + while (true) + { + var snapshots = await Sample(); + if (snapshots.All(snapshot => MatchesExpectedLoad(snapshot.Load, expected))) + { + break; + } + await Task.Delay(100, cancellationToken); + } + + var result = measurement.Complete(reports); + await Save("open-loop-summary.json", result); + return result; + } + catch (Exception exception) + { + await Save("failure-state.json", new + { + Phase = "bounded open-loop production and exact latest-state convergence", + Error = exception.ToString(), + Plans = plans, + Nodes = nodes.Select(node => node.Last), + }); + throw; + } + finally + { + await Save("open-loop-observations.json", measurement.Observations); + } + + async Task Sample() + { + var snapshots = await Task.WhenAll(nodes.Select(node => node.Send(cancellationToken, "snapshot"))); + Assert.Equal(owned.Count, snapshots.Select(snapshot => snapshot.Identity.ProcessId).Distinct().Count()); + foreach (var snapshot in snapshots) + { + Assert.True(owned.TryGetValue(snapshot.Identity.ProcessId, out var address)); + Assert.Equal(address, snapshot.Address); + Assert.Equal(owned.Values.Order(StringComparer.Ordinal), snapshot.ActiveMembers); + Assert.NotNull(snapshot.OpenLoopProgress); + Assert.Null(snapshot.OpenLoopProgress.Error); + Assert.Equal(0, snapshot.OpenLoopProgress.MissedPeriods); + Assert.Equal(0, snapshot.OpenLoopProgress.Overruns); + measurement.Observe(snapshot.Identity.ProcessId, snapshot.CapturedAtUtc, snapshot.LoadVersions, snapshot.Load); + } + return snapshots; + } + } +} diff --git a/test/Dissemination.PerformanceHarness/Tests/ProcessCluster.cs b/test/Dissemination.PerformanceHarness/Tests/ProcessCluster.cs index 07cac076baa..2782457c54e 100644 --- a/test/Dissemination.PerformanceHarness/Tests/ProcessCluster.cs +++ b/test/Dissemination.PerformanceHarness/Tests/ProcessCluster.cs @@ -16,7 +16,7 @@ internal sealed record BuildManifest( string Sha256, Dictionary Assemblies); -internal sealed class ProcessCluster : IAsyncDisposable +internal sealed partial class ProcessCluster : IAsyncDisposable { public const string Baseline = "6739589254b746a8790cf53524e6abe372bb53d4"; private readonly List _all = []; @@ -336,12 +336,13 @@ public async Task Start() public Task Send(string operation, string? peer = null) => Send(TestContext.Current.CancellationToken, operation, peer); - public async Task Send(CancellationToken cancellationToken, string operation, string? peer = null) + public async Task Send( + CancellationToken cancellationToken, string operation, string? peer = null, OpenLoopPlan? openLoop = null) { await _commandLock.WaitAsync(cancellationToken); try { - var command = new Command(Interlocked.Increment(ref _commandId), operation, peer); + var command = new Command(Interlocked.Increment(ref _commandId), operation, peer, openLoop); var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); _pending[command.Id] = completed; await _process.StandardInput.WriteLineAsync(JsonSerializer.Serialize(command).AsMemory(), cancellationToken); diff --git a/test/Dissemination.PerformanceHarness/Tests/ScalingComparison.cs b/test/Dissemination.PerformanceHarness/Tests/ScalingComparison.cs index 3be17bf690c..16027824412 100644 --- a/test/Dissemination.PerformanceHarness/Tests/ScalingComparison.cs +++ b/test/Dissemination.PerformanceHarness/Tests/ScalingComparison.cs @@ -27,6 +27,7 @@ public static ScalingComparison Create(string json) Validate(disabled); Validate(enabled); if (disabled.Iterations != enabled.Iterations + || disabled.Workload != enabled.Workload || disabled.OfferedPublications != enabled.OfferedPublications || disabled.Runtime.SourceRevision != enabled.Runtime.SourceRevision || disabled.Runtime.Sha256 != enabled.Runtime.Sha256 @@ -37,6 +38,7 @@ public static ScalingComparison Create(string json) pairs.Add(new( disabled.Size, disabled.Scenario, disabled.Repetition, disabled.Iterations, disabled.OfferedPublications, + disabled.Workload, disabled.Runtime.SourceRevision, disabled.Runtime.Sha256, disabled.Environment, ScalingCost.From(disabled), ScalingCost.From(enabled))); } @@ -49,12 +51,44 @@ private static void Validate(ScalingSample sample) if (sample.Size < 3 || sample.LiveSilos != sample.Size || sample.OfferedPublications <= 0 || sample.Runtime is null || string.IsNullOrEmpty(sample.Runtime.SourceRevision) || string.IsNullOrEmpty(sample.Runtime.Sha256) || sample.Environment.ValueKind != JsonValueKind.Object - || sample.Iterations < 3 || sample.ConvergenceMilliseconds is null || sample.ConvergenceMilliseconds.Length != sample.Iterations + || sample.Iterations < 3 || sample.ConvergenceMilliseconds is null || sample.TotalRpcs <= 0 || sample.SerializedBytesSent <= 0 || sample.AllocatedBytes <= 0 || sample.CpuMilliseconds < 0 || sample.ConvergenceMilliseconds.Any(value => !double.IsFinite(value) || value < 0)) { throw new InvalidOperationException($"Incomplete scaling measurement for {sample.RuntimePath}, {sample.Size} silos."); } + if (sample.Workload == "ClosedLoop") + { + if (sample.ConvergenceMilliseconds.Length != sample.Iterations || sample.OpenLoop is not null) + { + throw new InvalidOperationException("Closed-loop measurements require one convergence latency per round."); + } + } + else if (sample.Workload is "OpenLoopSynchronized" or "OpenLoopStaggered" + && sample.OpenLoop is { } openLoop) + { + if (sample.Scenario != "stable" || sample.Size > 32 || sample.Iterations > 30 + || sample.OfferedPublications != sample.Size * sample.Iterations + || sample.ConvergenceMilliseconds.Length != 0 || openLoop.ProducerRateHz != 1 + || openLoop.DurationSeconds != sample.Iterations || openLoop.MissedPeriods != 0 || openLoop.Overruns != 0 + || openLoop.FinalLatestLatencyUpperBoundMilliseconds.Length != sample.Size + || openLoop.FirstObservedAgeUpperBoundMilliseconds.Length == 0 + || openLoop.PublicationMilliseconds.Length != sample.OfferedPublications + || openLoop.ActualPublicationPeriodsMilliseconds.Length != sample.Size * (sample.Iterations - 1) + || openLoop.ObservedPublicationPairs < sample.Size * sample.Size || openLoop.UnobservedPublicationPairs < 0 + || openLoop.ObservedPublicationPairs + openLoop.UnobservedPublicationPairs != sample.Size * sample.OfferedPublications + || !double.IsFinite(openLoop.MaximumStartLatenessMilliseconds) + || openLoop.MaximumStartLatenessMilliseconds is < 0 or >= 1000 + || openLoop.FinalLatestLatencyUpperBoundMilliseconds.Concat(openLoop.FirstObservedAgeUpperBoundMilliseconds) + .Any(value => !double.IsFinite(value) || value < 0 || value > (sample.Iterations + 10) * 1000)) + { + throw new InvalidOperationException("Incomplete or out-of-bounds 1 Hz open-loop measurement."); + } + } + else + { + throw new InvalidOperationException($"Missing or unsupported workload measurement: {sample.Workload}."); + } } public string ToMarkdown() @@ -63,13 +97,14 @@ public string ToMarkdown() result.AppendLine("### Dissemination disabled vs enabled"); result.AppendLine("Enabled mode explicitly opts into the subsystem and both namespaces, retaining the candidate's production dissemination tuning defaults."); result.AppendLine("Each cell is disabled -> enabled. Costs per offered publication include background and control traffic. Single-host, resource-shared measurements; raw totals, environment and per-node data are in the artifacts."); + result.AppendLine("ClosedLoop latency is per-round convergence. OpenLoop latency is the polling upper bound for each producer's final value reaching all processes; cadence, observed intermediate values and their age upper bounds are in the raw open-loop artifacts."); result.AppendLine(); - result.AppendLine("| Silos / scenario / repetition | RPCs / publication | KiB / publication | Allocated KiB / publication | CPU ms / publication | Median ms | P95 ms |"); + result.AppendLine("| Silos / scenario / workload / repetition | RPCs / publication | KiB / publication | Allocated KiB / publication | CPU ms / publication | Median ms | P95 ms |"); result.AppendLine("|---|---:|---:|---:|---:|---:|---:|"); foreach (var pair in Pairs) { result.AppendLine(FormattableString.Invariant( - $"| {pair.Size} / {pair.Scenario} / {pair.Repetition} | {pair.Disabled.RpcsPerPublication:F2} -> {pair.Enabled.RpcsPerPublication:F2} | {pair.Disabled.SerializedBytesPerPublication / 1024:F2} -> {pair.Enabled.SerializedBytesPerPublication / 1024:F2} | {pair.Disabled.AllocatedBytesPerPublication / 1024:F2} -> {pair.Enabled.AllocatedBytesPerPublication / 1024:F2} | {pair.Disabled.CpuMillisecondsPerPublication:F2} -> {pair.Enabled.CpuMillisecondsPerPublication:F2} | {pair.Disabled.MedianConvergenceMilliseconds:F2} -> {pair.Enabled.MedianConvergenceMilliseconds:F2} | {pair.Disabled.P95ConvergenceMilliseconds:F2} -> {pair.Enabled.P95ConvergenceMilliseconds:F2} |")); + $"| {pair.Size} / {pair.Scenario} / {pair.Workload} / {pair.Repetition} | {pair.Disabled.RpcsPerPublication:F2} -> {pair.Enabled.RpcsPerPublication:F2} | {pair.Disabled.SerializedBytesPerPublication / 1024:F2} -> {pair.Enabled.SerializedBytesPerPublication / 1024:F2} | {pair.Disabled.AllocatedBytesPerPublication / 1024:F2} -> {pair.Enabled.AllocatedBytesPerPublication / 1024:F2} | {pair.Disabled.CpuMillisecondsPerPublication:F2} -> {pair.Enabled.CpuMillisecondsPerPublication:F2} | {pair.Disabled.MedianConvergenceMilliseconds:F2} -> {pair.Enabled.MedianConvergenceMilliseconds:F2} | {pair.Disabled.P95ConvergenceMilliseconds:F2} -> {pair.Enabled.P95ConvergenceMilliseconds:F2} |")); } result.AppendLine(); @@ -81,12 +116,14 @@ public string ToMarkdown() internal sealed record ScalingSample( string RuntimePath, int Size, int LiveSilos, string Scenario, int Repetition, int Iterations, int OfferedPublications, ScalingBinary Runtime, JsonElement Environment, long TotalRpcs, double SerializedBytesSent, - long AllocatedBytes, double CpuMilliseconds, long RetainedManagedBytes, double[] ConvergenceMilliseconds); + long AllocatedBytes, double CpuMilliseconds, long RetainedManagedBytes, double[] ConvergenceMilliseconds, + string Workload = "ClosedLoop", OpenLoopSummary? OpenLoop = null); internal sealed record ScalingBinary(string SourceRevision, string Sha256); internal sealed record ScalingPair( int Size, string Scenario, int Repetition, int Iterations, int OfferedPublications, + string Workload, string SourceRevision, string Sha256, JsonElement Environment, ScalingCost Disabled, ScalingCost Enabled); internal sealed record ScalingCost( @@ -96,7 +133,8 @@ internal sealed record ScalingCost( { public static ScalingCost From(ScalingSample sample) { - var ordered = sample.ConvergenceMilliseconds.Order().ToArray(); + var ordered = (sample.Workload == "ClosedLoop" ? sample.ConvergenceMilliseconds + : sample.OpenLoop!.FinalLatestLatencyUpperBoundMilliseconds).Order().ToArray(); var median = (ordered[(ordered.Length - 1) / 2] + ordered[ordered.Length / 2]) / 2; return new( sample.TotalRpcs, sample.SerializedBytesSent, sample.AllocatedBytes, sample.CpuMilliseconds, sample.RetainedManagedBytes, diff --git a/test/Dissemination.PerformanceHarness/Tests/ScalingTests.cs b/test/Dissemination.PerformanceHarness/Tests/ScalingTests.cs index d8ddf98d171..f653401dae3 100644 --- a/test/Dissemination.PerformanceHarness/Tests/ScalingTests.cs +++ b/test/Dissemination.PerformanceHarness/Tests/ScalingTests.cs @@ -25,6 +25,14 @@ public async Task Scaling_SelectedRuntimePathsConverge() Assert.All(scenarios, scenario => Assert.Contains(scenario, new[] { "stable", "churn", "partition" })); var paths = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_RUNTIME_PATHS") ?? "All"; Assert.Contains(paths, new[] { "All", "Current" }); + var workload = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_WORKLOAD") ?? "ClosedLoop"; + Assert.Contains(workload, new[] { "ClosedLoop", "OpenLoopSynchronized", "OpenLoopStaggered" }); + if (workload != "ClosedLoop") + { + Assert.Equal(new[] { "stable" }, scenarios); + Assert.InRange(iterations, 3, 30); + Assert.All(sizes, size => Assert.InRange(size, 3, 32)); + } var records = new List(); var output = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_RESULTS")!; @@ -49,7 +57,7 @@ public async Task Scaling_SelectedRuntimePathsConverge() for (var order = 0; order < variants.Length; order++) { var variant = variants[ExecutionIndex(repetition, order, variants.Length)]; - var record = await Measure(size, scenario, variant, iterations, repetition, order); + var record = await Measure(size, scenario, variant, iterations, repetition, order, workload); records.Add(record); var json = JsonSerializer.Serialize(records, new JsonSerializerOptions { WriteIndented = true }); await File.WriteAllTextAsync(Path.Combine(output, "cost-results.json"), json, TestContext.Current.CancellationToken); @@ -70,14 +78,15 @@ await File.WriteAllTextAsync(Path.Combine(output, "comparisons.json"), } } - private static async Task Measure(int size, string scenario, RuntimeVariant variant, int iterations, int repetition, int order) + private static async Task Measure( + int size, string scenario, RuntimeVariant variant, int iterations, int repetition, int order, string workload) { var enabled = variant.Enabled; var siloProcessorCount = Setting("SILO_PROCESSOR_COUNT", 0, 0, 64); var gcConserveMemory = Setting("GC_CONSERVE_MEMORY", 0, 0, 9); var bootstrap = Stopwatch.StartNew(); await using var cluster = new ProcessCluster( - $"scale-{size}-{scenario}-{variant.Name}-{repetition}", siloProcessorCount, gcConserveMemory); + $"scale-{size}-{scenario}-{workload}-{variant.Name}-{repetition}", siloProcessorCount, gcConserveMemory); for (var index = 0; index < size; index++) { await cluster.Start(variant.Runtime, enabled); @@ -109,7 +118,13 @@ private static async Task Measure(int size, string scenario, RuntimeVari var offeredPublications = 0; var restarts = 0; var partitions = 0; - for (var iteration = 0; iteration < iterations; iteration++) + OpenLoopSummary? openLoop = null; + if (workload != "ClosedLoop") + { + openLoop = await cluster.MeasureOpenLoop(workload, iterations); + offeredPublications = size * iterations; + } + for (var iteration = 0; workload == "ClosedLoop" && iteration < iterations; iteration++) { if (scenario == "churn" && iteration > 0 && iteration % 2 == 0) { @@ -189,6 +204,8 @@ async Task PublishRound() index >= 0 && (topology.AggregationTree ? index <= topology.Fanout : index < topology.Fanout), topology.OriginatorTargets, topology.ForwardingTargets, + topology.Fanout, + topology.FanoutSource, requestMessages.Count + oneWayMessages.Count, allMessages.Count, allMessages.Sum, @@ -232,12 +249,14 @@ async Task PublishRound() var record = new { - Schema = 1, + Schema = 2, RuntimePath = variant.Name, Runtime = finish[0].Identity, Size = size, LiveSilos = start.Count, Scenario = scenario, + Workload = workload, + OpenLoop = openLoop, Enabled = enabled, Iterations = iterations, Repetition = repetition, @@ -256,7 +275,7 @@ async Task PublishRound() TotalMessages = nodes.Sum(node => node.SentMessages), LoadBroadcastRequests = nodes.Sum(node => node.LoadBroadcastRequests), LoadValueTransmissions = nodes.Sum(node => node.LoadValueTransmissions), - LoadBroadcastRequestsPerRound = nodes.Sum(node => node.LoadBroadcastRequests) / iterations, + LoadBroadcastRequestsPerRound = openLoop is null ? nodes.Sum(node => node.LoadBroadcastRequests) / iterations : (double?)null, LoadBroadcastRequestsPerSecond = nodes.Sum(node => node.LoadBroadcastRequests) / (elapsed / 1000), SerializedBytesSent = nodes.Sum(node => node.SerializedBytesSent), SocketBytesSent = nodes.Sum(node => node.SocketBytesSent), @@ -283,6 +302,9 @@ async Task PublishRound() WarmupRounds = 2, SettlingMarginSeconds = 2, AutomaticLoadPublisherPaused = true, + ProducerCadence = workload == "ClosedLoop" ? "controller publication/convergence rounds" : "worker-local 1 Hz monotonic schedule", + OpenLoopTiming = "5-second common-start arming budget; 3..30-second stable publication window; 10-second completion/latest-state drain; missed periods and overruns fail without catch-up", + Latency = workload == "ClosedLoop" ? "per-round end-to-end exact convergence" : "first-observed ages and per-producer final-latest convergence are polling upper bounds using same-host UTC timestamps", ProductionDisseminationCadence = variant.Runtime == "New" && enabled, ExplicitDisseminationOptOut = variant.Runtime == "New" && !enabled, LoadNamespaceSupportConfirmedBeforeOfferedRounds = enabled, @@ -369,6 +391,8 @@ private sealed record NodeCost( bool UpperTree, string[] OriginatorTargets, string[] ForwardingTargets, + int Fanout, + string? FanoutSource, long Rpcs, long SentMessages, double SerializedBytesSent, diff --git a/test/Dissemination.PerformanceHarness/methodology.json b/test/Dissemination.PerformanceHarness/methodology.json index 235e0739d05..620b983a0e9 100644 --- a/test/Dissemination.PerformanceHarness/methodology.json +++ b/test/Dissemination.PerformanceHarness/methodology.json @@ -16,9 +16,10 @@ "comparison": "Candidate off/on pairs use one published binary directory and identical process counts, workload and environment. Original-to-candidate deltas include every intervening runtime change." }, "controls": { - "defaults": { "sizes": [4, 8], "iterations": 3, "repetitions": 2, "runtimePaths": "All", "scenarios": ["stable", "churn", "partition"] }, + "defaults": { "sizes": [4, 8], "iterations": 3, "repetitions": 2, "runtimePaths": "All", "scenarios": ["stable", "churn", "partition"], "workload": "ClosedLoop" }, "sizes": "One to six distinct counts, each 3..128", "iterations": "3..200", + "openLoop": "Workload OpenLoopSynchronized or OpenLoopStaggered requires stable only, 3..32 processes, and Iterations 3..30 as duration seconds at 1 Hz per producer.", "repetitions": "1..5; even repetitions run original/off/on, odd repetitions run on/off/original; Current selects off/on", "scenarios": "Distinct stable, churn and partition selections", "siloProcessorCount": "0..64; 0 inherits the runner processor count", @@ -33,6 +34,9 @@ "warmup": "Two production publication rounds and complete enabled-peer support confirmation precede the cost window. Preparation time, CPU and allocation totals are reported separately.", "publication": "Dispose the automatic DeploymentLoadPublisher timer on its scheduler while retaining the field reference for original-runtime lifecycle shutdown. Invoke the production PublishStatistics method once per active node, then require exact load dictionary inventory and values using IncludeFields=true.", "stable": "Repeat equal closed-loop publication/convergence rounds.", + "openLoopStable": "Arm each worker five seconds before a shared UTC epoch. Worker-local monotonic deadlines offer one production publication per second independently of controller polling. Synchronized producers share the boundary; staggered producers use index/count-second phases. The complete workload is bounded to 30 seconds and 32 processes; missed periods and publication overruns fail while preserving their counts and partial timestamps. A ten-second post-window deadline covers completion and exact latest-state convergence. Observed values must match actually published versions and fields; intermediate unobserved pairs are counted separately.", + "openLoopLatency": "First-observed peer ages and per-producer final-latest convergence are 100 ms polling upper bounds derived from same-host UTC timestamps. Planned/actual publication timestamps, durations and periods remain in raw artifacts. Comparisons keep work pattern/duration/rate/process count equal and distinguish these upper bounds from per-round closed-loop latency.", + "fanoutCapability": "For aggregation routing, read the optional public int Overlay.AggregationFanOutFactor and apply the runtime clamp 1..max(1,members). Absent capability selects the legacy 3fc membership fanout formula and labels that source explicitly; an incompatible property shape fails with a runtime-adapter diagnostic. Membership routing retains its selector.", "churn": "Gracefully stop and restart one process at its original endpoint every other round. Confirm membership and supported enabled peers before the next offered round. Replacement startup, shutdown, discovery and fallback costs remain in the measured window.", "partition": "Abort real inbound/outbound silo connections and reject new admissions at one endpoint. Publish once on each remaining node. Recovery drains middleware and close tasks at both endpoints, including unidentified handshakes, then confirms bidirectional acknowledged control RPCs before a single publication per active node. Teardown/readiness/retries remain inside recovery time and total cost, preserving the #11243 barrier.", "window": "Identical two-second socket-sampling margins; costs include background runtime, control RPCs and instrumentation. The post-window full-GC retained-memory probe is separate from CPU/allocation costs.", @@ -49,7 +53,7 @@ "reports": "cost-results.json keeps raw samples. comparisons.json and the job summary pair candidate off/on by size/scenario/repetition, checking equal live process count, binary/revision, environment, iterations and offered publications. Costs normalize per offered publication; median/P95 convergence and incomplete pairs remain explicit." }, "interpretation": { - "environment": "Loopback TCP processes share one runner's CPU, memory and local storage. Results describe a closed-loop control-plane workload with a convergence barrier after each round.", + "environment": "Loopback TCP processes share one runner's CPU, memory and local storage. ClosedLoop uses a convergence barrier after each round; optional OpenLoop fixes each producer at 1 Hz with independent state polling.", "timing": "100 ms state polling and one-second socket sampling introduce boundary uncertainty; raw UTC snapshots and separate publication/wait durations preserve that context.", "socketBytes": "Socket counters measure application bytes; transport counters include submitted buffered bytes, including bytes abandoned during connection abort.", "memory": "Retained memory is a post-GC live-size sample for surviving processes. Artifacts also retain retired-process snapshots and per-process RSS/private-memory samples.", @@ -72,7 +76,7 @@ }, "artifacts": { "root": "Artifacts/DisseminationPerformance", - "results": "results/-: selection, invocation, methodology, host resources, TRX, comparisons, raw costs and per-cluster process snapshots/stdout/stderr", + "results": "results/-: selection, invocation, methodology, host resources, TRX, comparisons, raw costs and per-cluster process snapshots/stdout/stderr. Open-loop runs also preserve plans, producer publication reports, first observations and summary JSON.", "binaries": "bin/{Old,New,Runner} and bin/selection.json for explicit -SkipBuild replay", "paths": "Artifact roots are single named directories under Artifacts; existing links/reparse points are rejected. Every invocation writes a fresh results directory.", "retentionDays": { "results": 30, "binaries": 7 } diff --git a/test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj b/test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj index 2a8e7d91efc..d3ae84f89ad 100644 --- a/test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj +++ b/test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj @@ -12,6 +12,7 @@ + From 80df772632014f0fbfa33ffac5da7e8a6c47d023 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 17 Sep 2026 02:17:38 -0700 Subject: [PATCH 11/12] fix: inspect dangling path entries directly --- .github/scripts/dissemination-performance.ps1 | 14 +++++++----- .../test-dissemination-performance.ps1 | 22 +++++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/.github/scripts/dissemination-performance.ps1 b/.github/scripts/dissemination-performance.ps1 index 579fd593fb8..8d9c1ced473 100644 --- a/.github/scripts/dissemination-performance.ps1 +++ b/.github/scripts/dissemination-performance.ps1 @@ -54,11 +54,15 @@ function Assert-NoLinks([string] $Path) { } $comparison = if ($IsWindows) { [StringComparison]::OrdinalIgnoreCase } else { [StringComparison]::Ordinal } while (![string]::Equals($current, $root, $comparison)) { - if (Test-Path -LiteralPath $current) { - $item = Get-Item -LiteralPath $current -Force - if ($item.Attributes -band [System.IO.FileAttributes]::ReparsePoint) { - throw "Artifact and runtime paths must use ordinary directories: $current" - } + $item = $null + try { + $item = Get-Item -LiteralPath $current -Force -ErrorAction Stop + } + catch [System.Management.Automation.ItemNotFoundException] { + # Missing descendants are created only after validating their existing parents. + } + if ($null -ne $item -and $item.Attributes -band [System.IO.FileAttributes]::ReparsePoint) { + throw "Artifact and runtime paths must use ordinary directories: $current" } $parent = Split-Path -Parent $current if (!$parent -or [string]::Equals($parent, $current, $comparison)) { diff --git a/.github/scripts/test-dissemination-performance.ps1 b/.github/scripts/test-dissemination-performance.ps1 index 97e284f3504..9d036c4d145 100644 --- a/.github/scripts/test-dissemination-performance.ps1 +++ b/.github/scripts/test-dissemination-performance.ps1 @@ -107,6 +107,28 @@ try { } Write-Output 'Passed 5 repository-boundary checks.' + & { + $inspected = [System.Collections.Generic.List[string]]::new() + $denied = Join-Path $testArtifacts 'unreadable' + function Get-Item { + [CmdletBinding()] + param([string] $LiteralPath, [switch] $Force) + $inspected.Add($LiteralPath) + throw [System.UnauthorizedAccessException]::new('Boundary inspection denied.') + } + $rejected = $false + try { + Assert-NoLinks $denied + } + catch [System.UnauthorizedAccessException] { + $rejected = $true + } + if (!$rejected -or $inspected.Count -ne 1 -or $inspected[0] -ne $denied) { + throw 'Boundary inspection must propagate access errors at the requested path.' + } + } + Write-Output 'Passed boundary access-error propagation check.' + $harnessSource = Join-Path $root 'test' 'Dissemination.PerformanceHarness' $checkout = Join-Path $testArtifacts 'runtime-checkout' $existingHarness = Join-Path $checkout 'test' 'Dissemination.IntegrationHarness' From fb0762e0787f60e7958877305cf80ed17316ab67 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 17 Sep 2026 02:42:16 -0700 Subject: [PATCH 12/12] fix: preserve duplicate membership suspect votes --- .../ProviderTests/FileMembershipTableTests.cs | 61 +++++++++++++++++++ .../README.md | 5 +- .../Silo/FileMembershipTable.cs | 8 ++- 3 files changed, 69 insertions(+), 5 deletions(-) diff --git a/test/Dissemination.PerformanceHarness/ProviderTests/FileMembershipTableTests.cs b/test/Dissemination.PerformanceHarness/ProviderTests/FileMembershipTableTests.cs index 4365ce7ab21..4876c66bb34 100644 --- a/test/Dissemination.PerformanceHarness/ProviderTests/FileMembershipTableTests.cs +++ b/test/Dissemination.PerformanceHarness/ProviderTests/FileMembershipTableTests.cs @@ -13,6 +13,67 @@ public sealed class FileMembershipTableTests : IDisposable private static readonly DateTime Cutoff = new(2026, 1, 15, 12, 0, 0, DateTimeKind.Utc); private readonly string _directory = Path.Combine(Path.GetTempPath(), $"orleans-dissemination-cleanup-{Guid.NewGuid():N}"); + [Fact] + public async Task PersistedSuspectVotesPreserveDuplicatesOrderAndLatestUpdate() + { + var table = CreateTable(); + var cancellationToken = TestContext.Current.CancellationToken; + await table.InitializeMembershipTableAsync(true, cancellationToken); + var entry = CreateEntry(SiloStatus.Dead, 1); + var repeatedVoter = CreateEntry(SiloStatus.Active, 2).SiloAddress; + entry.AddSuspector(repeatedVoter, Cutoff.AddSeconds(-1)); + entry.AddSuspector(repeatedVoter, Cutoff); + entry.AddSuspector(CreateEntry(SiloStatus.Active, 3).SiloAddress, Cutoff.AddSeconds(-2)); + var initial = await table.ReadAllAsync(cancellationToken); + Assert.True(await table.InsertRowAsync(entry, initial.Version.Next(), cancellationToken)); + + var reopened = CreateTable(); + var inserted = await reopened.ReadAllAsync(cancellationToken); + var insertedRow = Assert.Single(inserted.Members); + Assert.Equal(entry.SuspectTimes, insertedRow.Item1.SuspectTimes); + Assert.Equal(Cutoff, insertedRow.Item1.EffectiveUpdateTime); + + entry.AddSuspector(repeatedVoter, Cutoff.AddSeconds(1)); + Assert.True(await reopened.UpdateRowAsync(entry, insertedRow.Item2, inserted.Version.Next(), cancellationToken)); + var updatedTable = CreateTable(); + var updated = await updatedTable.ReadAllAsync(cancellationToken); + var updatedRow = Assert.Single(updated.Members); + Assert.Equal(entry.SuspectTimes, updatedRow.Item1.SuspectTimes); + Assert.Equal(Cutoff.AddSeconds(1), updatedRow.Item1.EffectiveUpdateTime); + + await updatedTable.CleanupDefunctSiloEntriesAsync(Cutoff.AddSeconds(1), cancellationToken); + var retained = Assert.Single((await updatedTable.ReadAllAsync(cancellationToken)).Members); + Assert.Equal(entry.SuspectTimes, retained.Item1.SuspectTimes); + await updatedTable.CleanupDefunctSiloEntriesAsync(Cutoff.AddSeconds(1).AddTicks(1), cancellationToken); + Assert.Empty((await updatedTable.ReadAllAsync(cancellationToken)).Members); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task PersistedSuspectVotesPreserveNullAndEmptyLists(bool absent) + { + var table = CreateTable(); + var cancellationToken = TestContext.Current.CancellationToken; + await table.InitializeMembershipTableAsync(true, cancellationToken); + var entry = CreateEntry(SiloStatus.Active, 1); + entry.SuspectTimes = absent ? null : []; + var initial = await table.ReadAllAsync(cancellationToken); + Assert.True(await table.InsertRowAsync(entry, initial.Version.Next(), cancellationToken)); + + var reopened = CreateTable(); + var persisted = Assert.Single((await reopened.ReadAllAsync(cancellationToken)).Members).Item1.SuspectTimes; + if (absent) + { + Assert.Null(persisted); + } + else + { + Assert.NotNull(persisted); + Assert.Empty(persisted); + } + } + [Fact] public async Task DeleteInvalidatesThePreviousTableEtag() { diff --git a/test/Dissemination.PerformanceHarness/README.md b/test/Dissemination.PerformanceHarness/README.md index 960802e07c8..1a52023d3ec 100644 --- a/test/Dissemination.PerformanceHarness/README.md +++ b/test/Dissemination.PerformanceHarness/README.md @@ -180,7 +180,8 @@ alongside the file-backed provider and its protocol. These BVT cases exercise persisted membership cleanup: all expired non-active statuses are removed, while active rows and rows updated at or after the cutoff retain their ETags. Cleanup uses the selected runtime's `EffectiveUpdateTime`, including startup, heartbeat and -suspect-vote timestamps. +suspect-vote timestamps. Persisted vote lists retain their order, repeated voters, +and null-versus-empty state across inserts and updates. Clearing membership advances the table ETag so writes from an earlier snapshot are rejected. Heartbeat dirty writes preserve the table version and update only the liveness timestamp. @@ -188,7 +189,7 @@ the liveness timestamp. ```powershell dotnet test --project test\Orleans.Runtime.Tests\Orleans.Runtime.Tests.csproj ` --framework net10.0 --filter-class 'Orleans.Dissemination.PerformanceHarness.FileMembershipTableTests' ` - --minimum-expected-tests 12 + --minimum-expected-tests 15 ``` The scaling test runs through the script's explicit manual-run marker and published diff --git a/test/Dissemination.PerformanceHarness/Silo/FileMembershipTable.cs b/test/Dissemination.PerformanceHarness/Silo/FileMembershipTable.cs index cfa1b5dba5f..84b63c6ba06 100644 --- a/test/Dissemination.PerformanceHarness/Silo/FileMembershipTable.cs +++ b/test/Dissemination.PerformanceHarness/Silo/FileMembershipTable.cs @@ -191,6 +191,8 @@ private sealed class TableState private sealed record Row(EntryData Entry, string Etag); + internal sealed record SuspectVote(string Address, DateTime Time); + internal sealed record EntryData( string Address, int Status, @@ -202,13 +204,13 @@ internal sealed record EntryData( int FaultZone, DateTime StartTime, DateTime IAmAliveTime, - Dictionary Suspects) + List? Suspects) { public static EntryData From(MembershipEntry entry) => new( entry.SiloAddress.ToParsableString(), (int)entry.Status, entry.ProxyPort, entry.HostName, entry.SiloName, entry.RoleName, entry.UpdateZone, entry.FaultZone, entry.StartTime, entry.IAmAliveTime, - entry.SuspectTimes?.ToDictionary(pair => pair.Item1.ToParsableString(), pair => pair.Item2) ?? []); + entry.SuspectTimes?.Select(pair => new SuspectVote(pair.Item1.ToParsableString(), pair.Item2)).ToList()); public MembershipEntry ToEntry() => new() { @@ -222,7 +224,7 @@ internal sealed record EntryData( FaultZone = FaultZone, StartTime = StartTime, IAmAliveTime = IAmAliveTime, - SuspectTimes = Suspects.Select(pair => Tuple.Create(SiloAddress.FromParsableString(pair.Key), pair.Value)).ToList(), + SuspectTimes = Suspects?.Select(vote => Tuple.Create(SiloAddress.FromParsableString(vote.Address), vote.Time)).ToList(), }; } }