diff --git a/.github/scripts/dissemination-performance.ps1 b/.github/scripts/dissemination-performance.ps1 new file mode 100644 index 00000000000..8d9c1ced473 --- /dev/null +++ b/.github/scripts/dissemination-performance.ps1 @@ -0,0 +1,307 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory)] + [ValidatePattern('^[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9][A-Za-z0-9._-]*$')] + [string] $CandidateRepository, + [Parameter(Mandatory)] + [ValidatePattern('^[A-Za-z0-9][A-Za-z0-9._/-]{0,199}$')] + [string] $CandidateRef, + [ValidatePattern('^Artifacts[\\/][A-Za-z0-9][A-Za-z0-9._-]{0,79}$')] + [string] $ArtifactsDirectory = (Join-Path 'Artifacts' 'DisseminationPerformance'), + [ValidatePattern('^\d+(,\d+){0,5}$')] + [string] $Sizes = '4,8', + [ValidateRange(3, 200)] + [int] $Iterations = 3, + [ValidateRange(1, 5)] + [int] $Repetitions = 2, + [ValidateSet('All', 'Current')] + [string] $RuntimePaths = 'All', + [ValidatePattern('^(stable|churn|partition)(,(stable|churn|partition)){0,2}$')] + [string] $Scenarios = 'stable,churn,partition', + [ValidateSet('ClosedLoop', 'OpenLoopSynchronized', 'OpenLoopStaggered')] + [string] $Workload = 'ClosedLoop', + [ValidateRange(0, 64)] + [int] $SiloProcessorCount = 0, + [ValidateRange(0, 9)] + [int] $GCConserveMemory = 0, + [switch] $SkipBuild +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +$baselineSha = '6739589254b746a8790cf53524e6abe372bb53d4' +$root = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..' '..')) +$harnessSource = Join-Path $root 'test' 'Dissemination.PerformanceHarness' +$sizesArray = @($Sizes.Split(',') | ForEach-Object { [int]$_ }) +if (@($sizesArray | Select-Object -Unique).Count -ne $sizesArray.Count -or + @($sizesArray | Where-Object { $_ -lt 3 -or $_ -gt 128 }).Count -gt 0) { + throw 'Select one to six distinct silo counts between 3 and 128.' +} +if (@($Scenarios.Split(',') | Select-Object -Unique).Count -ne $Scenarios.Split(',').Count) { + throw 'Select each scenario at most once.' +} +if ($Workload -ne 'ClosedLoop' -and ($Scenarios -ne 'stable' -or $Iterations -gt 30 -or + @($sizesArray | Where-Object { $_ -gt 32 }).Count -gt 0)) { + throw 'Open-loop workloads require stable only, 3..32 silo processes and 3..30 seconds (Iterations).' +} + +function Assert-NoLinks([string] $Path) { + $current = [System.IO.Path]::GetFullPath($Path) + $relative = [System.IO.Path]::GetRelativePath($root, $current) + if ([System.IO.Path]::IsPathRooted($relative) -or $relative -eq '..' -or + $relative.StartsWith("..$([System.IO.Path]::DirectorySeparatorChar)", [StringComparison]::Ordinal)) { + throw "Path is outside the repository boundary: $Path" + } + $comparison = if ($IsWindows) { [StringComparison]::OrdinalIgnoreCase } else { [StringComparison]::Ordinal } + while (![string]::Equals($current, $root, $comparison)) { + $item = $null + try { + $item = Get-Item -LiteralPath $current -Force -ErrorAction Stop + } + catch [System.Management.Automation.ItemNotFoundException] { + # Missing descendants are created only after validating their existing parents. + } + if ($null -ne $item -and $item.Attributes -band [System.IO.FileAttributes]::ReparsePoint) { + throw "Artifact and runtime paths must use ordinary directories: $current" + } + $parent = Split-Path -Parent $current + if (!$parent -or [string]::Equals($parent, $current, $comparison)) { + throw "Path is outside the repository boundary: $Path" + } + $current = $parent + } +} + +function Invoke-DotNet([string[]] $Arguments) { + & dotnet @Arguments + if ($LASTEXITCODE -ne 0) { + throw "dotnet failed ($LASTEXITCODE): $($Arguments -join ' ')" + } +} + +function Get-Revision([string] $Directory) { + $revision = & git -C $Directory rev-parse HEAD + if ($LASTEXITCODE -ne 0) { + throw "Not a source checkout: $Directory" + } + return $revision.Trim() +} + +function Assert-Checkout([string] $Directory, [string] $Repository) { + Assert-NoLinks $Directory + $topLevel = & git -C $Directory rev-parse --show-toplevel + if ($LASTEXITCODE -ne 0 -or [System.IO.Path]::GetFullPath($topLevel.Trim()) -ne $Directory) { + throw "Runtime must have its own isolated checkout: $Directory" + } + $url = & git -C $Directory remote get-url origin + if ($LASTEXITCODE -ne 0 -or $url.Trim().TrimEnd('/') -notin @("https://github.com/$Repository", "https://github.com/$Repository.git")) { + throw "Runtime checkout must originate from https://github.com/$Repository." + } + $changes = & git -C $Directory status --porcelain --untracked-files=all + if ($LASTEXITCODE -ne 0 -or $changes) { + throw "Selected runtime checkout must be pristine before copying worker support: $Directory" + } +} + +function Copy-Worker([string] $Checkout) { + $destination = Join-Path $Checkout 'test' 'Dissemination.PerformanceHarness' + Assert-NoLinks $destination + if (Test-Path -LiteralPath $destination) { + throw "Refusing to overwrite an existing runtime harness: $destination" + } + foreach ($part in @('Silo', 'Shared')) { + $partDestination = Join-Path $destination $part + New-Item -ItemType Directory -Path $partDestination -Force | Out-Null + Get-ChildItem -LiteralPath (Join-Path $harnessSource $part) -File | + Where-Object { $_.Extension -in '.cs', '.csproj' } | + Copy-Item -Destination $partDestination + } +} + +function Publish-Silo([string] $Checkout, [string] $Runtime, [string] $Revision) { + $project = Join-Path $Checkout 'test' 'Dissemination.PerformanceHarness' 'Silo' 'Dissemination.Silo.csproj' + $output = Join-Path $binaries $Runtime + Push-Location $Checkout + try { + Invoke-DotNet @( + 'publish', $project, '--configuration', 'Release', '--framework', 'net10.0', + '--output', $output, '--verbosity', 'minimal', + "-p:HarnessRuntime=$Runtime", "-p:SourceRevisionId=$Revision", + '-p:OfficialBuild=true', "-p:BUILD_SOURCEVERSION=$Revision", + "-bl:$(Join-Path $artifacts "build-$Runtime.binlog")" + ) + } + finally { + Pop-Location + } + $assemblyPath = Join-Path $output 'Orleans.Runtime.dll' + $identity = [System.Reflection.AssemblyName]::GetAssemblyName($assemblyPath) + $assemblies = @{} + foreach ($file in Get-ChildItem -LiteralPath $output -Filter 'Orleans*.dll' -File) { + $name = [System.Reflection.AssemblyName]::GetAssemblyName($file.FullName) + $assemblies[$name.Name] = @{ + AssemblyVersion = $name.Version.ToString() + Sha256 = (Get-FileHash -LiteralPath $file.FullName -Algorithm SHA256).Hash + } + } + @{ + Runtime = $Runtime + SourceRevision = $Revision + AssemblyName = $identity.Name + AssemblyVersion = $identity.Version.ToString() + Sha256 = (Get-FileHash -LiteralPath $assemblyPath -Algorithm SHA256).Hash + Assemblies = $assemblies + } | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath (Join-Path $output 'manifest.json') +} + +$artifactName = ($ArtifactsDirectory -split '[\\/]')[1] +$artifacts = [System.IO.Path]::GetFullPath((Join-Path $root 'Artifacts' $artifactName)) +Assert-NoLinks $artifacts +if (Test-Path -LiteralPath $artifacts) { + foreach ($item in Get-ChildItem -LiteralPath $artifacts -Recurse -Force) { + if ($item.Attributes -band [System.IO.FileAttributes]::ReparsePoint) { + throw "Artifact contents must use ordinary files and directories: $($item.FullName)" + } + } +} +$binaries = Join-Path $artifacts 'bin' +if (!$SkipBuild) { + if ($env:GITHUB_ACTIONS -ne 'true') { + throw 'Runtime builds run in CI. Download the binary artifact and use -SkipBuild for local reproduction.' + } + if (Test-Path -LiteralPath $binaries) { + throw "Use a fresh artifact directory for a new build: $binaries" + } + $baseline = Join-Path $root '.dissemination-runtime-original' + $candidate = Join-Path $root '.dissemination-runtime-candidate' + Assert-Checkout $baseline 'dotnet/orleans' + Assert-Checkout $candidate $CandidateRepository + if ((Get-Revision $baseline) -ne $baselineSha) { + throw "Original runtime must be pinned to $baselineSha." + } + $candidateSha = Get-Revision $candidate + if ($CandidateRef -match '^[0-9a-fA-F]{40}$' -and $candidateSha -ne $CandidateRef) { + throw "Candidate checkout does not match requested commit $CandidateRef." + } + if (!(Test-Path -LiteralPath (Join-Path $candidate 'src' 'Orleans.Runtime' 'Dissemination'))) { + throw 'Select a candidate revision containing the dissemination runtime APIs.' + } + if (Test-Path -LiteralPath (Join-Path $baseline 'src' 'Orleans.Runtime' 'Dissemination')) { + throw 'Original runtime unexpectedly contains dissemination.' + } + New-Item -ItemType Directory -Force $binaries | Out-Null + Copy-Worker $baseline + Copy-Worker $candidate + Publish-Silo $baseline 'Old' $baselineSha + Publish-Silo $candidate 'New' $candidateSha + Invoke-DotNet @( + 'publish', (Join-Path $harnessSource 'Tests' 'Dissemination.PerformanceHarness.Tests.csproj'), + '--configuration', 'Release', '--framework', 'net10.0', + '--output', (Join-Path $binaries 'Runner'), '--verbosity', 'minimal', + "-bl:$(Join-Path $artifacts 'build-runner.binlog')" + ) + @{ + ToolingCommit = Get-Revision $root + BaselineRepository = 'dotnet/orleans' + BaselineCommit = $baselineSha + CandidateRepository = $CandidateRepository + CandidateRef = $CandidateRef + CandidateCommit = $candidateSha + DotNetInfo = (& dotnet --info | Out-String) + } | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $binaries 'selection.json') +} + +$selection = Get-Content -LiteralPath (Join-Path $binaries 'selection.json') -Raw | ConvertFrom-Json +if ($selection.CandidateRepository -ne $CandidateRepository -or $selection.CandidateRef -cne $CandidateRef -or + $selection.BaselineCommit -ne $baselineSha) { + throw 'Requested revisions must match the downloaded binary selection.json.' +} +foreach ($runtime in @('Old', 'New')) { + $manifest = Get-Content -LiteralPath (Join-Path $binaries $runtime 'manifest.json') -Raw | ConvertFrom-Json + $expected = if ($runtime -eq 'Old') { $baselineSha } else { $selection.CandidateCommit } + if ($manifest.SourceRevision -ne $expected -or $manifest.Runtime -ne $runtime) { + throw "Binary manifest does not match selected $runtime revision." + } +} + +$runner = Join-Path $binaries 'Runner' 'Dissemination.PerformanceHarness.Tests.dll' +if (!(Test-Path -LiteralPath $runner)) { + throw "Published runner not found: $runner" +} +$results = Join-Path $artifacts 'results' "$([DateTime]::UtcNow.ToString('yyyyMMddTHHmmss'))-$([Guid]::NewGuid().ToString('N'))" +New-Item -ItemType Directory -Force $results | Out-Null +Copy-Item -LiteralPath (Join-Path $harnessSource 'methodology.json') -Destination $results +Copy-Item -LiteralPath (Join-Path $binaries 'selection.json') -Destination $results + +$largestSize = ($sizesArray | Measure-Object -Maximum).Maximum +$requiredBytes = [long]$largestSize * 128MB + 1GB +if ($IsLinux) { + $memory = Get-Content -LiteralPath '/proc/meminfo' + $availableLine = $memory | Where-Object { $_ -match '^MemAvailable:\s+(\d+)\s+kB$' } + if (!$availableLine -or $availableLine -notmatch '^MemAvailable:\s+(\d+)\s+kB$') { + throw 'Cannot establish available host memory.' + } + $availableBytes = [long]$Matches[1] * 1024 +} +elseif ($IsWindows) { + $availableBytes = [long](Get-CimInstance Win32_OperatingSystem).FreePhysicalMemory * 1024 +} +else { + throw 'Resource preflight supports Linux CI and Windows/Linux local reproduction.' +} +@{ + AvailableBytes = $availableBytes + RequiredPlanningBytes = $requiredBytes + PlanningBytesPerSilo = 128MB + ControllerHeadroomBytes = 1GB + HostProcessorCount = [Environment]::ProcessorCount + SiloProcessorCount = $SiloProcessorCount + GCConserveMemory = $GCConserveMemory +} | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $results 'host-resources.json') +if ($availableBytes -lt $requiredBytes) { + throw "Insufficient memory for $largestSize silos: available=$availableBytes; planning requirement=$requiredBytes." +} + +$env:ORLEANS_DISSEMINATION_OLD = Join-Path $binaries 'Old' +$env:ORLEANS_DISSEMINATION_NEW = Join-Path $binaries 'New' +$env:ORLEANS_DISSEMINATION_RESULTS = $results +$env:ORLEANS_DISSEMINATION_CANDIDATE_SHA = $selection.CandidateCommit +$env:ORLEANS_DISSEMINATION_MANUAL_RUN = 'true' +$env:ORLEANS_DISSEMINATION_SIZES = $Sizes +$env:ORLEANS_DISSEMINATION_ITERATIONS = $Iterations.ToString([System.Globalization.CultureInfo]::InvariantCulture) +$env:ORLEANS_DISSEMINATION_REPETITIONS = $Repetitions.ToString([System.Globalization.CultureInfo]::InvariantCulture) +$env:ORLEANS_DISSEMINATION_RUNTIME_PATHS = $RuntimePaths +$env:ORLEANS_DISSEMINATION_SCENARIOS = $Scenarios +$env:ORLEANS_DISSEMINATION_WORKLOAD = $Workload +$env:ORLEANS_DISSEMINATION_SILO_PROCESSOR_COUNT = $SiloProcessorCount.ToString([System.Globalization.CultureInfo]::InvariantCulture) +$env:ORLEANS_DISSEMINATION_GC_CONSERVE_MEMORY = $GCConserveMemory.ToString([System.Globalization.CultureInfo]::InvariantCulture) +@{ + ToolingCommit = Get-Revision $root + Selection = $selection + Sizes = $Sizes + Iterations = $Iterations + Repetitions = $Repetitions + RuntimePaths = $RuntimePaths + Scenarios = $Scenarios + Workload = $Workload + SiloProcessorCount = $SiloProcessorCount + GCConserveMemory = $GCConserveMemory + StartUtc = [DateTimeOffset]::UtcNow + ResultsDirectory = $results +} | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath (Join-Path $results 'invocation.json') + +Push-Location $root +try { + Invoke-DotNet @( + $runner, '--filter-class', 'Orleans.Dissemination.PerformanceHarness.MeasurementTests', + '--minimum-expected-tests', '50', '--report-trx', + '--results-directory', (Join-Path $results 'instrument-checks') + ) + Invoke-DotNet @( + $runner, '--filter-class', 'Orleans.Dissemination.PerformanceHarness.ScalingTests', + '--minimum-expected-tests', '1', '--report-trx', '--results-directory', $results + ) +} +finally { + Pop-Location +} diff --git a/.github/scripts/test-dissemination-performance.ps1 b/.github/scripts/test-dissemination-performance.ps1 new file mode 100644 index 00000000000..9d036c4d145 --- /dev/null +++ b/.github/scripts/test-dissemination-performance.ps1 @@ -0,0 +1,307 @@ +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +$script = Join-Path $PSScriptRoot 'dissemination-performance.ps1' +$errors = $null +$scriptAst = [System.Management.Automation.Language.Parser]::ParseFile($script, [ref]$null, [ref]$errors) +if ($errors) { + throw ($errors | Out-String) +} + +$cases = @( + @{ Name = 'minimum process count'; Parameter = 'Sizes'; Value = '2'; Error = '*distinct silo counts*' } + @{ Name = 'maximum process count'; Parameter = 'Sizes'; Value = '129'; Error = '*distinct silo counts*' } + @{ Name = 'duplicate sizes'; Parameter = 'Sizes'; Value = '4,4'; Error = '*distinct silo counts*' } + @{ Name = 'size cardinality'; Parameter = 'Sizes'; Value = '3,4,5,6,7,8,9'; Error = '*cannot validate argument*' } + @{ Name = 'minimum rounds'; Parameter = 'Iterations'; Value = 2; Error = '*cannot validate argument*' } + @{ Name = 'maximum rounds'; Parameter = 'Iterations'; Value = 201; Error = '*cannot validate argument*' } + @{ Name = 'maximum repetitions'; Parameter = 'Repetitions'; Value = 6; Error = '*cannot validate argument*' } + @{ Name = 'duplicate scenarios'; Parameter = 'Scenarios'; Value = 'stable,stable'; Error = '*at most once*' } + @{ Name = 'unknown scenario'; Parameter = 'Scenarios'; Value = 'unknown'; Error = '*cannot validate argument*' } + @{ Name = 'absolute artifact path'; Parameter = 'ArtifactsDirectory'; Value = 'C:\outside'; Error = '*cannot validate argument*' } + @{ Name = 'artifact traversal'; Parameter = 'ArtifactsDirectory'; Value = 'Artifacts\..\outside'; Error = '*cannot validate argument*' } + @{ Name = 'artifact root'; Parameter = 'ArtifactsDirectory'; Value = 'Artifacts'; Error = '*cannot validate argument*' } + @{ Name = 'nested artifact path'; Parameter = 'ArtifactsDirectory'; Value = 'Artifacts\run\bin'; Error = '*cannot validate argument*' } + @{ Name = 'repository URL'; Parameter = 'CandidateRepository'; Value = 'https://github.com/ReubenBond/orleans'; Error = '*cannot validate argument*' } + @{ Name = 'ref option'; Parameter = 'CandidateRef'; Value = '--upload-pack=command'; Error = '*cannot validate argument*' } + @{ Name = 'processor bound'; Parameter = 'SiloProcessorCount'; Value = 65; Error = '*cannot validate argument*' } + @{ Name = 'GC bound'; Parameter = 'GCConserveMemory'; Value = 10; Error = '*cannot validate argument*' } + @{ Name = 'unknown workload'; Parameter = 'Workload'; Value = 'unknown'; Error = '*cannot validate argument*' } + @{ Name = 'open-loop stable only'; Parameter = 'Workload'; Value = 'OpenLoopSynchronized'; Error = 'Open-loop workloads require*' } + @{ Name = 'open-loop duration bound'; Parameter = 'Iterations'; Value = 31; Additional = @{ Workload = 'OpenLoopStaggered'; Scenarios = 'stable' }; Error = 'Open-loop workloads require*' } + @{ Name = 'open-loop process bound'; Parameter = 'Sizes'; Value = '4,33'; Additional = @{ Workload = 'OpenLoopSynchronized'; Scenarios = 'stable' }; Error = 'Open-loop workloads require*' } +) +foreach ($case in $cases) { + $arguments = @{ + CandidateRepository = 'ReubenBond/orleans' + CandidateRef = '3fc0a4610cd2eb5feef27e958fcca59658aaae47' + SkipBuild = $true + } + $arguments[$case.Parameter] = $case.Value + if ($case.ContainsKey('Additional')) { + foreach ($entry in $case.Additional.GetEnumerator()) { + $arguments[$entry.Key] = $entry.Value + } + } + $rejected = $false + try { + & $script @arguments + } + catch { + if ($_.Exception.Message -notlike $case.Error) { + throw "Unexpected failure for $($case.Name): $_" + } + $rejected = $true + } + if (!$rejected) { + throw "Expected rejection: $($case.Name)" + } +} +Write-Output "Passed $($cases.Count) script-boundary checks." + +$root = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..' '..')) +$artifactName = "DisseminationPathCheck-$([Guid]::NewGuid().ToString('N'))" +$testArtifacts = Join-Path $root 'Artifacts' $artifactName +$binaries = Join-Path $testArtifacts 'bin' +$runner = Join-Path $binaries 'Runner' 'Dissemination.PerformanceHarness.Tests.dll' +$calls = [System.Collections.Generic.List[object]]::new() +$environment = @{} +foreach ($entry in Get-ChildItem Env:ORLEANS_DISSEMINATION_*) { + $environment[$entry.Name] = $entry.Value +} + +function dotnet { + param([Parameter(ValueFromRemainingArguments)][string[]] $Arguments) + if ($Arguments[0] -ne $runner) { + throw "Unexpected runner path: $($Arguments[0])" + } + $calls.Add($Arguments) + $global:LASTEXITCODE = 0 +} + +try { + & { + foreach ($name in @('Assert-NoLinks', 'Copy-Worker')) { + $definition = $scriptAst.Find({ + param($node) + $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq $name + }, $false) + . ([scriptblock]::Create($definition.Extent.Text)) + } + + Assert-NoLinks $root + Assert-NoLinks (Join-Path $testArtifacts 'not-created') + foreach ($outside in @((Split-Path -Parent $root), [System.IO.Path]::GetPathRoot($root), "$root-sibling")) { + $rejected = $false + try { + Assert-NoLinks $outside + } + catch { + if ($_.Exception.Message -notlike 'Path is outside the repository boundary:*') { + throw + } + $rejected = $true + } + if (!$rejected) { + throw "Expected repository-boundary rejection for $outside." + } + } + Write-Output 'Passed 5 repository-boundary checks.' + + & { + $inspected = [System.Collections.Generic.List[string]]::new() + $denied = Join-Path $testArtifacts 'unreadable' + function Get-Item { + [CmdletBinding()] + param([string] $LiteralPath, [switch] $Force) + $inspected.Add($LiteralPath) + throw [System.UnauthorizedAccessException]::new('Boundary inspection denied.') + } + $rejected = $false + try { + Assert-NoLinks $denied + } + catch [System.UnauthorizedAccessException] { + $rejected = $true + } + if (!$rejected -or $inspected.Count -ne 1 -or $inspected[0] -ne $denied) { + throw 'Boundary inspection must propagate access errors at the requested path.' + } + } + Write-Output 'Passed boundary access-error propagation check.' + + $harnessSource = Join-Path $root 'test' 'Dissemination.PerformanceHarness' + $checkout = Join-Path $testArtifacts 'runtime-checkout' + $existingHarness = Join-Path $checkout 'test' 'Dissemination.IntegrationHarness' + New-Item -ItemType Directory -Path $existingHarness -Force | Out-Null + $sentinel = Join-Path $existingHarness 'preserved.txt' + Set-Content -LiteralPath $sentinel -Value 'Existing candidate harness remains intact.' + Copy-Worker $checkout + $copied = Join-Path $checkout 'test' 'Dissemination.PerformanceHarness' + if (@(Get-ChildItem -LiteralPath $copied -Directory).Count -ne 2) { + throw 'Copy-Worker must copy exactly the Silo and Shared directories.' + } + foreach ($part in @('Silo', 'Shared')) { + $expected = @(Get-ChildItem -LiteralPath (Join-Path $harnessSource $part) -File | + Where-Object { $_.Extension -in '.cs', '.csproj' }) + $actual = @(Get-ChildItem -LiteralPath (Join-Path $copied $part) -File) + if ($expected.Count -ne $actual.Count) { + throw "Copied $part inventory differs from the tooling source." + } + foreach ($file in $expected) { + $copy = Join-Path $copied $part $file.Name + if ((Get-FileHash -LiteralPath $file.FullName).Hash -ne (Get-FileHash -LiteralPath $copy).Hash) { + throw "Copied file differs: $copy" + } + } + } + $rejected = $false + try { + Copy-Worker $checkout + } + catch { + if ($_.Exception.Message -notlike 'Refusing to overwrite an existing runtime harness:*') { + throw + } + $rejected = $true + } + if (!$rejected -or (Get-Content -LiteralPath $sentinel) -ne 'Existing candidate harness remains intact.') { + throw 'Copy-Worker must preserve both existing harness directories.' + } + Write-Output 'Passed worker-copy inventory, coexistence and overwrite checks.' + + foreach ($linkedPart in @('test', 'harness')) { + foreach ($dangling in @($false, $true)) { + $caseName = "$linkedPart-$dangling" + $linkedCheckout = Join-Path $testArtifacts "linked-checkout-$caseName" + $target = Join-Path $testArtifacts "outside-checkout-$caseName" + New-Item -ItemType Directory -Path $linkedCheckout, $target | Out-Null + $link = Join-Path $linkedCheckout 'test' + if ($linkedPart -eq 'harness') { + New-Item -ItemType Directory -Path $link | Out-Null + $link = Join-Path $link 'Dissemination.PerformanceHarness' + } + $linkType = if ($IsWindows) { 'Junction' } else { 'SymbolicLink' } + New-Item -ItemType $linkType -Path $link -Target $target | Out-Null + try { + if ($dangling) { + [System.IO.Directory]::Delete($target) + } + else { + Set-Content -LiteralPath (Join-Path $target 'preserved.txt') -Value 'Untouched link target.' + } + $rejected = $false + try { + Copy-Worker $linkedCheckout + } + catch { + if ($_.Exception.Message -ne "Artifact and runtime paths must use ordinary directories: $link") { + throw + } + $rejected = $true + } + if (!$rejected) { + throw "Copy-Worker accepted a linked destination parent: $caseName" + } + if ($dangling) { + if (Test-Path -LiteralPath $target) { + throw "Copy-Worker created the dangling link target: $caseName" + } + } + else { + $files = @(Get-ChildItem -LiteralPath $target -Force) + if ($files.Count -ne 1 -or $files[0].Name -ne 'preserved.txt' -or + (Get-Content -LiteralPath $files[0].FullName) -ne 'Untouched link target.') { + throw "Copy-Worker modified the link target: $caseName" + } + } + } + finally { + Remove-Item -LiteralPath $link -Force + } + } + } + Write-Output 'Passed 4 worker-copy linked-parent checks with untouched targets.' + } + + $methodologyPath = Join-Path $root 'test' 'Dissemination.PerformanceHarness' 'methodology.json' + $methodology = Get-Content -LiteralPath $methodologyPath -Raw | ConvertFrom-Json + $scriptText = Get-Content -LiteralPath $script -Raw + $baselineMatch = [regex]::Match($scriptText, '(?m)^\$baselineSha = ''([0-9a-f]{40})''\r?$') + if (!$baselineMatch.Success -or $baselineMatch.Groups[1].Value -ne $methodology.baseline.commit) { + throw 'Script and methodology baseline pins differ.' + } + $baselineSha = $baselineMatch.Groups[1].Value + $controller = Get-Content -LiteralPath (Join-Path $root 'test' 'Dissemination.PerformanceHarness' 'Tests' 'ProcessCluster.cs') -Raw + $workflow = Get-Content -LiteralPath (Join-Path $PSScriptRoot '..' 'workflows' 'dissemination-performance.yml') -Raw + $readme = Get-Content -LiteralPath (Join-Path $root 'test' 'Dissemination.PerformanceHarness' 'README.md') -Raw + if (!$controller.Contains("public const string Baseline = `"$baselineSha`";") -or + $workflow -notmatch "repository: dotnet/orleans\s+ref: $baselineSha\b" -or !$readme.Contains($baselineSha)) { + throw 'Controller, workflow and documentation must retain the same pinned original runtime.' + } + Write-Output 'Passed baseline-pin consistency checks.' + + $selection = @{ + CandidateRepository = 'ReubenBond/orleans' + CandidateRef = '3fc0a4610cd2eb5feef27e958fcca59658aaae47' + CandidateCommit = '3fc0a4610cd2eb5feef27e958fcca59658aaae47' + BaselineCommit = '6739589254b746a8790cf53524e6abe372bb53d4' + } + foreach ($runtime in @('Old', 'New')) { + $directory = Join-Path $binaries $runtime + New-Item -ItemType Directory -Path $directory -Force | Out-Null + @{ + Runtime = $runtime + SourceRevision = if ($runtime -eq 'Old') { $selection.BaselineCommit } else { $selection.CandidateCommit } + } | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $directory 'manifest.json') + } + New-Item -ItemType Directory -Path (Split-Path -Parent $runner) | Out-Null + New-Item -ItemType File -Path $runner | Out-Null + $selection | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $binaries 'selection.json') + + foreach ($separator in @('/', '\')) { + & $script -CandidateRepository $selection.CandidateRepository -CandidateRef $selection.CandidateRef ` + -ArtifactsDirectory "Artifacts$separator$artifactName" -Sizes '3' -Iterations 3 -Repetitions 1 -SkipBuild + } + foreach ($workload in @('OpenLoopSynchronized', 'OpenLoopStaggered')) { + & $script -CandidateRepository $selection.CandidateRepository -CandidateRef $selection.CandidateRef ` + -ArtifactsDirectory (Join-Path 'Artifacts' $artifactName) -Sizes '3' -Iterations 30 -Repetitions 1 ` + -Scenarios stable -Workload $workload -SkipBuild + if ($env:ORLEANS_DISSEMINATION_WORKLOAD -ne $workload -or $env:ORLEANS_DISSEMINATION_ITERATIONS -ne '30') { + throw "Open-loop invocation lost its workload selection: $workload" + } + } + if ($calls.Count -ne 8) { + throw "Expected measurement and scaling invocations for two path spellings and two open-loop patterns; got $($calls.Count)." + } + for ($index = 0; $index -lt $calls.Count; $index++) { + $expectedClass = if ($index % 2 -eq 0) { 'MeasurementTests' } else { 'ScalingTests' } + if ($calls[$index][2] -ne "Orleans.Dissemination.PerformanceHarness.$expectedClass") { + throw "Unexpected test selection in invocation $index." + } + } + $runs = @(Get-ChildItem -LiteralPath (Join-Path $testArtifacts 'results') -Directory) + if ($runs.Count -ne 4) { + throw "Expected four results directories under the same artifact root; got $($runs.Count)." + } + foreach ($run in $runs) { + foreach ($filename in @('invocation.json', 'selection.json', 'methodology.json', 'host-resources.json')) { + $null = Get-Content -LiteralPath (Join-Path $run.FullName $filename) -Raw | ConvertFrom-Json + } + $invocation = Get-Content -LiteralPath (Join-Path $run.FullName 'invocation.json') -Raw | ConvertFrom-Json + if ($invocation.ResultsDirectory -ne $run.FullName) { + throw 'The recorded results path differs from the created directory.' + } + } + Write-Output 'Passed 4 SkipBuild path/workload checks with worker execution stubbed.' +} +finally { + foreach ($entry in Get-ChildItem Env:ORLEANS_DISSEMINATION_*) { + [Environment]::SetEnvironmentVariable($entry.Name, $null) + } + foreach ($entry in $environment.GetEnumerator()) { + [Environment]::SetEnvironmentVariable($entry.Key, $entry.Value) + } + if (Test-Path -LiteralPath $testArtifacts) { + Remove-Item -LiteralPath $testArtifacts -Recurse -Force + } +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20a90248ade..933b0111aff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,6 +70,9 @@ jobs: if: matrix.os == 'ubuntu-latest' shell: pwsh run: ./.github/scripts/test-summarize-coverage.ps1 + - name: Test performance harness script paths + shell: pwsh + run: ./.github/scripts/test-dissemination-performance.ps1 - name: Build run: dotnet build -bl - name: Pack and validate package compatibility diff --git a/.github/workflows/dissemination-performance.yml b/.github/workflows/dissemination-performance.yml new file mode 100644 index 00000000000..ebc569c2a82 --- /dev/null +++ b/.github/workflows/dissemination-performance.yml @@ -0,0 +1,128 @@ +name: Dissemination performance + +on: + workflow_dispatch: + inputs: + candidate_repository: + description: 'Public runtime repository (owner/repository); selected code executes on this runner' + type: string + required: true + default: 'ReubenBond/orleans' + candidate_ref: + description: 'Runtime commit, tag or branch; resolved commit is preserved in selection.json' + type: string + required: true + default: '3fc0a4610cd2eb5feef27e958fcca59658aaae47' + sizes: + description: 'Distinct process counts (3..128, at most six)' + type: string + default: '4,8' + iterations: + description: 'Closed-loop rounds (3..200), or open-loop 1 Hz duration in seconds (3..30)' + type: number + default: 3 + repetitions: + description: 'Paired repetitions with alternating forward/reverse order (1..5)' + type: number + default: 2 + runtime_paths: + description: 'All includes the pinned original; Current compares candidate off/on' + type: choice + options: [All, Current] + default: All + scenarios: + description: 'Distinct comma-separated scenarios: stable,churn,partition' + type: string + default: 'stable,churn,partition' + workload: + description: 'Open-loop requires stable only and at most 32 processes' + type: choice + options: [ClosedLoop, OpenLoopSynchronized, OpenLoopStaggered] + default: ClosedLoop + silo_processor_count: + description: 'Logical processors per silo (0 = inherit runner, maximum 64)' + type: number + default: 0 + gc_conserve_memory: + description: 'DOTNET_GCConserveMemory for every silo (0..9)' + type: number + default: 0 + +permissions: + contents: read + +concurrency: + group: dissemination-performance-${{ github.ref }} + cancel-in-progress: false + +env: + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + DOTNET_NOLOGO: true + +jobs: + measure: + name: Isolated original and candidate off/on measurements + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - name: Tooling checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + persist-credentials: false + - name: Original runtime checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + repository: dotnet/orleans + ref: 6739589254b746a8790cf53524e6abe372bb53d4 + path: .dissemination-runtime-original + persist-credentials: false + - name: Selected candidate runtime checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + repository: ${{ inputs.candidate_repository }} + ref: ${{ inputs.candidate_ref }} + path: .dissemination-runtime-candidate + persist-credentials: false + - uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5 + with: + global-json-file: global.json + - name: Check script input and artifact boundaries + shell: pwsh + run: ./.github/scripts/test-dissemination-performance.ps1 + - name: Build isolated binaries and measure selected workloads + shell: pwsh + env: + CANDIDATE_REPOSITORY: ${{ inputs.candidate_repository }} + CANDIDATE_REF: ${{ inputs.candidate_ref }} + PERFORMANCE_SIZES: ${{ inputs.sizes }} + PERFORMANCE_ITERATIONS: ${{ inputs.iterations }} + PERFORMANCE_REPETITIONS: ${{ inputs.repetitions }} + PERFORMANCE_PATHS: ${{ inputs.runtime_paths }} + PERFORMANCE_SCENARIOS: ${{ inputs.scenarios }} + PERFORMANCE_WORKLOAD: ${{ inputs.workload }} + PERFORMANCE_PROCESSORS: ${{ inputs.silo_processor_count }} + PERFORMANCE_GC: ${{ inputs.gc_conserve_memory }} + run: | + ./.github/scripts/dissemination-performance.ps1 ` + -CandidateRepository $env:CANDIDATE_REPOSITORY -CandidateRef $env:CANDIDATE_REF ` + -Sizes $env:PERFORMANCE_SIZES -Iterations ([int]$env:PERFORMANCE_ITERATIONS) ` + -Repetitions ([int]$env:PERFORMANCE_REPETITIONS) -RuntimePaths $env:PERFORMANCE_PATHS ` + -Scenarios $env:PERFORMANCE_SCENARIOS -Workload $env:PERFORMANCE_WORKLOAD ` + -SiloProcessorCount ([int]$env:PERFORMANCE_PROCESSORS) ` + -GCConserveMemory ([int]$env:PERFORMANCE_GC) + - name: Preserve raw measurements and build diagnostics + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: dissemination-performance-results + retention-days: 30 + path: | + Artifacts/DisseminationPerformance/results + Artifacts/DisseminationPerformance/*.binlog + - name: Preserve reproducible binaries and exact revision selection + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: dissemination-performance-binaries + retention-days: 7 + path: Artifacts/DisseminationPerformance/bin diff --git a/test/Dissemination.PerformanceHarness/ProviderTests/FileMembershipTableTests.cs b/test/Dissemination.PerformanceHarness/ProviderTests/FileMembershipTableTests.cs new file mode 100644 index 00000000000..4876c66bb34 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/ProviderTests/FileMembershipTableTests.cs @@ -0,0 +1,234 @@ +using System.Net; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Dissemination.PerformanceHarness; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Runtime")] +[TestCategory("BVT")] +public sealed class FileMembershipTableTests : IDisposable +{ + private static readonly DateTime Cutoff = new(2026, 1, 15, 12, 0, 0, DateTimeKind.Utc); + private readonly string _directory = Path.Combine(Path.GetTempPath(), $"orleans-dissemination-cleanup-{Guid.NewGuid():N}"); + + [Fact] + public async Task PersistedSuspectVotesPreserveDuplicatesOrderAndLatestUpdate() + { + var table = CreateTable(); + var cancellationToken = TestContext.Current.CancellationToken; + await table.InitializeMembershipTableAsync(true, cancellationToken); + var entry = CreateEntry(SiloStatus.Dead, 1); + var repeatedVoter = CreateEntry(SiloStatus.Active, 2).SiloAddress; + entry.AddSuspector(repeatedVoter, Cutoff.AddSeconds(-1)); + entry.AddSuspector(repeatedVoter, Cutoff); + entry.AddSuspector(CreateEntry(SiloStatus.Active, 3).SiloAddress, Cutoff.AddSeconds(-2)); + var initial = await table.ReadAllAsync(cancellationToken); + Assert.True(await table.InsertRowAsync(entry, initial.Version.Next(), cancellationToken)); + + var reopened = CreateTable(); + var inserted = await reopened.ReadAllAsync(cancellationToken); + var insertedRow = Assert.Single(inserted.Members); + Assert.Equal(entry.SuspectTimes, insertedRow.Item1.SuspectTimes); + Assert.Equal(Cutoff, insertedRow.Item1.EffectiveUpdateTime); + + entry.AddSuspector(repeatedVoter, Cutoff.AddSeconds(1)); + Assert.True(await reopened.UpdateRowAsync(entry, insertedRow.Item2, inserted.Version.Next(), cancellationToken)); + var updatedTable = CreateTable(); + var updated = await updatedTable.ReadAllAsync(cancellationToken); + var updatedRow = Assert.Single(updated.Members); + Assert.Equal(entry.SuspectTimes, updatedRow.Item1.SuspectTimes); + Assert.Equal(Cutoff.AddSeconds(1), updatedRow.Item1.EffectiveUpdateTime); + + await updatedTable.CleanupDefunctSiloEntriesAsync(Cutoff.AddSeconds(1), cancellationToken); + var retained = Assert.Single((await updatedTable.ReadAllAsync(cancellationToken)).Members); + Assert.Equal(entry.SuspectTimes, retained.Item1.SuspectTimes); + await updatedTable.CleanupDefunctSiloEntriesAsync(Cutoff.AddSeconds(1).AddTicks(1), cancellationToken); + Assert.Empty((await updatedTable.ReadAllAsync(cancellationToken)).Members); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task PersistedSuspectVotesPreserveNullAndEmptyLists(bool absent) + { + var table = CreateTable(); + var cancellationToken = TestContext.Current.CancellationToken; + await table.InitializeMembershipTableAsync(true, cancellationToken); + var entry = CreateEntry(SiloStatus.Active, 1); + entry.SuspectTimes = absent ? null : []; + var initial = await table.ReadAllAsync(cancellationToken); + Assert.True(await table.InsertRowAsync(entry, initial.Version.Next(), cancellationToken)); + + var reopened = CreateTable(); + var persisted = Assert.Single((await reopened.ReadAllAsync(cancellationToken)).Members).Item1.SuspectTimes; + if (absent) + { + Assert.Null(persisted); + } + else + { + Assert.NotNull(persisted); + Assert.Empty(persisted); + } + } + + [Fact] + public async Task DeleteInvalidatesThePreviousTableEtag() + { + var table = CreateTable(); + var cancellationToken = TestContext.Current.CancellationToken; + await table.InitializeMembershipTableAsync(true, cancellationToken); + var entry = CreateEntry(SiloStatus.Active, 1); + var initial = await table.ReadAllAsync(cancellationToken); + Assert.True(await table.InsertRowAsync(entry, initial.Version.Next(), cancellationToken)); + var before = await table.ReadAllAsync(cancellationToken); + var previousRow = Assert.Single(before.Members); + + await table.DeleteMembershipTableEntriesAsync("cleanup", cancellationToken); + var reopened = CreateTable(); + var deleted = await reopened.ReadAllAsync(cancellationToken); + Assert.Empty(deleted.Members); + Assert.Equal(before.Version.Version + 1, deleted.Version.Version); + Assert.NotEqual(before.Version.VersionEtag, deleted.Version.VersionEtag); + Assert.False(await reopened.InsertRowAsync(entry, before.Version.Next(), cancellationToken)); + Assert.False(await reopened.UpdateRowAsync(entry, previousRow.Item2, before.Version.Next(), cancellationToken)); + var afterRejectedWrites = await reopened.ReadAllAsync(cancellationToken); + Assert.Empty(afterRejectedWrites.Members); + Assert.Equal(deleted.Version, afterRejectedWrites.Version); + + Assert.True(await reopened.InsertRowAsync(entry, deleted.Version.Next(), cancellationToken)); + var inserted = await reopened.ReadAllAsync(cancellationToken); + var currentRow = Assert.Single(inserted.Members); + Assert.Equal(entry.SiloAddress, currentRow.Item1.SiloAddress); + Assert.False(await reopened.UpdateRowAsync(entry, currentRow.Item2, before.Version.Next(), cancellationToken)); + Assert.Equal(inserted.Version, (await reopened.ReadAllAsync(cancellationToken)).Version); + } + + [Fact] + public async Task HeartbeatPreservesTableVersionAndOtherColumns() + { + var table = CreateTable(); + var cancellationToken = TestContext.Current.CancellationToken; + await table.InitializeMembershipTableAsync(true, cancellationToken); + var entry = CreateEntry(SiloStatus.Active, 1); + var initial = await table.ReadAllAsync(cancellationToken); + Assert.True(await table.InsertRowAsync(entry, initial.Version.Next(), cancellationToken)); + var before = await table.ReadAllAsync(cancellationToken); + var previousRow = Assert.Single(before.Members); + + var update = CreateEntry(SiloStatus.Dead, 1); + update.HostName = "ignored"; + update.IAmAliveTime = Cutoff; + await table.UpdateIAmAliveAsync(update, cancellationToken); + var reopened = CreateTable(); + var after = await reopened.ReadAllAsync(cancellationToken); + var currentRow = Assert.Single(after.Members); + Assert.Equal(before.Version, after.Version); + Assert.Equal(previousRow.Item2, currentRow.Item2); + var expected = FileMembershipTable.EntryData.From(previousRow.Item1) with { IAmAliveTime = Cutoff }; + Assert.Equal(StateComparison.Serialize(expected), StateComparison.Serialize(FileMembershipTable.EntryData.From(currentRow.Item1))); + } + + [Fact] + public async Task CleanupRemovesEveryExpiredNonActiveStatus() + { + var table = CreateTable(); + await table.InitializeMembershipTableAsync(true, TestContext.Current.CancellationToken); + foreach (var status in Enum.GetValues()) + { + var entry = CreateEntry(status, (int)status + 1); + var data = await table.ReadAllAsync(TestContext.Current.CancellationToken); + Assert.True(await table.InsertRowAsync(entry, data.Version.Next(), TestContext.Current.CancellationToken)); + } + + var before = await table.ReadAllAsync(TestContext.Current.CancellationToken); + var expected = Assert.Single(before.Members, row => row.Item1.Status == SiloStatus.Active); + var reopened = CreateTable(); + await reopened.CleanupDefunctSiloEntriesAsync(Cutoff, TestContext.Current.CancellationToken); + var after = await reopened.ReadAllAsync(TestContext.Current.CancellationToken); + + var remaining = Assert.Single(after.Members); + Assert.Equal(expected.Item1.SiloAddress, remaining.Item1.SiloAddress); + Assert.Equal(SiloStatus.Active, remaining.Item1.Status); + Assert.Equal(expected.Item2, remaining.Item2); + Assert.Equal(before.Version.Version, after.Version.Version); + Assert.Equal(before.Version.VersionEtag, after.Version.VersionEtag); + } + + [Theory] + [InlineData("start", -1)] + [InlineData("start", 0)] + [InlineData("start", 1)] + [InlineData("heartbeat", -1)] + [InlineData("heartbeat", 0)] + [InlineData("heartbeat", 1)] + [InlineData("suspect", -1)] + [InlineData("suspect", 0)] + [InlineData("suspect", 1)] + public async Task CleanupUsesEffectiveUpdateTimeAndExclusiveCutoff(string timestamp, int offset) + { + var table = CreateTable(); + await table.InitializeMembershipTableAsync(true, TestContext.Current.CancellationToken); + var entry = CreateEntry(SiloStatus.Dead, 1); + var latest = Cutoff.AddTicks(offset); + switch (timestamp) + { + case "start": + entry.StartTime = latest; + break; + case "heartbeat": + entry.IAmAliveTime = latest; + break; + case "suspect": + entry.SuspectTimes = + [ + Tuple.Create(CreateEntry(SiloStatus.Active, 2).SiloAddress, latest), + Tuple.Create(CreateEntry(SiloStatus.Active, 3).SiloAddress, Cutoff.AddDays(-2)), + ]; + break; + } + + var data = await table.ReadAllAsync(TestContext.Current.CancellationToken); + Assert.True(await table.InsertRowAsync(entry, data.Version.Next(), TestContext.Current.CancellationToken)); + var reopened = CreateTable(); + var persisted = Assert.Single((await reopened.ReadAllAsync(TestContext.Current.CancellationToken)).Members); + Assert.Equal(latest, persisted.Item1.EffectiveUpdateTime); + + await reopened.CleanupDefunctSiloEntriesAsync(Cutoff, TestContext.Current.CancellationToken); + var after = await reopened.ReadAllAsync(TestContext.Current.CancellationToken); + if (offset < 0) + { + Assert.Empty(after.Members); + } + else + { + var remaining = Assert.Single(after.Members); + Assert.Equal(entry.SiloAddress, remaining.Item1.SiloAddress); + Assert.Equal(latest, remaining.Item1.EffectiveUpdateTime); + Assert.Equal(persisted.Item2, remaining.Item2); + } + } + + private FileMembershipTable CreateTable() => + new(new("cleanup", "cleanup", _directory, 0, Environment.ProcessId, Enabled: false)); + + private static MembershipEntry CreateEntry(SiloStatus status, int port) => new() + { + SiloAddress = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 10000 + port), 1), + HostName = "localhost", + SiloName = $"cleanup-{port}", + Status = status, + StartTime = Cutoff.AddDays(-1), + IAmAliveTime = Cutoff.AddDays(-1), + }; + + public void Dispose() + { + if (Directory.Exists(_directory)) + { + Directory.Delete(_directory, recursive: true); + } + } +} diff --git a/test/Dissemination.PerformanceHarness/README.md b/test/Dissemination.PerformanceHarness/README.md new file mode 100644 index 00000000000..1a52023d3ec --- /dev/null +++ b/test/Dissemination.PerformanceHarness/README.md @@ -0,0 +1,197 @@ +# Dissemination performance harness + +This manually invoked harness measures the original Orleans runtime and a selected +candidate with dissemination explicitly off and on. Each silo is a separate OS +process using the real runtime serializer, transport, membership and load publisher. +The controller reports raw per-node costs and equal-work off/on comparisons. + +## Select runtime revisions + +Run the **Dissemination performance** workflow from the tooling branch with an +explicit `candidate_repository` and `candidate_ref`. Prefer a full commit SHA for +reproducibility. The initial comparison uses `ReubenBond/orleans` at +`3fc0a4610cd2eb5feef27e958fcca59658aaae47`; the original baseline is pinned to +`dotnet/orleans` at `6739589254b746a8790cf53524e6abe372bb53d4`. + +```powershell +gh workflow run dissemination-performance.yml --repo ReubenBond/orleans ` + --ref ` + -f candidate_repository=ReubenBond/orleans ` + -f candidate_ref=3fc0a4610cd2eb5feef27e958fcca59658aaae47 ` + -f sizes=4,8 -f iterations=3 -f repetitions=2 ` + -f runtime_paths=All -f scenarios=stable,churn,partition ` + -f workload=ClosedLoop ` + -f silo_processor_count=0 -f gc_conserve_memory=0 +``` + +The workflow runs only through `workflow_dispatch`. GitHub requires a workflow +to be registered on the repository's default branch before it can be dispatched. +Select trusted public runtime revisions: their build code executes on the runner +with read-only repository permissions and checkout credentials disabled. +The operator who can dispatch this workflow authorizes the exact repository/ref to +execute. Inspect that revision's build inputs and dependencies before dispatch; +selected code has the hosted runner's network access. Results record both the +requested ref and resolved commit for audit. + +CI creates separate original and candidate checkouts, verifies their origins and +clean state, and copies only `Silo` and `Shared` support sources into each checkout. +Before creating worker files, the copy step validates every existing destination +parent through the checkout, rejecting symlinks and reparse points within it. +Each worker builds against that checkout's runtime projects and package settings. +The controller builds against the tooling checkout's shared protocol, instrumentation +and reflection-boundary helpers. +The tooling branch can therefore be based on main while the selected candidate +provides the dissemination APIs. `NewRuntime.cs` describes the expected internal API +shape; update this adapter deliberately when testing a candidate with changed APIs. +The shared publisher adapter validates the publication method's signature and the +disposable timer field, reporting the incompatible member and selected assembly. +Aggregation-tree reports use the optional public `Overlay.AggregationFanOutFactor` +property when present, clamped to the candidate's effective range +`1..max(1, memberCount)`. Older candidates such as `3fc0a461` use their membership +fanout formula. `FanoutSource` names the detected capability; an incompatible +property shape raises an adapter diagnostic. Membership routing retains its selector. + +Published manifests record DLL hashes, versions and source commits. Every worker +checks the loaded Orleans assemblies and exact process identity. Candidate off/on +runs share one binary directory. `selection.json` preserves the resolved candidate, +original and tooling commits and build SDK information. + +## Workload and bounds + +Defaults are 4 and 8 processes, three rounds, two paired repetitions, and all three +paths across `stable`, `churn` and `partition`. Sizes accept one to six distinct +counts from 3 through 128; rounds accept 3 through 200; repetitions accept 1 through +5. Successive repetitions reverse the complete execution order, including off/on. + +`workload=ClosedLoop` preserves the publication/convergence barrier after each round. +For fixed-rate stable measurement, choose `OpenLoopSynchronized` or +`OpenLoopStaggered`, `scenarios=stable`, 3 through 32 processes, and `iterations` +from 3 through 30 (duration in seconds at one publication per producer per second). +Synchronized producers share each one-second boundary; staggered producers use +evenly spaced phases within that second. Each worker arms a local monotonic schedule +at a common future UTC epoch, then calls the production publisher on its scheduler. +Controller polling observes state independently of these publication deadlines. + +```powershell +gh workflow run dissemination-performance.yml --repo ReubenBond/orleans ` + --ref -f candidate_repository=ReubenBond/orleans ` + -f candidate_ref= -f sizes=4,8,32 -f iterations=10 ` + -f repetitions=2 -f runtime_paths=Current -f scenarios=stable ` + -f workload=OpenLoopStaggered -f silo_processor_count=0 -f gc_conserve_memory=0 +``` + +Arming must finish within the five-second lead. A producer fails on a missed period +or a publication extending past its one-second slot, preserving partial timestamps, +missed-period and overrun counts. Each accepted run offers exactly +`processes * durationSeconds` publications. Producers retain the configured rate +without catch-up bursts. The controller has a separate ten-second completion and +latest-state drain after the scheduled end. It verifies active process identities, +exact latest dictionary inventory and values, and that observed versions and values +were actually published. + +Open-loop artifacts preserve planned/actual start and completion timestamps, +actual inter-publication periods, first-observed peer ages, and each producer's +final value reaching every process. Ages and final-latest latency are **polling +upper bounds**, using same-host UTC timestamps and 100 ms controller polling. +Intermediate values may coalesce; observed and unobserved publication/observer pairs +are reported separately. Exact latest-state convergence remains required. + +Every round invokes the actual load publisher once per active node and checks exact +state inventory and field values at every node. Churn restarts a process at the same +endpoint. Partition recovery drains connections at both endpoints, including +unfinished handshakes, then confirms bidirectional acknowledged control RPCs before +one publication round. This preserves the #11243 transport-readiness ordering. +Readiness time and RPC costs stay inside recovery and total cost windows. + +Automatic load publication is paused by disposing its timer on the publisher +scheduler. The timer field retains its disposed reference for original-runtime +shutdown. Enabled runs explicitly opt into the subsystem and both namespaces while +retaining the candidate's production dissemination tuning defaults. Offered rounds +begin after peer support confirmation; replacement startup and support confirmation +remain inside churn costs. + +The Linux/Windows memory guard plans 128 MiB per silo plus 1 GiB controller headroom. +Commands, startup, shutdown and processes have deadlines; the workflow has a +90-minute budget. For an explicitly requested large stable curve, select +`sizes=8,16,32,64,100`, `runtime_paths=Current`, `scenarios=stable`, +`silo_processor_count=1`, `gc_conserve_memory=9`, and two repetitions on a suitably +provisioned host. These controls are equal across each off/on pair. + +## Results and interpretation + +`Artifacts\DisseminationPerformance\results\-` contains raw +`cost-results.json`, normalized `comparisons.json`, `selection.json`, invocation, +methodology, host resources, TRX, and per-cluster logs and snapshots. The workflow +also preserves build binlogs and `bin\{Old,New,Runner}` as separate downloadable +artifacts. Results retain partial samples and explicit incomplete pairs on failure. + +Metrics include RPC/message counts, complete serialized bytes, socket and transport +bytes, CPU, allocations, retained memory, publication/convergence/recovery latency, +and per-node topology. Load broadcast requests and logical value transmissions are +separate counters. Reports compare equal binary, environment, process count and +offered work, with costs per publication and median/P95 convergence. +Open-loop comparisons use median/P95 final-latest polling upper bounds instead of +per-round closed-loop latency. Raw `open-loop-{plans,producers,observations,summary}.json` +files preserve cadence and observation evidence. Total costs include arming, the +fixed-rate window, final-state drain, and the common socket-sampling margin. + +These are shared-runner, single-host loopback measurements. The workload is +closed-loop by default, with a convergence barrier after each round. The optional +open-loop mode fixes the offered producer rate at 1 Hz. Silo costs include control +and instrumentation work; controller costs, storage disk bytes, cross-machine latency, +TCP/IP wire overhead and application-grain workloads need separate experiments. +The 100 ms polling cadence and one-second socket sampling introduce boundary +uncertainty. Retained memory represents post-GC live bytes on surviving nodes. +Original-to-candidate changes include all intervening commits; same-candidate off/on +pairs isolate enablement. See [methodology.json](methodology.json) for exact windows. + +## Local reproduction and focused checks + +Download `dissemination-performance-binaries` from a successful dispatch into +`Artifacts\DisseminationPerformance\bin` in the same tooling revision. Install a +matching .NET 10/ASP.NET Core shared framework for Linux or Windows, then replay +the exact repository/ref recorded in `bin\selection.json`: + +```powershell +.\.github\scripts\dissemination-performance.ps1 -SkipBuild ` + -CandidateRepository ReubenBond/orleans ` + -CandidateRef 3fc0a4610cd2eb5feef27e958fcca59658aaae47 ` + -Sizes '4,8' -Iterations 3 -Repetitions 2 ` + -RuntimePaths All -Scenarios 'stable,churn,partition' +``` + +`-ArtifactsDirectory` accepts a single named directory under `Artifacts`; paths +accept either slash spelling and are assembled from platform-native components. +Existing links/reparse points are rejected. Builds require fresh binary output; +each replay creates its own results directory. + +Run only the lightweight controller measurement cases locally: + +```powershell +.\.github\scripts\test-dissemination-performance.ps1 +dotnet test --project test\Dissemination.PerformanceHarness\Tests\Dissemination.PerformanceHarness.Tests.csproj ` + --framework net10.0 --filter-class '*MeasurementTests' --minimum-expected-tests 50 +``` + +Ordinary Linux/Windows CI runs script checks for repository containment, worker-copy +coexistence, baseline-pin consistency and replay paths with worker execution stubbed. +`ProviderTests\FileMembershipTableTests.cs` is linked into `Orleans.Runtime.Tests` +alongside the file-backed provider and its protocol. These BVT cases exercise +persisted membership cleanup: all expired non-active statuses are removed, while +active rows and rows updated at or after the cutoff retain their ETags. Cleanup uses +the selected runtime's `EffectiveUpdateTime`, including startup, heartbeat and +suspect-vote timestamps. Persisted vote lists retain their order, repeated voters, +and null-versus-empty state across inserts and updates. +Clearing membership advances the table ETag so writes from an earlier snapshot +are rejected. Heartbeat dirty writes preserve the table version and update only +the liveness timestamp. + +```powershell +dotnet test --project test\Orleans.Runtime.Tests\Orleans.Runtime.Tests.csproj ` + --framework net10.0 --filter-class 'Orleans.Dissemination.PerformanceHarness.FileMembershipTableTests' ` + --minimum-expected-tests 15 +``` + +The scaling test runs through the script's explicit manual-run marker and published +binary selection. The measurement controller retains its independent build using +shared protocol and instrumentation sources. diff --git a/test/Dissemination.PerformanceHarness/Shared/AggregationFanout.cs b/test/Dissemination.PerformanceHarness/Shared/AggregationFanout.cs new file mode 100644 index 00000000000..fb74a3024b2 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Shared/AggregationFanout.cs @@ -0,0 +1,32 @@ +using System.Reflection; + +namespace Orleans.Dissemination.PerformanceHarness; + +internal sealed record AggregationFanout(int Value, string Source) +{ + public static AggregationFanout Read(object overlay, bool aggregationTree, int memberCount, Func legacyFanout) + { + if (!aggregationTree) + { + return new(legacyFanout(), "MembershipFanout"); + } + + var type = overlay.GetType(); + var property = type.GetProperty("AggregationFanOutFactor", BindingFlags.Instance | BindingFlags.Public); + if (property is null) + { + return new(legacyFanout(), "LegacyAggregationMembershipFanout"); + } + + if (property.PropertyType != typeof(int) || property.GetIndexParameters().Length != 0 + || property.GetMethod is not { IsPublic: true } + || property.GetValue(overlay) is not int value) + { + throw new InvalidOperationException( + $"Selected runtime '{type.Assembly.FullName}' must expose a public int {type.FullName}.AggregationFanOutFactor. " + + "Update the performance harness runtime adapter for this candidate."); + } + + return new(Math.Clamp(value, 1, Math.Max(1, memberCount)), "Overlay.AggregationFanOutFactor"); + } +} diff --git a/test/Dissemination.PerformanceHarness/Shared/OpenLoopProducer.cs b/test/Dissemination.PerformanceHarness/Shared/OpenLoopProducer.cs new file mode 100644 index 00000000000..5f4eda4d3ac --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Shared/OpenLoopProducer.cs @@ -0,0 +1,148 @@ +namespace Orleans.Dissemination.PerformanceHarness; + +internal sealed record OpenLoopPlan(DateTimeOffset StartUtc, int DurationSeconds, int ProducerIndex, int ProducerCount, string Workload) +{ + public void Validate() + { + if (DurationSeconds is < 3 or > 30 || ProducerCount is < 3 or > 32 + || ProducerIndex < 0 || ProducerIndex >= ProducerCount + || Workload is not ("OpenLoopSynchronized" or "OpenLoopStaggered")) + { + throw new InvalidOperationException("Open-loop requires 3..32 producers, 3..30 seconds, and a synchronized or staggered pattern."); + } + } + + public TimeSpan Offset(int sequence) => + TimeSpan.FromSeconds(sequence) + (Workload == "OpenLoopStaggered" + ? TimeSpan.FromTicks(TimeSpan.TicksPerSecond * ProducerIndex / ProducerCount) + : TimeSpan.Zero); +} + +internal sealed record ProducedValue(DateTimeOffset StartedAtUtc, DateTimeOffset CompletedAtUtc, long Version, string Value); +internal sealed record OpenLoopPublication(int Sequence, DateTimeOffset ScheduledAtUtc, ProducedValue Result); +internal sealed record OpenLoopReport(OpenLoopPlan Plan, OpenLoopPublication[] Publications, int MissedPeriods, int Overruns); +internal sealed record OpenLoopProgress(int PublishedCount, bool Completed, int MissedPeriods, int Overruns, string? Error); + +internal sealed class OpenLoopProducer(TimeProvider timeProvider) : IAsyncDisposable +{ + private CancellationTokenSource? _cancellation; + private Task? _completion; + private int _published; + private int _missedPeriods; + private int _overruns; + private OpenLoopPlan? _plan; + private readonly List _publications = []; + private readonly object _lock = new(); + + public Task Completion => _completion + ?? throw new InvalidOperationException("The open-loop producer has not been armed."); + + public OpenLoopProgress? Progress => _completion is { } task + ? new(Volatile.Read(ref _published), task.IsCompletedSuccessfully, Volatile.Read(ref _missedPeriods), + Volatile.Read(ref _overruns), task.Exception?.GetBaseException().ToString()) + : null; + + public OpenLoopReport Report + { + get + { + lock (_lock) + { + return new(_plan ?? throw new InvalidOperationException("The open-loop producer has not been armed."), + _publications.ToArray(), _missedPeriods, _overruns); + } + } + } + + public void Start(OpenLoopPlan plan, Func> publish, CancellationToken cancellationToken) + { + if (_completion is not null) + { + throw new InvalidOperationException("The open-loop producer has already been armed."); + } + + plan.Validate(); + var lead = plan.StartUtc - timeProvider.GetUtcNow(); + if (lead <= TimeSpan.Zero || lead > TimeSpan.FromSeconds(10)) + { + throw new InvalidOperationException("Open-loop start must be in the next 10 seconds; all workers must acknowledge arming before it."); + } + + _cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + _plan = plan; + _completion = Run(plan, lead, publish, _cancellation.Token); + _ = _completion.ContinueWith( + static task => _ = task.Exception, CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); + } + + private async Task Run( + OpenLoopPlan plan, TimeSpan lead, Func> publish, CancellationToken cancellationToken) + { + var origin = timeProvider.GetTimestamp(); + for (var sequence = 0; sequence < plan.DurationSeconds; sequence++) + { + var due = lead + plan.Offset(sequence); + await DelayUntil(due); + var elapsed = timeProvider.GetElapsedTime(origin); + if (elapsed - due >= TimeSpan.FromSeconds(1)) + { + Volatile.Write(ref _missedPeriods, (int)(elapsed - due).TotalSeconds); + throw new InvalidOperationException($"Open-loop producer {plan.ProducerIndex} missed publication slot {sequence}."); + } + + var result = await publish(cancellationToken); + lock (_lock) + { + if (_publications.Count > 0 && result.Version <= _publications[^1].Result.Version) + { + throw new InvalidOperationException("The production publisher did not create a strictly newer sample."); + } + _publications.Add(new(sequence, plan.StartUtc + plan.Offset(sequence), result)); + Volatile.Write(ref _published, _publications.Count); + } + if (timeProvider.GetElapsedTime(origin) - due >= TimeSpan.FromSeconds(1)) + { + Volatile.Write(ref _overruns, 1); + throw new InvalidOperationException($"Open-loop producer {plan.ProducerIndex} publication {sequence} exceeded its 1 Hz slot."); + } + } + + await DelayUntil(lead + TimeSpan.FromSeconds(plan.DurationSeconds)); + return Report; + + async Task DelayUntil(TimeSpan due) + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + var remaining = due - timeProvider.GetElapsedTime(origin); + if (remaining <= TimeSpan.Zero) + { + return; + } + await Task.Delay(TimeSpan.FromMilliseconds(Math.Ceiling(remaining.TotalMilliseconds)), timeProvider, cancellationToken); + } + } + } + + public async ValueTask DisposeAsync() + { + if (_cancellation is { } cancellation) + { + await cancellation.CancelAsync(); + try + { + await Completion.WaitAsync(TimeSpan.FromSeconds(5)); + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + } + finally + { + cancellation.Dispose(); + _cancellation = null; + } + } + } +} diff --git a/test/Dissemination.PerformanceHarness/Shared/Protocol.cs b/test/Dissemination.PerformanceHarness/Shared/Protocol.cs new file mode 100644 index 00000000000..4ea011d22d5 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Shared/Protocol.cs @@ -0,0 +1,89 @@ +using System.Text.Json; + +namespace Orleans.Dissemination.PerformanceHarness; + +internal sealed record NodeConfiguration( + string Name, + string ClusterId, + string MembershipDirectory, + int SiloPort, + int ParentProcessId, + bool Enabled, + int SiloProcessorCount = 0, + int GCConserveMemory = 0); + +internal sealed record Command(int Id, string Operation, string? Peer = null, OpenLoopPlan? OpenLoop = null); + +internal sealed record Response(int Id, NodeSnapshot? Snapshot, string? Error); + +internal sealed record BinaryIdentity( + int ProcessId, + string Runtime, + string SourceRevision, + string AssemblyName, + string AssemblyVersion, + string InformationalVersion, + string ModuleVersionId, + string Sha256, + string AssemblyPath, + bool HasDissemination) +{ + public Dictionary Assemblies { get; init; } = []; +} + +internal sealed record AssemblyProof( + string Version, + string InformationalVersion, + string ModuleVersionId, + string Sha256, + string Path); + +internal sealed record MetricValue(long Count, double Sum); + +internal static class StateComparison +{ + private static readonly JsonSerializerOptions Options = new() { IncludeFields = true }; + + public static string Serialize(T value) => JsonSerializer.Serialize(value, Options); +} + +internal sealed record NodeSnapshot +{ + public DateTimeOffset CapturedAtUtc { get; init; } + public required BinaryIdentity Identity { get; init; } + public required string Address { get; init; } + public bool Enabled { get; init; } + public bool NamespaceEnabled { get; init; } + public long MembershipVersion { get; init; } + public required SortedDictionary Membership { get; init; } + public required SortedDictionary Load { get; init; } + public required SortedDictionary LoadVersions { get; init; } + public required string[] ActiveMembers { get; init; } + public required string[] UnconfirmedPeers { get; init; } + public required string[] OriginatorTargets { get; init; } + public required string[] ForwardingTargets { get; init; } + public string[] TopologyMembers { get; init; } = []; + public int Fanout { get; init; } + public string? FanoutSource { get; init; } + public bool AggregationTree { get; init; } + public int AntiEntropyPeerCount { get; init; } + public double AntiEntropyIntervalMilliseconds { get; init; } + public required Dictionary Metrics { get; init; } + public long TransportBytesWritten { get; init; } + public long TransportBytesRead { get; init; } + public double SocketBytesSent { get; init; } + public double SocketBytesReceived { get; init; } + public long SocketCounterSamples { get; init; } + public double CpuMilliseconds { get; init; } + public long AllocatedBytes { get; init; } + public long ManagedHeapBytes { get; init; } + public long WorkingSetBytes { get; init; } + public long PrivateBytes { get; init; } + public int ProcessorCount { get; init; } + public bool ServerGC { get; init; } + public bool Partitioned { get; init; } + public int? RemoteProcessId { get; init; } + public string? ProbeError { get; init; } + public OpenLoopProgress? OpenLoopProgress { get; init; } + public OpenLoopReport? OpenLoopReport { get; init; } +} diff --git a/test/Dissemination.PerformanceHarness/Shared/PublisherApi.cs b/test/Dissemination.PerformanceHarness/Shared/PublisherApi.cs new file mode 100644 index 00000000000..0ac9ce653a7 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Shared/PublisherApi.cs @@ -0,0 +1,33 @@ +using System.Reflection; + +namespace Orleans.Dissemination.PerformanceHarness; + +internal static class PublisherApi +{ + public static MethodInfo GetPublishMethod(Type publisherType) + { + var method = publisherType.GetMethod( + "PublishStatistics", BindingFlags.Instance | BindingFlags.NonPublic, [typeof(CancellationToken)]); + if (method is null || method.ReturnType != typeof(Task)) + { + throw Incompatible(publisherType, "Task PublishStatistics(CancellationToken)"); + } + + return method; + } + + public static FieldInfo GetTimerField(Type publisherType) + { + var field = publisherType.GetField("_publishTimer", BindingFlags.Instance | BindingFlags.NonPublic); + if (field is null || !typeof(IDisposable).IsAssignableFrom(field.FieldType)) + { + throw Incompatible(publisherType, "IDisposable _publishTimer"); + } + + return field; + } + + private static InvalidOperationException Incompatible(Type publisherType, string member) => + new($"Selected runtime '{publisherType.Assembly.FullName}' must provide {publisherType.FullName}.{member}. " + + "Update the performance harness runtime adapter for this candidate."); +} diff --git a/test/Dissemination.PerformanceHarness/Silo/ControlTarget.cs b/test/Dissemination.PerformanceHarness/Silo/ControlTarget.cs new file mode 100644 index 00000000000..e0a1acded30 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Silo/ControlTarget.cs @@ -0,0 +1,24 @@ +using Orleans.Runtime; + +namespace Orleans.Dissemination.PerformanceHarness; + +[Alias("dissemination-performance-control-v1")] +internal interface IControlTarget : ISystemTarget +{ + [Alias("echo-v1")] + Task Echo(CancellationToken cancellationToken); +} + +internal sealed class ControlTarget : SystemTarget, IControlTarget +{ + public static readonly GrainType TargetType = SystemTargetGrainId.CreateGrainType("test.dissemination-performance"); + + public ControlTarget(SystemTargetShared shared) : base(TargetType, shared) => + shared.ActivationDirectory.RecordNewTarget(this); + + public Task Echo(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult(Environment.ProcessId); + } +} diff --git a/test/Dissemination.PerformanceHarness/Silo/Dissemination.Silo.csproj b/test/Dissemination.PerformanceHarness/Silo/Dissemination.Silo.csproj new file mode 100644 index 00000000000..7a4162a987b --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Silo/Dissemination.Silo.csproj @@ -0,0 +1,24 @@ + + + net10.0 + Exe + + Orleans.Runtime.Tests + Orleans.Dissemination.PerformanceHarness + true + $(DefineConstants);NEW_RUNTIME + + + + + + + + + + + + + + diff --git a/test/Dissemination.PerformanceHarness/Silo/FileMembershipTable.cs b/test/Dissemination.PerformanceHarness/Silo/FileMembershipTable.cs new file mode 100644 index 00000000000..84b63c6ba06 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Silo/FileMembershipTable.cs @@ -0,0 +1,230 @@ +using System.Text.Json; +using Orleans.Runtime; + +namespace Orleans.Dissemination.PerformanceHarness; + +// A shared JSON schema and exclusive file lease coordinate the loopback silo processes. +internal sealed class FileMembershipTable(NodeConfiguration configuration) : IMembershipTable +{ + private readonly string _dataPath = Path.Combine(configuration.MembershipDirectory, "membership.json"); + private readonly string _lockPath = Path.Combine(configuration.MembershipDirectory, "membership.lock"); + +#if NEW_RUNTIME + Task IMembershipTable.InitializeMembershipTable(bool tryInitTableVersion) => + InitializeMembershipTableAsync(tryInitTableVersion, CancellationToken.None); + + Task IMembershipTable.DeleteMembershipTableEntries(string clusterId) => + DeleteMembershipTableEntriesAsync(clusterId, CancellationToken.None); + + Task IMembershipTable.CleanupDefunctSiloEntries(DateTimeOffset beforeDate) => + CleanupDefunctSiloEntriesAsync(beforeDate, CancellationToken.None); + + Task IMembershipTable.ReadRow(SiloAddress key) => ReadRowAsync(key, CancellationToken.None); + + Task IMembershipTable.ReadAll() => ReadAllAsync(CancellationToken.None); + + Task IMembershipTable.InsertRow(MembershipEntry entry, TableVersion tableVersion) => + InsertRowAsync(entry, tableVersion, CancellationToken.None); + + Task IMembershipTable.UpdateRow(MembershipEntry entry, string etag, TableVersion tableVersion) => + UpdateRowAsync(entry, etag, tableVersion, CancellationToken.None); + + Task IMembershipTable.UpdateIAmAlive(MembershipEntry entry) => UpdateIAmAliveAsync(entry, CancellationToken.None); +#else + public Task InitializeMembershipTable(bool tryInitTableVersion) => + InitializeMembershipTableAsync(tryInitTableVersion, CancellationToken.None); + + public Task DeleteMembershipTableEntries(string clusterId) => + DeleteMembershipTableEntriesAsync(clusterId, CancellationToken.None); + + public Task CleanupDefunctSiloEntries(DateTimeOffset beforeDate) => + CleanupDefunctSiloEntriesAsync(beforeDate, CancellationToken.None); + + public Task ReadRow(SiloAddress key) => ReadRowAsync(key, CancellationToken.None); + + public Task ReadAll() => ReadAllAsync(CancellationToken.None); + + public Task InsertRow(MembershipEntry entry, TableVersion tableVersion) => + InsertRowAsync(entry, tableVersion, CancellationToken.None); + + public Task UpdateRow(MembershipEntry entry, string etag, TableVersion tableVersion) => + UpdateRowAsync(entry, etag, tableVersion, CancellationToken.None); + + public Task UpdateIAmAlive(MembershipEntry entry) => UpdateIAmAliveAsync(entry, CancellationToken.None); +#endif + + public Task InitializeMembershipTableAsync(bool tryInitTableVersion, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Directory.CreateDirectory(configuration.MembershipDirectory); + return Access(static _ => true, write: true, cancellationToken); + } + + public Task DeleteMembershipTableEntriesAsync(string clusterId, CancellationToken cancellationToken) => Access(state => + { + if (!StringComparer.Ordinal.Equals(clusterId, configuration.ClusterId)) + { + throw new InvalidOperationException("The harness provider cannot delete another cluster."); + } + + state.Rows.Clear(); + state.Version++; + state.Etag = Guid.NewGuid().ToString("N"); + return true; + }, write: true, cancellationToken); + + public Task CleanupDefunctSiloEntriesAsync(DateTimeOffset beforeDate, CancellationToken cancellationToken) => Access(state => + { + foreach (var key in state.Rows.Where(pair => + pair.Value.Entry.Status != (int)SiloStatus.Active + && pair.Value.Entry.ToEntry().EffectiveUpdateTime < beforeDate.UtcDateTime).Select(pair => pair.Key).ToArray()) + { + state.Rows.Remove(key); + } + + return true; + }, write: true, cancellationToken); + + public async Task ReadRowAsync(SiloAddress key, CancellationToken cancellationToken) + { + var all = await ReadAllAsync(cancellationToken); + return new MembershipTableData(all.Members.Where(row => row.Item1.SiloAddress.Equals(key)).ToList(), all.Version); + } + + public async Task ReadAllAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var state = await Access(static state => state, write: false, cancellationToken); + return new MembershipTableData( + state.Rows.Values.Select(row => Tuple.Create(row.Entry.ToEntry(), row.Etag)).ToList(), + new TableVersion(state.Version, state.Etag)); + } + + public Task InsertRowAsync(MembershipEntry entry, TableVersion tableVersion, CancellationToken cancellationToken) => Access(state => + { + var key = entry.SiloAddress.ToParsableString(); + if (state.Rows.ContainsKey(key) || !StringComparer.Ordinal.Equals(tableVersion.VersionEtag, state.Etag)) + { + return false; + } + + state.Rows.Add(key, new Row(EntryData.From(entry), Guid.NewGuid().ToString("N"))); + state.Version = tableVersion.Version; + state.Etag = Guid.NewGuid().ToString("N"); + return true; + }, write: true, cancellationToken); + + public Task UpdateRowAsync(MembershipEntry entry, string etag, TableVersion tableVersion, CancellationToken cancellationToken) => Access(state => + { + var key = entry.SiloAddress.ToParsableString(); + if (!state.Rows.TryGetValue(key, out var row) + || !StringComparer.Ordinal.Equals(row.Etag, etag) + || !StringComparer.Ordinal.Equals(tableVersion.VersionEtag, state.Etag)) + { + return false; + } + + state.Rows[key] = new Row(EntryData.From(entry), Guid.NewGuid().ToString("N")); + state.Version = tableVersion.Version; + state.Etag = Guid.NewGuid().ToString("N"); + return true; + }, write: true, cancellationToken); + + public Task UpdateIAmAliveAsync(MembershipEntry entry, CancellationToken cancellationToken) => Access(state => + { + if (state.Rows.TryGetValue(entry.SiloAddress.ToParsableString(), out var row)) + { + state.Rows[entry.SiloAddress.ToParsableString()] = row with + { + Entry = row.Entry with { IAmAliveTime = entry.IAmAliveTime }, + }; + } + + return true; + }, write: true, cancellationToken); + + private async Task Access(Func action, bool write, CancellationToken cancellationToken) + { + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(10)); + FileStream lease; + while (true) + { + deadline.Token.ThrowIfCancellationRequested(); + try + { + lease = new FileStream(_lockPath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + break; + } + catch (IOException) + { + await Task.Delay(10, deadline.Token); + } + } + + await using (lease) + { + var state = File.Exists(_dataPath) + ? JsonSerializer.Deserialize(await File.ReadAllTextAsync(_dataPath, deadline.Token))! + : new TableState(); + deadline.Token.ThrowIfCancellationRequested(); + var result = action(state); + if (write) + { + // The sibling file is in the owned artifact directory, never the system temporary directory. + var staging = _dataPath + ".next"; + await File.WriteAllTextAsync(staging, JsonSerializer.Serialize(state), deadline.Token); + deadline.Token.ThrowIfCancellationRequested(); + File.Move(staging, _dataPath, overwrite: true); + } + + return result; + } + } + + private sealed class TableState + { + public int Version { get; set; } + public string Etag { get; set; } = "0"; + public Dictionary Rows { get; set; } = []; + } + + private sealed record Row(EntryData Entry, string Etag); + + internal sealed record SuspectVote(string Address, DateTime Time); + + internal sealed record EntryData( + string Address, + int Status, + int ProxyPort, + string HostName, + string SiloName, + string? RoleName, + int UpdateZone, + int FaultZone, + DateTime StartTime, + DateTime IAmAliveTime, + List? Suspects) + { + public static EntryData From(MembershipEntry entry) => new( + entry.SiloAddress.ToParsableString(), (int)entry.Status, entry.ProxyPort, + entry.HostName, entry.SiloName, entry.RoleName, entry.UpdateZone, entry.FaultZone, + entry.StartTime, entry.IAmAliveTime, + entry.SuspectTimes?.Select(pair => new SuspectVote(pair.Item1.ToParsableString(), pair.Item2)).ToList()); + + public MembershipEntry ToEntry() => new() + { + SiloAddress = SiloAddress.FromParsableString(Address), + Status = (SiloStatus)Status, + ProxyPort = ProxyPort, + HostName = HostName, + SiloName = SiloName, + RoleName = RoleName, + UpdateZone = UpdateZone, + FaultZone = FaultZone, + StartTime = StartTime, + IAmAliveTime = IAmAliveTime, + SuspectTimes = Suspects?.Select(vote => Tuple.Create(SiloAddress.FromParsableString(vote.Address), vote.Time)).ToList(), + }; + } +} diff --git a/test/Dissemination.PerformanceHarness/Silo/NewRuntime.cs b/test/Dissemination.PerformanceHarness/Silo/NewRuntime.cs new file mode 100644 index 00000000000..03412525df3 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Silo/NewRuntime.cs @@ -0,0 +1,51 @@ +#if NEW_RUNTIME +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Runtime.Dissemination; + +namespace Orleans.Dissemination.PerformanceHarness; + +internal static class NewRuntime +{ + public static void Configure(IServiceCollection services, NodeConfiguration configuration) + { + services.Configure(options => options.Enabled = configuration.Enabled); + services.Configure(options => options.Dissemination.Enabled = configuration.Enabled); + services.Configure(options => options.Dissemination.Enabled = configuration.Enabled); + } + + public static NodeSnapshot Decorate(IServiceProvider services, NodeSnapshot snapshot) + { + var options = services.GetRequiredService>().Value; + if (!options.Enabled) + { + return snapshot with + { + NamespaceEnabled = services.GetRequiredService>().Value.Dissemination.Enabled, + }; + } + + var ns = services.GetRequiredService(); + var topology = services.GetRequiredService().CurrentSnapshots.ActiveMembers; + var aggregationTree = ns.RoutingMode == DisseminationRoutingMode.AggregationTree; + var fanout = AggregationFanout.Read( + options.Overlay, aggregationTree, topology.Members.Length, () => options.Overlay.GetFanOutFactor(topology.Members.Length)); + return snapshot with + { + Enabled = options.Enabled, + NamespaceEnabled = ns.Options.Enabled, + UnconfirmedPeers = services.GetRequiredService() + .GetUnconfirmedPeers(ns).Select(address => address.ToParsableString()).Order(StringComparer.Ordinal).ToArray(), + OriginatorTargets = topology.GetOriginatorTargets(ns.RoutingMode).Select(address => address.ToParsableString()).ToArray(), + ForwardingTargets = topology.GetForwardingTargets(ns.RoutingMode).Select(address => address.ToParsableString()).ToArray(), + TopologyMembers = topology.Members.Select(address => address.ToParsableString()).ToArray(), + Fanout = fanout.Value, + FanoutSource = fanout.Source, + AggregationTree = aggregationTree, + AntiEntropyPeerCount = options.Overlay.AntiEntropyPeerCount, + AntiEntropyIntervalMilliseconds = options.Overlay.AntiEntropyInterval.TotalMilliseconds, + }; + } +} +#endif diff --git a/test/Dissemination.PerformanceHarness/Silo/Observation.cs b/test/Dissemination.PerformanceHarness/Silo/Observation.cs new file mode 100644 index 00000000000..ec8b2472355 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Silo/Observation.cs @@ -0,0 +1,316 @@ +using System.Buffers; +using System.Diagnostics.Metrics; +using System.Diagnostics.Tracing; +using System.IO.Pipelines; +using Microsoft.AspNetCore.Connections; + +namespace Orleans.Dissemination.PerformanceHarness; + +internal sealed class Observation : IDisposable +{ + private readonly object _lock = new(); + private readonly MeterListener _meters = new(); + private readonly Dictionary _metrics = []; + private readonly SocketCounters _sockets = new(); + private readonly object _connectionLock = new(); + private readonly Dictionary _connections = []; + private readonly HashSet _drainingConnections = []; + private Func? _blockedConnections; + private int _partitioned; + private long _bytesWritten; + private long _bytesRead; + + public Observation() + { + _meters.InstrumentPublished = (instrument, listener) => + { + if (instrument.Meter.Name.StartsWith("Microsoft.Orleans", StringComparison.Ordinal) + && (instrument.Name is "orleans-messaging-sent-messages-size" or "orleans-messaging-received-messages-size" + || instrument.Name.StartsWith("orleans-dissemination-", StringComparison.Ordinal))) + { + listener.EnableMeasurementEvents(instrument); + } + }; + _meters.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); + _meters.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); + _meters.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); + _meters.Start(); + } + + public bool Partitioned => Volatile.Read(ref _partitioned) != 0; + public long BytesWritten => Interlocked.Read(ref _bytesWritten); + public long BytesRead => Interlocked.Read(ref _bytesRead); + public (double Sent, double Received, long Samples) SocketSnapshot => _sockets.Snapshot(); + + public void Partition() + { + ConnectionLifetime[] connections; + lock (_connectionLock) + { + Volatile.Write(ref _partitioned, 1); + connections = CaptureConnectionsUnsafe(static _ => true); + } + + foreach (var connection in connections) + { + connection.Context.Abort(new ConnectionAbortedException("Harness partition")); + } + } + + public async Task DrainConnections( + Func matches, + Func close, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + ConnectionLifetime[] connections; + lock (_connectionLock) + { + _blockedConnections = matches; + connections = CaptureConnectionsUnsafe(matches); + } + + var closing = new List(connections.Length * 2); + foreach (var connection in connections) + { + if (!connection.Completion.Task.IsCompleted) + { + connection.Context.Abort(new ConnectionAbortedException("Harness partition")); + } + + var closeTask = close(connection.Context); + lock (_connectionLock) + { + connection.CloseTask = closeTask; + } + + closing.Add(closeTask); + closing.Add(connection.Completion.Task); + } + + var drained = Task.WhenAll(closing); + _ = drained.ContinueWith( + static task => _ = task.Exception, + CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + await drained.WaitAsync(cancellationToken); + } + + public void ResumeConnections() + { + lock (_connectionLock) + { + if (_drainingConnections.Any(static connection => + !connection.Completion.Task.IsCompleted || connection.CloseTask is not { IsCompletedSuccessfully: true })) + { + throw new InvalidOperationException("Partitioned connection middleware must drain before reopening."); + } + + _drainingConnections.Clear(); + _blockedConnections = null; + Volatile.Write(ref _partitioned, 0); + } + } + + private ConnectionLifetime[] CaptureConnectionsUnsafe(Func matches) + { + foreach (var connection in _connections.Values) + { + if (matches(connection.Context)) + { + _drainingConnections.Add(connection); + } + } + + return [.. _drainingConnections]; + } + + public async Task Connection(ConnectionContext context, ConnectionDelegate next) + { + ConnectionLifetime? lifetime = null; + lock (_connectionLock) + { + if (!Partitioned && _blockedConnections?.Invoke(context) != true) + { + lifetime = new(context); + _connections.Add(context, lifetime); + } + } + + if (lifetime is null) + { + context.Abort(new ConnectionAbortedException("Harness partition")); + return; + } + + var original = context.Transport; + context.Transport = new CountingDuplexPipe(original, this); + try + { + await next(context); + } + finally + { + context.Transport = original; + lock (_connectionLock) + { + _connections.Remove(context); + lifetime.Completion.SetResult(); + } + } + } + + private sealed class ConnectionLifetime(ConnectionContext context) + { + public ConnectionContext Context { get; } = context; + public TaskCompletionSource Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public Task? CloseTask { get; set; } + } + + public Dictionary Metrics() + { + _meters.RecordObservableInstruments(); + lock (_lock) + { + return new(_metrics); + } + } + + private void Record(Instrument instrument, double value, ReadOnlySpan> tags) + { + var tagValues = new string[tags.Length]; + for (var index = 0; index < tags.Length; index++) + { + tagValues[index] = $"{tags[index].Key}={tags[index].Value}"; + } + + Array.Sort(tagValues, StringComparer.Ordinal); + var key = $"{instrument.Name}|{string.Join(';', tagValues)}"; + lock (_lock) + { + var previous = _metrics.GetValueOrDefault(key, new(0, 0)); + _metrics[key] = instrument.IsObservable + ? new(previous.Count + 1, value) + : new(previous.Count + 1, previous.Sum + value); + } + } + + public void Dispose() + { + _meters.Dispose(); + _sockets.Dispose(); + } + + private sealed class CountingDuplexPipe(IDuplexPipe inner, Observation owner) : IDuplexPipe + { + public PipeReader Input { get; } = new CountingReader(inner.Input, owner); + public PipeWriter Output { get; } = new CountingWriter(inner.Output, owner); + } + + private sealed class CountingWriter(PipeWriter inner, Observation owner) : PipeWriter + { + public override void Advance(int bytes) + { + inner.Advance(bytes); + Interlocked.Add(ref owner._bytesWritten, bytes); + } + + public override Memory GetMemory(int sizeHint = 0) => inner.GetMemory(sizeHint); + public override Span GetSpan(int sizeHint = 0) => inner.GetSpan(sizeHint); + public override void CancelPendingFlush() => inner.CancelPendingFlush(); + public override void Complete(Exception? exception = null) => inner.Complete(exception); + public override ValueTask FlushAsync(CancellationToken cancellationToken = default) => inner.FlushAsync(cancellationToken); + } + + private sealed class CountingReader(PipeReader inner, Observation owner) : PipeReader + { + private ReadOnlySequence _buffer; + + public override void AdvanceTo(SequencePosition consumed) => AdvanceTo(consumed, consumed); + + public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) + { + Interlocked.Add(ref owner._bytesRead, _buffer.Slice(0, consumed).Length); + inner.AdvanceTo(consumed, examined); + _buffer = default; + } + + public override async ValueTask ReadAsync(CancellationToken cancellationToken = default) + { + var result = await inner.ReadAsync(cancellationToken); + _buffer = result.Buffer; + return result; + } + + public override bool TryRead(out ReadResult result) + { + if (!inner.TryRead(out result)) + { + return false; + } + + _buffer = result.Buffer; + return true; + } + + public override void CancelPendingRead() => inner.CancelPendingRead(); + public override void Complete(Exception? exception = null) => inner.Complete(exception); + } + + private sealed class SocketCounters : EventListener + { + private readonly object _lock = new(); + private double _sent; + private double _received; + private long _samples; + + protected override void OnEventSourceCreated(EventSource eventSource) + { + if (eventSource.Name == "System.Net.Sockets") + { + EnableEvents(eventSource, EventLevel.LogAlways, EventKeywords.All, + new Dictionary { ["EventCounterIntervalSec"] = "1" }); + } + } + + protected override void OnEventWritten(EventWrittenEventArgs eventData) + { + if (eventData.EventName != "EventCounters" + || eventData.Payload?.Count != 1 + || eventData.Payload[0] is not IDictionary counter + || !counter.TryGetValue("Name", out var name)) + { + return; + } + + var incremental = counter.TryGetValue("Increment", out var measurement); + if (!incremental && !counter.TryGetValue("Mean", out measurement)) + { + return; + } + + lock (_lock) + { + var value = Convert.ToDouble(measurement, System.Globalization.CultureInfo.InvariantCulture); + if ((string)name == "bytes-sent") + { + _sent = incremental ? _sent + value : value; + _samples++; + } + else if ((string)name == "bytes-received") + { + _received = incremental ? _received + value : value; + } + } + } + + public (double Sent, double Received, long Samples) Snapshot() + { + lock (_lock) + { + return (_sent, _received, _samples); + } + } + } +} diff --git a/test/Dissemination.PerformanceHarness/Silo/Program.cs b/test/Dissemination.PerformanceHarness/Silo/Program.cs new file mode 100644 index 00000000000..0b68c90a763 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Silo/Program.cs @@ -0,0 +1,350 @@ +using System.Diagnostics; +using System.Net; +using System.Reflection; +using System.Security.Cryptography; +using System.Text.Json; +using Microsoft.AspNetCore.Connections; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.MembershipService; +using Orleans.Runtime.Messaging; +using Orleans.Runtime.Scheduler; +using Orleans.TestingHost; + +namespace Orleans.Dissemination.PerformanceHarness; + +internal static class Program +{ + private static readonly MethodInfo PublishMethod = PublisherApi.GetPublishMethod(typeof(DeploymentLoadPublisher)); + private static readonly FieldInfo PublishTimerField = PublisherApi.GetTimerField(typeof(DeploymentLoadPublisher)); + + public static async Task Main(string[] args) + { + if (args is not ["--silo", var configurationPath]) + { + Console.Error.WriteLine("Expected --silo ."); + return 2; + } + + var configuration = JsonSerializer.Deserialize(await File.ReadAllTextAsync(configurationPath))!; +#if !NEW_RUNTIME + if (configuration.Enabled) + { + throw new InvalidOperationException("The pinned old runtime cannot be opted into dissemination."); + } +#endif + using var lifetime = new CancellationTokenSource(TimeSpan.FromMinutes(45)); + using var observation = new Observation(); + var openLoop = new OpenLoopProducer(TimeProvider.System); + var membership = new FileMembershipTable(configuration); + using var host = TestClusterHostFactory.CreateSiloHost( + configuration.Name, + new ConfigurationBuilder().Build(), + builder => builder.ConfigureServices(services => + { + services.AddSingleton(configuration); + services.AddSingleton(observation); + services.AddSingleton(openLoop); + services.AddSingleton(membership); + services.AddSingleton(membership); + services.AddSingleton(); + services.Configure(options => + { + options.ClusterId = configuration.ClusterId; + options.ServiceId = "dissemination-performance"; + }); + services.Configure(options => options.SiloName = configuration.Name); + services.Configure(options => + { + options.AdvertisedIPAddress = IPAddress.Loopback; + options.SiloPort = configuration.SiloPort; + options.GatewayPort = 0; + }); + services.Configure(options => + { + // Keep membership stable while the transport is partitioned. + options.LivenessEnabled = false; + options.TableRefreshTimeout = TimeSpan.FromHours(1); + options.IAmAliveTablePublishTimeout = TimeSpan.FromHours(1); + options.MaxJoinAttemptTime = TimeSpan.FromSeconds(60); + }); + services.Configure(options => + options.DeploymentLoadPublisherRefreshTime = TimeSpan.FromHours(1)); + services.Configure(options => options.ResponseTimeout = TimeSpan.FromSeconds(5)); + services.Configure(options => options.ShutdownTimeout = TimeSpan.FromSeconds(15)); + services.Configure(options => + { + options.ConfigureSiloInboundConnection(connection => connection.Use(next => context => observation.Connection(context, next))); + options.ConfigureSiloOutboundConnection(connection => connection.Use(next => context => observation.Connection(context, next))); + }); + services.AddLogging(logging => + { + logging.SetMinimumLevel(LogLevel.Warning).AddConsole(options => + options.LogToStandardErrorThreshold = LogLevel.Trace); + }); +#if NEW_RUNTIME + NewRuntime.Configure(services, configuration); +#endif + })); + + _ = host.Services.GetRequiredService(); + var monitor = MonitorParent(configuration.ParentProcessId, lifetime); + try + { + using var startup = CancellationTokenSource.CreateLinkedTokenSource(lifetime.Token); + startup.CancelAfter(TimeSpan.FromSeconds(90)); + await host.StartAsync(startup.Token); + var publisher = host.Services.GetRequiredService(); + // Commands own publication cadence; lifecycle shutdown retains its disposable timer reference. + await publisher.RunOrQueueTask(() => + { + ((IDisposable)PublishTimerField.GetValue(publisher)!).Dispose(); + return Task.CompletedTask; + }); + await Write(new Response(0, Snapshot(host.Services), null)); + + while (await Console.In.ReadLineAsync(lifetime.Token) is { } line) + { + var command = JsonSerializer.Deserialize(line)!; + using var operation = CancellationTokenSource.CreateLinkedTokenSource(lifetime.Token); + operation.CancelAfter(TimeSpan.FromSeconds(60)); + try + { + var result = await Execute(host.Services, command, operation.Token, lifetime.Token); + await Write(new Response(command.Id, result, null)); + if (command.Operation == "stop") + { + break; + } + } + catch (Exception exception) + { + await Write(new Response(command.Id, null, exception.ToString())); + } + } + + using var shutdown = new CancellationTokenSource(TimeSpan.FromSeconds(20)); + await openLoop.DisposeAsync(); + await host.StopAsync(shutdown.Token).WaitAsync(shutdown.Token); + // Drain the last one-second OS socket-counter bucket before recording shutdown cost. + await Task.Delay(TimeSpan.FromMilliseconds(1100)); + await Write(new Response(-1, Snapshot(host.Services, detailed: true), null)); + return 0; + } + finally + { + await lifetime.CancelAsync(); + await openLoop.DisposeAsync(); + await monitor; + } + } + + private static async Task Execute( + IServiceProvider services, Command command, CancellationToken cancellationToken, CancellationToken lifetimeToken) + { + var manager = services.GetRequiredService(); + var publisher = services.GetRequiredService(); + switch (command.Operation) + { + case "snapshot": + case "stop": + break; + case "measure": + return Snapshot(services, detailed: true); + case "refresh": + await manager.Refresh(null, cancellationToken); + break; + case "publish": + await publisher.RunOrQueueTask(() => (Task)PublishMethod.Invoke(publisher, [cancellationToken])!) + .WaitAsync(cancellationToken); + + break; + case "open-loop-arm": + services.GetRequiredService().Start( + command.OpenLoop ?? throw new InvalidOperationException("Missing open-loop plan."), + token => publisher.RunOrQueueTaskResult(async () => + { + token.ThrowIfCancellationRequested(); + var startedAt = DateTimeOffset.UtcNow; + await (Task)PublishMethod.Invoke(publisher, [token])!; + var local = services.GetRequiredService().SiloAddress; + var value = publisher.PeriodicStatistics[local]; + return new ProducedValue(startedAt, DateTimeOffset.UtcNow, value.DateTime.Ticks, StateComparison.Serialize(value)); + }).Unwrap(), lifetimeToken); + break; + case "open-loop-finish": + var report = await services.GetRequiredService().Completion.WaitAsync(cancellationToken); + return Snapshot(services) with { OpenLoopReport = report }; + case "partition": + services.GetRequiredService().Partition(); + break; + case "drain-connections": + await DrainConnections(services, command.Peer, cancellationToken); + break; + case "resume-connections": + services.GetRequiredService().ResumeConnections(); + break; + case "retained-memory": + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + break; + case "echo": + { + var peer = services.GetRequiredService().GetSystemTarget( + ControlTarget.TargetType, SiloAddress.FromParsableString(command.Peer!)); + var processId = await peer.Echo(cancellationToken).WaitAsync(cancellationToken); + return Snapshot(services) with { RemoteProcessId = processId }; + } + case "probe-echo": + { + var peer = services.GetRequiredService().GetSystemTarget( + ControlTarget.TargetType, SiloAddress.FromParsableString(command.Peer!)); + using var probeCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + try + { + var request = peer.Echo(probeCancellation.Token); + request.Ignore(); + var processId = await request.WaitAsync(TimeSpan.FromSeconds(2), cancellationToken); + return Snapshot(services) with { RemoteProcessId = processId }; + } + catch (Exception exception) when (exception is ConnectionFailedException + or SiloUnavailableException or OrleansMessageRejectionException or TimeoutException) + { + return Snapshot(services) with + { + ProbeError = $"{exception.GetType().FullName}: {exception.Message}", + }; + } + finally + { + await probeCancellation.CancelAsync(); + } + } + default: + throw new NotSupportedException(command.Operation); + } + + return Snapshot(services); + } + + private static Task DrainConnections(IServiceProvider services, string? peer, CancellationToken cancellationToken) + { + var address = peer is null ? null : SiloAddress.FromParsableString(peer); + return services.GetRequiredService().DrainConnections( + context => + { + var connection = (SiloConnection)context.Features.Get()!; + // Inbound handshakes have no remote silo identity until their preamble completes. + return address is null || connection.RemoteSiloAddress is null || connection.RemoteSiloAddress.Equals(address); + }, + context => context.Features.Get()!.CloseAsync(new ConnectionAbortedException("Harness partition")), + cancellationToken); + } + + internal static NodeSnapshot Snapshot(IServiceProvider services, bool detailed = false) + { + var manager = services.GetRequiredService(); + var publisher = services.GetRequiredService(); + var observation = services.GetRequiredService(); + var membership = manager.CurrentSnapshot; + using var process = Process.GetCurrentProcess(); + var socket = observation.SocketSnapshot; + var load = publisher.PeriodicStatistics.ToArray(); + var openLoop = services.GetRequiredService(); + var progress = openLoop.Progress; + var result = new NodeSnapshot + { + CapturedAtUtc = DateTimeOffset.UtcNow, + Identity = Identity(), + Address = services.GetRequiredService().SiloAddress.ToParsableString(), + MembershipVersion = membership.Version.Value, + Membership = new(membership.Entries.ToDictionary( + pair => pair.Key.ToParsableString(), + pair => JsonSerializer.Serialize(FileMembershipTable.EntryData.From(pair.Value))), StringComparer.Ordinal), + ActiveMembers = membership.Entries.Values.Where(entry => entry.Status == SiloStatus.Active) + .Select(entry => entry.SiloAddress.ToParsableString()).Order(StringComparer.Ordinal).ToArray(), + Load = new(load.ToDictionary( + pair => pair.Key.ToParsableString(), pair => StateComparison.Serialize(pair.Value)), StringComparer.Ordinal), + LoadVersions = new(load.ToDictionary( + pair => pair.Key.ToParsableString(), pair => pair.Value.DateTime.Ticks), StringComparer.Ordinal), + UnconfirmedPeers = [], + OriginatorTargets = [], + ForwardingTargets = [], + Metrics = detailed ? observation.Metrics() : [], + TransportBytesWritten = observation.BytesWritten, + TransportBytesRead = observation.BytesRead, + SocketBytesSent = socket.Sent, + SocketBytesReceived = socket.Received, + SocketCounterSamples = socket.Samples, + CpuMilliseconds = process.TotalProcessorTime.TotalMilliseconds, + AllocatedBytes = GC.GetTotalAllocatedBytes(precise: true), + ManagedHeapBytes = GC.GetTotalMemory(forceFullCollection: false), + WorkingSetBytes = process.WorkingSet64, + PrivateBytes = process.PrivateMemorySize64, + ProcessorCount = Environment.ProcessorCount, + ServerGC = System.Runtime.GCSettings.IsServerGC, + Partitioned = observation.Partitioned, + OpenLoopProgress = progress, + OpenLoopReport = progress?.Error is not null ? openLoop.Report : null, + }; +#if NEW_RUNTIME + result = NewRuntime.Decorate(services, result); +#endif + return result; + } + + private static BinaryIdentity Identity() + { + // Hash once: periodic snapshots must not read the runtime image repeatedly. + return CachedIdentity.Value; + } + + private static readonly Lazy CachedIdentity = new(() => + { + var assembly = typeof(Silo).Assembly; + var metadata = typeof(Program).Assembly.GetCustomAttributes() + .ToDictionary(attribute => attribute.Key, attribute => attribute.Value); + return new( + Environment.ProcessId, metadata["HarnessRuntime"]!, metadata["HarnessSourceRevision"]!, + assembly.GetName().Name!, assembly.GetName().Version!.ToString(), + assembly.GetCustomAttribute()!.InformationalVersion, + assembly.ManifestModule.ModuleVersionId.ToString(), + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(assembly.Location))), + assembly.Location, + assembly.GetType("Orleans.Runtime.Dissemination.DisseminationSystemTarget") is not null) + { + Assemblies = AppDomain.CurrentDomain.GetAssemblies() + .Where(candidate => !candidate.IsDynamic && candidate.GetName().Name!.StartsWith("Orleans", StringComparison.Ordinal)) + .ToDictionary(candidate => candidate.GetName().Name!, candidate => new AssemblyProof( + candidate.GetName().Version!.ToString(), + candidate.GetCustomAttribute()?.InformationalVersion ?? "", + candidate.ManifestModule.ModuleVersionId.ToString(), + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(candidate.Location))), + candidate.Location)), + }; + }); + + private static async Task MonitorParent(int parentProcessId, CancellationTokenSource lifetime) + { + try + { + using var parent = Process.GetProcessById(parentProcessId); + await parent.WaitForExitAsync(lifetime.Token); + await lifetime.CancelAsync(); + } + catch (ArgumentException) + { + await lifetime.CancelAsync(); + } + catch (OperationCanceledException) when (lifetime.IsCancellationRequested) + { + } + } + + private static Task Write(Response response) => Console.Out.WriteLineAsync("HARNESS " + JsonSerializer.Serialize(response)); +} diff --git a/test/Dissemination.PerformanceHarness/Tests/Dissemination.PerformanceHarness.Tests.csproj b/test/Dissemination.PerformanceHarness/Tests/Dissemination.PerformanceHarness.Tests.csproj new file mode 100644 index 00000000000..2f57d3b8d7a --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Tests/Dissemination.PerformanceHarness.Tests.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + + + + + + + + + + + + diff --git a/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.AggregationFanout.cs b/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.AggregationFanout.cs new file mode 100644 index 00000000000..7466d6ffd2a --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.AggregationFanout.cs @@ -0,0 +1,42 @@ +using Xunit; + +namespace Orleans.Dissemination.PerformanceHarness; + +public sealed partial class MeasurementTests +{ + [Theory] + [InlineData(0, 1)] + [InlineData(4, 4)] + [InlineData(32, 8)] + public void AggregationFanoutReportsTheCandidateRoutingClamp(int members, int expected) + { + var result = AggregationFanout.Read(new ModernOverlay(), true, members, + () => throw new InvalidOperationException("A modern aggregation tree must use its own fanout.")); + Assert.Equal(new(expected, "Overlay.AggregationFanOutFactor"), result); + } + + [Fact] + public void AggregationFanoutIdentifiesLegacyCapabilityAndPreservesMembershipRouting() + { + Assert.Equal(new(7, "LegacyAggregationMembershipFanout"), AggregationFanout.Read(new object(), true, 32, () => 7)); + Assert.Equal(new(5, "MembershipFanout"), AggregationFanout.Read(new ModernOverlay(), false, 32, () => 5)); + } + + [Fact] + public void AggregationFanoutRejectsIncompatibleCandidateCapability() + { + var exception = Assert.Throws(() => AggregationFanout.Read(new IncompatibleOverlay(), true, 32, () => 7)); + Assert.Contains("public int", exception.Message, StringComparison.Ordinal); + Assert.Contains("AggregationFanOutFactor", exception.Message, StringComparison.Ordinal); + } + + private sealed class ModernOverlay + { + public int AggregationFanOutFactor => 8; + } + + private sealed class IncompatibleOverlay + { + public string AggregationFanOutFactor => "8"; + } +} diff --git a/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.OpenLoop.cs b/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.OpenLoop.cs new file mode 100644 index 00000000000..a8be7cf825f --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.OpenLoop.cs @@ -0,0 +1,265 @@ +using System.Threading.Channels; +using Microsoft.Extensions.Time.Testing; +using Xunit; + +namespace Orleans.Dissemination.PerformanceHarness; + +public sealed partial class MeasurementTests +{ + [Theory] + [InlineData("OpenLoopSynchronized", 0)] + [InlineData("OpenLoopStaggered", 250)] + public async Task OpenLoopKeepsOneHertzDeadlinesIndependentOfPublicationDuration(string workload, int phaseMilliseconds) + { + var clock = new RecordingClock(); + var epoch = clock.GetUtcNow().AddSeconds(1); + var producer = new OpenLoopProducer(clock); + await using var cleanup = producer; + producer.Start(new(epoch, 3, 1, 4, workload), async token => + { + var start = clock.GetUtcNow(); + await Task.Delay(TimeSpan.FromMilliseconds(200), clock, token); + return new(start, clock.GetUtcNow(), start.UtcTicks, start.ToString("O")); + }, TestContext.Current.CancellationToken); + + clock.Advance(await clock.NextDelay()); + for (var index = 0; index < 3; index++) + { + var publicationDelay = await clock.NextDelay(); + Assert.Equal(TimeSpan.FromMilliseconds(200), publicationDelay); + clock.Advance(publicationDelay); + var nextDeadline = await clock.NextDelay(); + Assert.Equal(TimeSpan.FromMilliseconds(index < 2 ? 800 : 800 - phaseMilliseconds), nextDeadline); + clock.Advance(nextDeadline); + } + var result = await producer.Completion.WaitAsync(TestContext.Current.CancellationToken); + + Assert.Equal(3, result.Publications.Length); + Assert.Equal(0, result.MissedPeriods); + Assert.Equal(0, result.Overruns); + for (var index = 0; index < 3; index++) + { + var sample = result.Publications[index]; + Assert.Equal(epoch.AddSeconds(index).AddMilliseconds(phaseMilliseconds), sample.ScheduledAtUtc); + Assert.Equal(sample.ScheduledAtUtc, sample.Result.StartedAtUtc); + Assert.Equal(TimeSpan.FromMilliseconds(200), sample.Result.CompletedAtUtc - sample.Result.StartedAtUtc); + } + Assert.Equal(new OpenLoopProgress(3, true, 0, 0, null), producer.Progress); + } + + [Fact] + public void StaggeredScheduleEvenlyDistributesEveryProducerAcrossOneSecond() + { + for (var count = 3; count <= 32; count++) + { + var offsets = Enumerable.Range(0, count) + .Select(index => new OpenLoopPlan(default, 3, index, count, "OpenLoopStaggered").Offset(0)).ToArray(); + for (var index = 0; index < count; index++) + { + Assert.Equal(TimeSpan.TicksPerSecond * index / count, offsets[index].Ticks); + Assert.Equal(TimeSpan.Zero, new OpenLoopPlan(default, 3, index, count, "OpenLoopSynchronized").Offset(0)); + } + } + } + + [Theory] + [InlineData(2, 4, 0, "OpenLoopSynchronized")] + [InlineData(31, 4, 0, "OpenLoopSynchronized")] + [InlineData(3, 2, 0, "OpenLoopSynchronized")] + [InlineData(3, 33, 0, "OpenLoopSynchronized")] + [InlineData(3, 4, -1, "OpenLoopSynchronized")] + [InlineData(3, 4, 4, "OpenLoopSynchronized")] + [InlineData(3, 4, 0, "ClosedLoop")] + public void OpenLoopRejectsUnboundedOrInvalidPlans(int seconds, int count, int index, string workload) => + Assert.Throws(() => new OpenLoopPlan(default, seconds, index, count, workload).Validate()); + + [Fact] + public async Task OpenLoopReportsMissedPeriodsWithoutCatchUp() + { + var clock = new RecordingClock(); + var producer = new OpenLoopProducer(clock); + producer.Start(new(clock.GetUtcNow().AddSeconds(1), 3, 0, 3, "OpenLoopSynchronized"), + _ => throw new InvalidOperationException("A missed slot must not publish."), TestContext.Current.CancellationToken); + Assert.Equal(TimeSpan.FromSeconds(1), await clock.NextDelay()); + clock.Advance(TimeSpan.FromSeconds(2)); + var error = await Assert.ThrowsAsync(() => producer.Completion); + Assert.Contains("missed publication slot 0", error.Message, StringComparison.Ordinal); + Assert.Equal(1, producer.Progress!.MissedPeriods); + Assert.Equal(0, producer.Progress.PublishedCount); + Assert.Empty(producer.Report.Publications); + await Assert.ThrowsAsync(() => producer.DisposeAsync().AsTask()); + } + + [Fact] + public async Task OpenLoopReportsOverrunAndPreservesTheCompletedPublication() + { + var clock = new RecordingClock(); + var producer = new OpenLoopProducer(clock); + producer.Start(new(clock.GetUtcNow().AddSeconds(1), 3, 0, 3, "OpenLoopSynchronized"), async token => + { + var start = clock.GetUtcNow(); + await Task.Delay(TimeSpan.FromMilliseconds(1500), clock, token); + return new(start, clock.GetUtcNow(), start.UtcTicks, "overrun"); + }, TestContext.Current.CancellationToken); + clock.Advance(await clock.NextDelay()); + clock.Advance(await clock.NextDelay()); + var error = await Assert.ThrowsAsync(() => producer.Completion); + Assert.Contains("exceeded its 1 Hz slot", error.Message, StringComparison.Ordinal); + Assert.Equal(1, producer.Progress!.Overruns); + var sample = Assert.Single(producer.Report.Publications); + Assert.Equal(TimeSpan.FromMilliseconds(1500), sample.Result.CompletedAtUtc - sample.Result.StartedAtUtc); + await Assert.ThrowsAsync(() => producer.DisposeAsync().AsTask()); + } + + [Fact] + public async Task OpenLoopCancellationStopsArmedPublication() + { + var clock = new RecordingClock(); + var producer = new OpenLoopProducer(clock); + using var cancellation = new CancellationTokenSource(); + producer.Start(new(clock.GetUtcNow().AddSeconds(1), 3, 0, 3, "OpenLoopSynchronized"), + _ => throw new InvalidOperationException("Canceled work must not publish."), cancellation.Token); + await clock.NextDelay(); + await cancellation.CancelAsync(); + await Assert.ThrowsAnyAsync(() => producer.Completion); + Assert.Equal(0, producer.Progress!.PublishedCount); + await producer.DisposeAsync(); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(11)] + public void OpenLoopRejectsElapsedOrUnboundedStartEpoch(int leadSeconds) + { + var clock = new RecordingClock(); + var producer = new OpenLoopProducer(clock); + Assert.Throws(() => + producer.Start(new(clock.GetUtcNow().AddSeconds(leadSeconds), 3, 0, 3, "OpenLoopSynchronized"), + _ => throw new InvalidOperationException("Invalid plans must not publish."), TestContext.Current.CancellationToken)); + Assert.Null(producer.Progress); + } + + [Fact] + public async Task OpenLoopArmingIsSingleUse() + { + var clock = new RecordingClock(); + var producer = new OpenLoopProducer(clock); + var plan = new OpenLoopPlan(clock.GetUtcNow().AddSeconds(1), 3, 0, 3, "OpenLoopSynchronized"); + producer.Start(plan, _ => throw new InvalidOperationException("The producer is canceled before its start."), TestContext.Current.CancellationToken); + Assert.Throws(() => producer.Start(plan, _ => throw new InvalidOperationException(), TestContext.Current.CancellationToken)); + await producer.DisposeAsync(); + Assert.Equal(0, producer.Progress!.PublishedCount); + } + + [Fact] + public void OpenLoopReportsPollingUpperBoundsAndUnobservedIntermediateValues() + { + var reports = CreateOpenLoopReports(); + var measurements = CreateOpenLoopMeasurements(); + ObserveOpenLoop(measurements, reports, sequence: 2); + var result = measurements.Complete(reports); + + Assert.Equal(1, result.ProducerRateHz); + Assert.Equal(9, result.ObservedPublicationPairs); + Assert.Equal(18, result.UnobservedPublicationPairs); + Assert.Equal(6, result.FirstObservedAgeUpperBoundMilliseconds.Length); + Assert.Equal(new double[] { 102, 102, 102 }, result.FinalLatestLatencyUpperBoundMilliseconds); + Assert.Equal(6, result.ActualPublicationPeriodsMilliseconds.Length); + Assert.All(result.ActualPublicationPeriodsMilliseconds, value => Assert.Equal(1000, value)); + Assert.All(result.PublicationMilliseconds, value => Assert.Equal(10, value)); + Assert.Equal(0, result.MaximumStartLatenessMilliseconds); + } + + [Theory] + [InlineData("value")] + [InlineData("version")] + [InlineData("latest")] + [InlineData("process")] + [InlineData("overrun")] + public void OpenLoopRejectsMismatchedStateMissingLatestOrInvalidProducer(string mismatch) + { + var reports = CreateOpenLoopReports(); + var measurements = CreateOpenLoopMeasurements(); + ObserveOpenLoop(measurements, reports, mismatch == "latest" ? 1 : 2, mismatch); + if (mismatch == "process") + { + reports[0] = reports[0] with { ProcessId = 999 }; + } + else if (mismatch == "overrun") + { + reports[0] = reports[0] with { Report = reports[0].Report with { Overruns = 1 } }; + } + Assert.Throws(() => measurements.Complete(reports)); + } + + [Fact] + public void ScalingComparisonSeparatesOpenLoopLatencyAndRejectsMixedWorkloads() + { + var measurements = CreateOpenLoopMeasurements(); + ObserveOpenLoop(measurements, CreateOpenLoopReports(), 2); + var summary = measurements.Complete(CreateOpenLoopReports()); + var sample = new ScalingSample("CurrentDisabled", 3, 3, "stable", 0, 3, 9, + new("candidate", "same"), System.Text.Json.JsonSerializer.SerializeToElement(new { ProcessorCount = 4 }), + 90, 900, 9000, 90, 1000, [], "OpenLoopSynchronized", summary); + var enabled = sample with { RuntimePath = "CurrentEnabledSupported" }; + var pair = Assert.Single(ScalingComparison.Create(System.Text.Json.JsonSerializer.Serialize(new[] { sample, enabled })).Pairs); + Assert.Equal("OpenLoopSynchronized", pair.Workload); + Assert.Equal(102, pair.Disabled.MedianConvergenceMilliseconds); + Assert.Equal(10, pair.Disabled.RpcsPerPublication); + enabled = enabled with { Workload = "OpenLoopStaggered" }; + Assert.Throws(() => ScalingComparison.Create(System.Text.Json.JsonSerializer.Serialize(new[] { sample, enabled }))); + } + + private static OpenLoopNodeReport[] CreateOpenLoopReports() + { + var epoch = new DateTimeOffset(2030, 1, 1, 0, 0, 0, TimeSpan.Zero); + return Enumerable.Range(0, 3).Select(index => new OpenLoopNodeReport(10 + index, $"node-{index}", + new(new(epoch, 3, index, 3, "OpenLoopSynchronized"), + Enumerable.Range(0, 3).Select(sequence => + { + var start = epoch.AddSeconds(sequence); + return new OpenLoopPublication(sequence, start, new(start, start.AddMilliseconds(10), start.UtcTicks, $"{index}/{sequence}")); + }).ToArray(), 0, 0))).ToArray(); + } + + private static OpenLoopMeasurements CreateOpenLoopMeasurements() => + new(Enumerable.Range(0, 3).ToDictionary(index => 10 + index, index => $"node-{index}"), + Enumerable.Range(0, 3).ToDictionary(index => $"node-{index}", _ => 0L), 3); + + private static void ObserveOpenLoop(OpenLoopMeasurements measurements, OpenLoopNodeReport[] reports, int sequence, string? mismatch = null) + { + var values = reports.ToDictionary(report => report.Address, report => report.Report.Publications[sequence].Result); + var versions = values.ToDictionary(pair => pair.Key, pair => pair.Value.Version); + var load = values.ToDictionary(pair => pair.Key, pair => pair.Value.Value); + if (mismatch == "value") + { + load["node-0"] = "wrong"; + } + else if (mismatch == "version") + { + versions["node-0"]++; + } + for (var index = 0; index < 3; index++) + { + measurements.Observe(10 + index, values["node-0"].StartedAtUtc.AddMilliseconds(100 + index), versions, load); + } + } + + private sealed class RecordingClock : TimeProvider + { + private readonly FakeTimeProvider _inner = new(new DateTimeOffset(2030, 1, 1, 0, 0, 0, TimeSpan.Zero)); + private readonly Channel _delays = Channel.CreateUnbounded(); + public override DateTimeOffset GetUtcNow() => _inner.GetUtcNow(); + public override long GetTimestamp() => _inner.GetTimestamp(); + public override long TimestampFrequency => _inner.TimestampFrequency; + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + var timer = _inner.CreateTimer(callback, state, dueTime, period); + Assert.True(_delays.Writer.TryWrite(dueTime)); + return timer; + } + public ValueTask NextDelay() => _delays.Reader.ReadAsync(TestContext.Current.CancellationToken); + public void Advance(TimeSpan delta) => _inner.Advance(delta); + } +} diff --git a/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.PublisherApi.cs b/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.PublisherApi.cs new file mode 100644 index 00000000000..08a20654c3e --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.PublisherApi.cs @@ -0,0 +1,81 @@ +using Xunit; + +namespace Orleans.Dissemination.PerformanceHarness; + +public sealed partial class MeasurementTests +{ + [Theory] + [InlineData(typeof(object))] + [InlineData(typeof(WrongPublishParameters))] + [InlineData(typeof(WrongPublishResult))] + public void PublisherApiReportsMissingOrIncompatiblePublication(Type publisherType) + { + var error = Assert.Throws(() => PublisherApi.GetPublishMethod(publisherType)); + Assert.Contains(publisherType.FullName!, error.Message, StringComparison.Ordinal); + Assert.Contains("Task PublishStatistics(CancellationToken)", error.Message, StringComparison.Ordinal); + Assert.Contains("Update the performance harness runtime adapter", error.Message, StringComparison.Ordinal); + } + + [Theory] + [InlineData(typeof(object))] + [InlineData(typeof(WrongTimerType))] + public void PublisherApiReportsMissingOrIncompatibleTimer(Type publisherType) + { + var error = Assert.Throws(() => PublisherApi.GetTimerField(publisherType)); + Assert.Contains(publisherType.FullName!, error.Message, StringComparison.Ordinal); + Assert.Contains("IDisposable _publishTimer", error.Message, StringComparison.Ordinal); + Assert.Contains("Update the performance harness runtime adapter", error.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task PublisherApiPreservesPublicationTokenAndDisposedTimerReference() + { + var publisher = new CompatiblePublisher(); + using var cancellation = new CancellationTokenSource(); + var publishMethod = PublisherApi.GetPublishMethod(typeof(CompatiblePublisher)); + var timerField = PublisherApi.GetTimerField(typeof(CompatiblePublisher)); + var timer = Assert.IsType(timerField.GetValue(publisher)); + timer.Dispose(); + await (Task)publishMethod.Invoke(publisher, [cancellation.Token])!; + + Assert.True(timer.Disposed); + Assert.Same(timer, timerField.GetValue(publisher)); + Assert.Equal(cancellation.Token, publisher.LastToken); + } + + private sealed class CompatiblePublisher + { + private readonly IDisposable _publishTimer = new PublisherTimer(); + public CancellationToken LastToken { get; private set; } + + internal Task PublishStatistics(CancellationToken cancellationToken) + { + Assert.True(((PublisherTimer)_publishTimer).Disposed); + LastToken = cancellationToken; + return Task.CompletedTask; + } + } + + private sealed class PublisherTimer : IDisposable + { + public bool Disposed { get; private set; } + public void Dispose() => Disposed = true; + } + + private sealed class WrongPublishParameters + { + internal Task PublishStatistics() => Task.FromResult(this); + } + + private sealed class WrongPublishResult + { + internal ValueTask PublishStatistics(CancellationToken cancellationToken) => + new(Task.FromResult((this, cancellationToken))); + } + + private sealed class WrongTimerType + { + private readonly int _publishTimer = 1; + public int Timer => _publishTimer; + } +} diff --git a/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.cs b/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.cs new file mode 100644 index 00000000000..71c8f48b820 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Tests/MeasurementTests.cs @@ -0,0 +1,351 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.IO.Pipelines; +using System.Net; +using System.Net.Sockets; +using System.Text.Json; +using Microsoft.AspNetCore.Connections; +using Xunit; + +namespace Orleans.Dissemination.PerformanceHarness; + +public sealed partial class MeasurementTests +{ + [Theory] + [InlineData(2)] + [InlineData(3)] + public void PairedRepetitionsReverseEveryRuntimePath(int count) + { + var forward = Enumerable.Range(0, count).Select(order => ScalingTests.ExecutionIndex(0, order, count)).ToArray(); + var reverse = Enumerable.Range(0, count).Select(order => ScalingTests.ExecutionIndex(1, order, count)).ToArray(); + var repeated = Enumerable.Range(0, count).Select(order => ScalingTests.ExecutionIndex(2, order, count)).ToArray(); + Assert.Equal(Enumerable.Range(0, count), forward); + Assert.Equal(forward.Reverse(), reverse); + Assert.Equal(forward, repeated); + } + + [Fact] + public void ScalingComparisonNormalizesEqualOfferedWorkAndReportsLatencyDistribution() + { + var environment = JsonSerializer.SerializeToElement(new { ProcessorCount = 4, SiloProcessorCount = 1, GCConserveMemory = 9 }); + var disabled = new ScalingSample( + "CurrentDisabled", 100, 100, "stable", 0, 4, 400, + new("candidate", "same-binary"), environment, + 39600, 800000, 4000000, 2000, 1000000, [40, 10, 30, 20]); + var enabled = disabled with + { + RuntimePath = "CurrentEnabledSupported", + TotalRpcs = 4000, + SerializedBytesSent = 400000, + AllocatedBytes = 2000000, + CpuMilliseconds = 1000, + ConvergenceMilliseconds = [30, 10, 20, 10], + }; + + var result = ScalingComparison.Create(JsonSerializer.Serialize(new[] { enabled, disabled })); + + Assert.Equal(0, result.IncompletePairs); + var pair = Assert.Single(result.Pairs); + Assert.Equal(100, pair.Size); + Assert.Equal(400, pair.OfferedPublications); + Assert.Equal(99, pair.Disabled.RpcsPerPublication); + Assert.Equal(10, pair.Enabled.RpcsPerPublication); + Assert.Equal(2000, pair.Disabled.SerializedBytesPerPublication); + Assert.Equal(1000, pair.Enabled.SerializedBytesPerPublication); + Assert.Equal(10000, pair.Disabled.AllocatedBytesPerPublication); + Assert.Equal(5000, pair.Enabled.AllocatedBytesPerPublication); + Assert.Equal(5, pair.Disabled.CpuMillisecondsPerPublication); + Assert.Equal(2.5, pair.Enabled.CpuMillisecondsPerPublication); + Assert.Equal(25, pair.Disabled.MedianConvergenceMilliseconds); + Assert.Equal(15, pair.Enabled.MedianConvergenceMilliseconds); + Assert.Equal(40, pair.Disabled.P95ConvergenceMilliseconds); + Assert.Equal(30, pair.Enabled.P95ConvergenceMilliseconds); + Assert.Contains("99.00 -> 10.00", result.ToMarkdown(), StringComparison.Ordinal); + Assert.Contains("Enabled mode explicitly opts into the subsystem and both namespaces", result.ToMarkdown(), StringComparison.Ordinal); + Assert.Contains("production dissemination tuning defaults", result.ToMarkdown(), StringComparison.Ordinal); + } + + [Theory] + [InlineData("binary")] + [InlineData("workload")] + [InlineData("environment")] + [InlineData("silo-count")] + public void ScalingComparisonRejectsIncomparableMeasurements(string mismatch) + { + var sample = new ScalingSample( + "CurrentDisabled", 100, 100, "stable", 0, 3, 300, + new("candidate", "same-binary"), JsonSerializer.SerializeToElement(new { ProcessorCount = 4 }), + 100, 1000, 10000, 100, 1000, [1, 2, 3]); + var enabled = sample with { RuntimePath = "CurrentEnabledSupported" }; + enabled = mismatch switch + { + "binary" => enabled with { Runtime = new("different", "other-binary") }, + "workload" => enabled with { OfferedPublications = 301 }, + "environment" => enabled with { Environment = JsonSerializer.SerializeToElement(new { ProcessorCount = 1 }) }, + "silo-count" => enabled with { LiveSilos = 99 }, + _ => throw new InvalidOperationException(mismatch), + }; + + Assert.Throws(() => ScalingComparison.Create(JsonSerializer.Serialize(new[] { sample, enabled }))); + } + + [Fact] + public void ScalingComparisonReportsPartialPairsExplicitly() + { + var sample = new ScalingSample( + "CurrentDisabled", 8, 8, "stable", 0, 3, 24, + new("candidate", "same-binary"), JsonSerializer.SerializeToElement(new { ProcessorCount = 4 }), + 100, 1000, 10000, 100, 1000, [1, 2, 3]); + + var result = ScalingComparison.Create(JsonSerializer.Serialize(new[] { sample })); + + Assert.Empty(result.Pairs); + Assert.Equal(1, result.IncompletePairs); + } + + [Fact] + public void LoadStateComparisonRequiresExactInventoryAndValues() + { + var expected = new Dictionary + { + ["silo@2"] = "current", + ["peer@1"] = "peer-value", + }; + var actual = new Dictionary(expected); + Assert.True(ProcessCluster.MatchesExpectedLoad(actual, expected)); + + actual["silo@1"] = "retired"; + Assert.False(ProcessCluster.MatchesExpectedLoad(actual, expected)); + actual.Remove("silo@2"); + Assert.False(ProcessCluster.MatchesExpectedLoad(actual, expected)); + actual.Remove("silo@1"); + actual["silo@2"] = "older-value"; + Assert.False(ProcessCluster.MatchesExpectedLoad(actual, expected)); + actual["silo@2"] = "current"; + Assert.True(ProcessCluster.MatchesExpectedLoad(actual, expected)); + } + + [Fact] + public async Task ConnectionDrain_WaitsForInitializationAndCloseBeforeReopening() + { + using var observation = new Observation(); + await using var context = CreateConnectionContext("initializing"); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var closed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var middleware = observation.Connection(context, async _ => + { + entered.SetResult(); + await release.Task; + }); + await entered.Task.WaitAsync(TestContext.Current.CancellationToken); + observation.Partition(); + Assert.True(context.ConnectionClosed.IsCancellationRequested); + var closeCalls = 0; + var drain = observation.DrainConnections( + static _ => true, + connection => + { + Assert.Same(context, connection); + closeCalls++; + return closed.Task; + }, + TestContext.Current.CancellationToken); + try + { + Assert.Equal(1, closeCalls); + Assert.False(drain.IsCompleted); + Assert.Throws(observation.ResumeConnections); + await using var rejected = CreateConnectionContext("rejected"); + await observation.Connection(rejected, _ => throw new InvalidOperationException("A partitioned connection entered.")); + release.SetResult(); + await middleware; + Assert.False(drain.IsCompleted); + Assert.Throws(observation.ResumeConnections); + closed.SetResult(); + await drain.WaitAsync(TestContext.Current.CancellationToken); + observation.ResumeConnections(); + Assert.False(observation.Partitioned); + var resumed = false; + await using var reopened = CreateConnectionContext("reopened"); + await observation.Connection(reopened, _ => + { + resumed = true; + return Task.CompletedTask; + }); + Assert.True(resumed); + } + finally + { + release.TrySetResult(); + closed.TrySetResult(); + await middleware; + await drain; + } + } + + [Fact] + public async Task ConnectionDrain_CapturesUnidentifiedPeersAndPreservesHealthyConnections() + { + using var observation = new Observation(); + await using var unknown = CreateConnectionContext("unknown"); + await using var healthy = CreateConnectionContext("healthy"); + healthy.Items["peer"] = "healthy"; + var releaseUnknown = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseHealthy = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var unknownMiddleware = observation.Connection(unknown, _ => releaseUnknown.Task); + var healthyMiddleware = observation.Connection(healthy, _ => releaseHealthy.Task); + using var cancellation = new CancellationTokenSource(); + var closed = new List(); + var drain = observation.DrainConnections( + context => !context.Items.TryGetValue("peer", out var peer) || Equals(peer, "partitioned"), + context => + { + closed.Add(context.ConnectionId); + return Task.CompletedTask; + }, + cancellation.Token); + try + { + Assert.Equal(new[] { "unknown" }, closed); + Assert.True(unknown.ConnectionClosed.IsCancellationRequested); + Assert.False(healthy.ConnectionClosed.IsCancellationRequested); + Assert.False(healthyMiddleware.IsCompleted); + await cancellation.CancelAsync(); + await Assert.ThrowsAnyAsync(() => drain); + await using var rejected = CreateConnectionContext("new-unknown"); + await observation.Connection(rejected, _ => throw new InvalidOperationException("An unidentified connection entered.")); + releaseUnknown.SetResult(); + await unknownMiddleware; + observation.ResumeConnections(); + Assert.False(healthyMiddleware.IsCompleted); + } + finally + { + releaseUnknown.TrySetResult(); + releaseHealthy.TrySetResult(); + await Task.WhenAll(unknownMiddleware, healthyMiddleware); + } + } + + [Fact] + public void StateComparison_ObservesNestedPublicReadonlyFieldChanges() + { + var original = new { Version = 7, EnvironmentStatistics = new ReadonlyStatistics(20) }; + var changed = new { Version = 7, EnvironmentStatistics = new ReadonlyStatistics(35) }; + // This is the previous comparison's blind spot: properties alone produce identical JSON. + Assert.Equal(JsonSerializer.Serialize(original), JsonSerializer.Serialize(changed)); + var before = StateComparison.Serialize(original); + var after = StateComparison.Serialize(changed); + Assert.NotEqual(before, after); + using var beforeDocument = JsonDocument.Parse(before); + using var afterDocument = JsonDocument.Parse(after); + Assert.Equal(20, beforeDocument.RootElement.GetProperty("EnvironmentStatistics").GetProperty("RawCpuUsagePercentage").GetSingle()); + Assert.Equal(35, afterDocument.RootElement.GetProperty("EnvironmentStatistics").GetProperty("RawCpuUsagePercentage").GetSingle()); + } + + [Fact] + public void MessageObservation_KeepsRequestOneWayAndBytesDistinct() + { + using var observation = new Observation(); + using var meter = new Meter("Microsoft.Orleans.HarnessMeasurementTest"); + var histogram = meter.CreateHistogram("orleans-messaging-sent-messages-size"); + histogram.Record(97, new KeyValuePair("MessageDirection", "Request")); + histogram.Record(31, new KeyValuePair("MessageDirection", "OneWay")); + histogram.Record(17, new KeyValuePair("MessageDirection", "Response")); + var metrics = observation.Metrics(); + Assert.Equal(new MetricValue(1, 97), metrics["orleans-messaging-sent-messages-size|MessageDirection=Request"]); + Assert.Equal(new MetricValue(1, 31), metrics["orleans-messaging-sent-messages-size|MessageDirection=OneWay"]); + Assert.Equal(new MetricValue(1, 17), metrics["orleans-messaging-sent-messages-size|MessageDirection=Response"]); + } + + [Fact] + public async Task TransportObservation_CountsConsumedAndSubmittedBytes() + { + var cancellationToken = TestContext.Current.CancellationToken; + using var observation = new Observation(); + var input = new Pipe(); + var output = new Pipe(); + await using var context = new DefaultConnectionContext("instrumentation-test") + { + Transport = new TestPipe(input.Reader, output.Writer), + }; + var payload = new byte[] { 1, 2, 3, 4, 5 }; + await observation.Connection(context, async connection => + { + await input.Writer.WriteAsync(payload, cancellationToken); + var received = await connection.Transport.Input.ReadAsync(cancellationToken); + connection.Transport.Input.AdvanceTo(received.Buffer.End); + await connection.Transport.Output.WriteAsync(payload, cancellationToken); + }); + Assert.Equal(payload.Length, observation.BytesRead); + Assert.Equal(payload.Length, observation.BytesWritten); + await input.Reader.CompleteAsync(); + await input.Writer.CompleteAsync(); + await output.Reader.CompleteAsync(); + await output.Writer.CompleteAsync(); + } + + [Fact] + public async Task SocketObservation_ReportsActualLoopbackTransfer() + { + var cancellationToken = TestContext.Current.CancellationToken; + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + using var observation = new Observation(); + using var client = new TcpClient(); + var accepted = listener.AcceptTcpClientAsync(cancellationToken); + await client.ConnectAsync(IPAddress.Loopback, ((IPEndPoint)listener.LocalEndpoint).Port, cancellationToken); + using var server = await accepted; + await WaitFor(() => observation.SocketSnapshot.Samples > 0, cancellationToken); + var before = observation.SocketSnapshot; + var payload = new byte[4096]; + await client.GetStream().WriteAsync(payload, cancellationToken); + await server.GetStream().ReadExactlyAsync(payload, cancellationToken); + await WaitFor(() => + { + var snapshot = observation.SocketSnapshot; + return snapshot.Sent - before.Sent >= payload.Length && snapshot.Received - before.Received >= payload.Length; + }, cancellationToken); + } + + private static async Task WaitFor(Func predicate, CancellationToken cancellationToken) + { + var clock = Stopwatch.StartNew(); + while (!predicate()) + { + Assert.True(clock.Elapsed < TimeSpan.FromSeconds(8), "The OS socket EventCounters did not report the measured transfer."); + await Task.Delay(50, cancellationToken); + } + } + + private sealed record TestPipe(PipeReader Input, PipeWriter Output) : IDuplexPipe; + + private static DefaultConnectionContext CreateConnectionContext(string id) + { + var pipe = new Pipe(); + return new TestConnectionContext(id) { Transport = new TestPipe(pipe.Reader, pipe.Writer) }; + } + + // DefaultConnectionContext queues Cancel independently of Dispose. These tests own abort synchronously; + // separate barriers continue to control middleware and transport completion. + private sealed class TestConnectionContext : DefaultConnectionContext + { + private readonly CancellationTokenSource _closed = new(); + + public TestConnectionContext(string id) : base(id) => ConnectionClosed = _closed.Token; + + public override void Abort(ConnectionAbortedException abortReason) => _closed.Cancel(); + + public override ValueTask DisposeAsync() + { + _closed.Dispose(); + return base.DisposeAsync(); + } + } + + private readonly struct ReadonlyStatistics(float rawCpuUsagePercentage) + { + public readonly float RawCpuUsagePercentage = rawCpuUsagePercentage; + } +} diff --git a/test/Dissemination.PerformanceHarness/Tests/OpenLoopMeasurements.cs b/test/Dissemination.PerformanceHarness/Tests/OpenLoopMeasurements.cs new file mode 100644 index 00000000000..7801b571be3 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Tests/OpenLoopMeasurements.cs @@ -0,0 +1,105 @@ +namespace Orleans.Dissemination.PerformanceHarness; + +internal sealed record OpenLoopSummary( + int DurationSeconds, int ProducerRateHz, int ObservedPublicationPairs, int UnobservedPublicationPairs, + int MissedPeriods, int Overruns, double MaximumStartLatenessMilliseconds, + double[] ActualPublicationPeriodsMilliseconds, double[] PublicationMilliseconds, + double[] FirstObservedAgeUpperBoundMilliseconds, double[] FinalLatestLatencyUpperBoundMilliseconds); + +internal sealed record OpenLoopObservation(int ObserverProcessId, string Source, long Version, DateTimeOffset ObservedAtUtc, string Value); +internal sealed record OpenLoopNodeReport(int ProcessId, string Address, OpenLoopReport Report); + +internal sealed class OpenLoopMeasurements(Dictionary nodes, Dictionary initialVersions, int seconds) +{ + private readonly Dictionary<(int Observer, string Source, long Version), OpenLoopObservation> _observations = []; + + public OpenLoopObservation[] Observations => _observations.Values.ToArray(); + + public void Observe(int processId, DateTimeOffset capturedAtUtc, IReadOnlyDictionary versions, IReadOnlyDictionary load) + { + if (!nodes.ContainsKey(processId) || versions.Count != nodes.Count || load.Count != nodes.Count + || versions.Keys.Any(address => !initialVersions.ContainsKey(address))) + { + throw new InvalidOperationException("Open-loop observation has an unexpected process or load inventory."); + } + + foreach (var (address, version) in versions) + { + if (!load.TryGetValue(address, out var value)) + { + throw new InvalidOperationException($"Missing observed load value for {address}."); + } + if (version > initialVersions[address]) + { + _observations.TryAdd((processId, address, version), new(processId, address, version, capturedAtUtc, value)); + } + } + if (_observations.Count > nodes.Count * nodes.Count * seconds) + { + throw new InvalidOperationException("Observed more publications than the bounded open-loop workload offered."); + } + } + + public OpenLoopSummary Complete(OpenLoopNodeReport[] reports) + { + if (reports.Length != nodes.Count || reports.Select(report => report.ProcessId).Distinct().Count() != nodes.Count + || reports.Any(report => !nodes.TryGetValue(report.ProcessId, out var address) || address != report.Address + || report.Report.Publications.Length != seconds || report.Report.MissedPeriods != 0 || report.Report.Overruns != 0)) + { + throw new InvalidOperationException("Open-loop completion has missing producers, missed periods or overruns."); + } + + var publications = reports.SelectMany(report => report.Report.Publications + .Select(publication => (report.Address, Publication: publication))) + .ToDictionary(item => (item.Address, item.Publication.Result.Version), item => item.Publication); + var ages = new List(); + foreach (var observation in _observations.Values) + { + if (!publications.TryGetValue((observation.Source, observation.Version), out var publication) + || observation.Value != publication.Result.Value) + { + throw new InvalidOperationException($"Observed an unoffered or mismatched value from {observation.Source}."); + } + + var age = (observation.ObservedAtUtc - publication.Result.StartedAtUtc).TotalMilliseconds; + if (age < 0 || age > (seconds + 10) * 1000) + { + throw new InvalidOperationException("Open-loop observation timestamps exceed the bounded same-host latency window."); + } + if (nodes[observation.ObserverProcessId] != observation.Source) + { + ages.Add(age); + } + } + + var latestLatencies = new List(); + foreach (var report in reports) + { + var latest = report.Report.Publications[^1].Result; + var latencies = new List(); + foreach (var processId in nodes.Keys) + { + if (!_observations.TryGetValue((processId, report.Address, latest.Version), out var observation)) + { + throw new InvalidOperationException($"Process {processId} has not observed the latest value from {report.Address}."); + } + latencies.Add((observation.ObservedAtUtc - latest.StartedAtUtc).TotalMilliseconds); + } + latestLatencies.Add(latencies.Max()); + } + + var all = reports.SelectMany(report => report.Report.Publications).ToArray(); + var lateness = all.Select(value => (value.Result.StartedAtUtc - value.ScheduledAtUtc).TotalMilliseconds).ToArray(); + var durations = all.Select(value => (value.Result.CompletedAtUtc - value.Result.StartedAtUtc).TotalMilliseconds).ToArray(); + if (lateness.Any(value => value < 0 || value >= 1000) || durations.Any(value => value < 0 || value >= 1000)) + { + throw new InvalidOperationException("Actual open-loop publication timing exceeded its 1 Hz slot."); + } + + var periods = reports.SelectMany(report => report.Report.Publications.Skip(1) + .Zip(report.Report.Publications, (current, previous) => (current.Result.StartedAtUtc - previous.Result.StartedAtUtc).TotalMilliseconds)) + .ToArray(); + return new(seconds, 1, _observations.Count, nodes.Count * nodes.Count * seconds - _observations.Count, + 0, 0, lateness.Max(), periods, durations, ages.ToArray(), latestLatencies.ToArray()); + } +} diff --git a/test/Dissemination.PerformanceHarness/Tests/ProcessCluster.OpenLoop.cs b/test/Dissemination.PerformanceHarness/Tests/ProcessCluster.OpenLoop.cs new file mode 100644 index 00000000000..0748ef53b6e --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Tests/ProcessCluster.OpenLoop.cs @@ -0,0 +1,94 @@ +using Xunit; + +namespace Orleans.Dissemination.PerformanceHarness; + +internal sealed partial class ProcessCluster +{ + public async Task MeasureOpenLoop(string workload, int seconds) + { + var nodes = Active; + var owned = nodes.ToDictionary(node => node.Last.Identity.ProcessId, node => node.Last.Address); + var initialVersions = nodes.ToDictionary(node => node.Last.Address, node => node.Last.LoadVersions[node.Last.Address]); + var measurement = new OpenLoopMeasurements(owned, initialVersions, seconds); + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(5 + seconds + 10)); + var cancellationToken = deadline.Token; + var startUtc = DateTimeOffset.UtcNow.AddSeconds(5); + var plans = nodes.Select((_, index) => new OpenLoopPlan(startUtc, seconds, index, nodes.Length, workload)).ToArray(); + await Save("open-loop-plans.json", plans); + try + { + var armed = await Task.WhenAll(nodes.Select((node, index) => + node.Send(cancellationToken, "open-loop-arm", openLoop: plans[index]))); + Assert.True(DateTimeOffset.UtcNow < startUtc, "All open-loop workers must acknowledge arming before the common start epoch."); + Assert.All(armed, snapshot => + { + Assert.NotNull(snapshot.OpenLoopProgress); + Assert.Equal(0, snapshot.OpenLoopProgress.PublishedCount); + Assert.Null(snapshot.OpenLoopProgress.Error); + }); + + while (true) + { + var snapshots = await Sample(); + if (snapshots.All(snapshot => snapshot.OpenLoopProgress!.Completed)) + { + break; + } + await Task.Delay(100, cancellationToken); + } + + var completed = await Task.WhenAll(nodes.Select(node => node.Send(cancellationToken, "open-loop-finish"))); + var reports = completed.Select(snapshot => new OpenLoopNodeReport(snapshot.Identity.ProcessId, snapshot.Address, + snapshot.OpenLoopReport ?? throw new InvalidOperationException("Worker completion is missing its open-loop report."))).ToArray(); + await Save("open-loop-producers.json", reports); + var expected = reports.ToDictionary(report => report.Address, report => report.Report.Publications[^1].Result.Value); + while (true) + { + var snapshots = await Sample(); + if (snapshots.All(snapshot => MatchesExpectedLoad(snapshot.Load, expected))) + { + break; + } + await Task.Delay(100, cancellationToken); + } + + var result = measurement.Complete(reports); + await Save("open-loop-summary.json", result); + return result; + } + catch (Exception exception) + { + await Save("failure-state.json", new + { + Phase = "bounded open-loop production and exact latest-state convergence", + Error = exception.ToString(), + Plans = plans, + Nodes = nodes.Select(node => node.Last), + }); + throw; + } + finally + { + await Save("open-loop-observations.json", measurement.Observations); + } + + async Task Sample() + { + var snapshots = await Task.WhenAll(nodes.Select(node => node.Send(cancellationToken, "snapshot"))); + Assert.Equal(owned.Count, snapshots.Select(snapshot => snapshot.Identity.ProcessId).Distinct().Count()); + foreach (var snapshot in snapshots) + { + Assert.True(owned.TryGetValue(snapshot.Identity.ProcessId, out var address)); + Assert.Equal(address, snapshot.Address); + Assert.Equal(owned.Values.Order(StringComparer.Ordinal), snapshot.ActiveMembers); + Assert.NotNull(snapshot.OpenLoopProgress); + Assert.Null(snapshot.OpenLoopProgress.Error); + Assert.Equal(0, snapshot.OpenLoopProgress.MissedPeriods); + Assert.Equal(0, snapshot.OpenLoopProgress.Overruns); + measurement.Observe(snapshot.Identity.ProcessId, snapshot.CapturedAtUtc, snapshot.LoadVersions, snapshot.Load); + } + return snapshots; + } + } +} diff --git a/test/Dissemination.PerformanceHarness/Tests/ProcessCluster.cs b/test/Dissemination.PerformanceHarness/Tests/ProcessCluster.cs new file mode 100644 index 00000000000..2782457c54e --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Tests/ProcessCluster.cs @@ -0,0 +1,458 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Net; +using System.Net.Sockets; +using System.Text.Json; +using Xunit; + +namespace Orleans.Dissemination.PerformanceHarness; + +internal sealed record AssemblyManifest(string AssemblyVersion, string Sha256); +internal sealed record BuildManifest( + string Runtime, + string SourceRevision, + string AssemblyName, + string AssemblyVersion, + string Sha256, + Dictionary Assemblies); + +internal sealed partial class ProcessCluster : IAsyncDisposable +{ + public const string Baseline = "6739589254b746a8790cf53524e6abe372bb53d4"; + private readonly List _all = []; + private readonly string _membershipDirectory; + private int _sequence; + + public ProcessCluster(string scenario, int siloProcessorCount = 0, int gcConserveMemory = 0) + { + var artifactRoot = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_RESULTS") + ?? throw new InvalidOperationException("Run .github/scripts/dissemination-performance.ps1 first."); + Directory = Path.Combine(Path.GetFullPath(artifactRoot), $"{scenario}-{Guid.NewGuid():N}"); + System.IO.Directory.CreateDirectory(Directory); + _membershipDirectory = Path.Combine(Directory, "membership"); + System.IO.Directory.CreateDirectory(_membershipDirectory); + Assert.InRange(siloProcessorCount, 0, 64); + Assert.InRange(gcConserveMemory, 0, 9); + SiloProcessorCount = siloProcessorCount; + GCConserveMemory = gcConserveMemory; + } + + public string Directory { get; } + public int SiloProcessorCount { get; } + public int GCConserveMemory { get; } + public IReadOnlyList All => _all; + public SiloProcess[] Active => _all.Where(node => !node.Stopped).ToArray(); + public double LastPublicationMilliseconds { get; private set; } + public double LastConvergenceWaitMilliseconds { get; private set; } + + public async Task Start(string runtime, bool enabled, int? port = null) + { + var binaryDirectory = Path.GetFullPath(Environment.GetEnvironmentVariable($"ORLEANS_DISSEMINATION_{runtime.ToUpperInvariant()}") + ?? throw new InvalidOperationException($"Missing {runtime} published binary directory.")); + var manifest = JsonSerializer.Deserialize(await File.ReadAllTextAsync(Path.Combine(binaryDirectory, "manifest.json")))!; + Assert.Equal(runtime, manifest.Runtime); + if (runtime == "Old") + { + Assert.Equal(Baseline, manifest.SourceRevision); + } + else + { + Assert.Equal(Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_CANDIDATE_SHA"), manifest.SourceRevision); + Assert.NotEqual(Baseline, manifest.SourceRevision); + } + + var name = $"{runtime}-{_sequence++}-{(enabled ? "enabled" : "disabled")}"; + var node = new SiloProcess( + binaryDirectory, + new(name, Path.GetFileName(Directory), _membershipDirectory, port ?? AvailablePort(), Environment.ProcessId, + enabled, SiloProcessorCount, GCConserveMemory), + Directory, + manifest); + _all.Add(node); + await node.Start(); + Assert.DoesNotContain(_all.Where(other => !ReferenceEquals(other, node) && !other.Stopped), + other => other.Last.Identity.ProcessId == node.Last.Identity.ProcessId); + return node; + } + + public async Task Stabilize() + { + var expected = Active.Select(node => node.Last.Address).Order(StringComparer.Ordinal).ToArray(); + await Eventually("every process has the exact active membership", async () => + { + var snapshots = await Task.WhenAll(Active.Select(node => node.Send("refresh"))); + return snapshots.All(snapshot => snapshot.ActiveMembers.SequenceEqual(expected)); + }); + } + + public async Task AssertControlRpcs(bool allPairs = true) + { + var nodes = Active; + for (var index = 0; index < nodes.Length; index++) + { + var peers = allPairs ? nodes.Where(peer => !ReferenceEquals(peer, nodes[index])) : [nodes[(index + 1) % nodes.Length]]; + foreach (var peer in peers) + { + var result = await nodes[index].Send("echo", peer: peer.Last.Address); + Assert.Equal(peer.Last.Identity.ProcessId, result.RemoteProcessId); + Assert.NotEqual(nodes[index].Last.Identity.ProcessId, result.RemoteProcessId); + } + } + } + + public async Task PublishAndConverge() + { + var clock = Stopwatch.StartNew(); + var expected = new Dictionary(StringComparer.Ordinal); + foreach (var node in Active) + { + var previous = node.Last.LoadVersions.GetValueOrDefault(node.Last.Address); + var snapshot = await node.Send("publish"); + Assert.True(snapshot.LoadVersions[snapshot.Address] > previous, "The production publisher did not create a new sample."); + expected.Add(snapshot.Address, snapshot.Load[snapshot.Address]); + } + + var publicationMilliseconds = clock.Elapsed.TotalMilliseconds; + await Eventually("exact production load state at every active process", async () => + { + var snapshots = await Task.WhenAll(Active.Select(node => node.Send("snapshot"))); + return snapshots.All(snapshot => MatchesExpectedLoad(snapshot.Load, expected)); + }, expected: expected); + var elapsed = clock.Elapsed.TotalMilliseconds; + LastPublicationMilliseconds = publicationMilliseconds; + LastConvergenceWaitMilliseconds = elapsed - publicationMilliseconds; + return elapsed; + } + + internal static bool MatchesExpectedLoad( + IReadOnlyDictionary actual, + IReadOnlyDictionary expected) => + actual.Count == expected.Count + && expected.All(pair => actual.TryGetValue(pair.Key, out var value) + && StringComparer.Ordinal.Equals(pair.Value, value)); + + public async Task HealPartition(SiloProcess partitioned) + { + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(60)); + var cancellationToken = deadline.Token; + var phase = "retiring partitioned connections at both endpoints"; + try + { + var nodes = Active; + // Drain admitted middleware, including unfinished handshakes, before reopening any affected endpoint. + await Task.WhenAll(nodes.Select(node => node.Send( + cancellationToken, + "drain-connections", + peer: ReferenceEquals(node, partitioned) ? null : partitioned.Last.Address))); + await Task.WhenAll(nodes.Select(node => node.Send(cancellationToken, "resume-connections"))); + phase = "bidirectional control RPCs after transport healing"; + await Eventually(phase, async () => + { + foreach (var node in nodes) + { + foreach (var peer in nodes.Where(peer => !ReferenceEquals(peer, node))) + { + var result = await node.Send(cancellationToken, "probe-echo", peer: peer.Last.Address); + if (result.RemoteProcessId is not { } processId) + { + Assert.NotNull(result.ProbeError); + return false; + } + + Assert.Equal(peer.Last.Identity.ProcessId, processId); + Assert.NotEqual(node.Last.Identity.ProcessId, processId); + } + } + + return true; + }); + } + catch (OperationCanceledException) when (deadline.IsCancellationRequested && !TestContext.Current.CancellationToken.IsCancellationRequested) + { + await Save("failure-state.json", new { Phase = phase, Nodes = _all.Select(node => node.Last) }); + Assert.Fail($"Timed out during {phase}. Exact snapshots/logs: {Directory}"); + } + } + + public async Task Eventually(string phase, Func> assertion, TimeSpan? timeout = null, object? expected = null) + { + var clock = Stopwatch.StartNew(); + while (true) + { + if (await assertion()) + { + return; + } + + if (clock.Elapsed > (timeout ?? TimeSpan.FromSeconds(60))) + { + await Save("failure-state.json", new { Phase = phase, Expected = expected, Nodes = _all.Select(node => node.Last) }); + Assert.Fail($"Timed out during {phase}. Exact snapshots/logs: {Directory}"); + } + + await Task.Delay(100, TestContext.Current.CancellationToken); + } + } + + public Task Save(string filename, object value) => + File.WriteAllTextAsync(Path.Combine(Directory, filename), JsonSerializer.Serialize(value, new JsonSerializerOptions { WriteIndented = true })); + + public async ValueTask DisposeAsync() + { + List failures = []; + foreach (var node in _all.AsEnumerable().Reverse()) + { + try + { + await node.DisposeAsync(); + } + catch (Exception exception) + { + failures.Add(exception); + } + } + + await Save("final-processes.json", _all.Select(node => new { node.Configuration, node.Last, node.ShutdownMilliseconds, node.ForcedKill })); + if (failures.Count > 0) + { + throw new AggregateException("One or more silos exceeded shutdown bounds.", failures); + } + } + + private static int AvailablePort() + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + try + { + return ((IPEndPoint)listener.LocalEndpoint).Port; + } + finally + { + listener.Stop(); + } + } +} + +internal sealed class SiloProcess : IAsyncDisposable +{ + private readonly Process _process; + private readonly string _configurationPath; + private readonly string _logPath; + private readonly string _errorPath; + private readonly BuildManifest _manifest; + private readonly ConcurrentDictionary> _pending = new(); + private readonly SemaphoreSlim _commandLock = new(1); + private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously); + private Task _output = Task.CompletedTask; + private Task _error = Task.CompletedTask; + private int _commandId; + private bool _started; + private bool _disposed; + + public SiloProcess(string binaryDirectory, NodeConfiguration configuration, string artifacts, BuildManifest manifest) + { + Configuration = configuration; + _manifest = manifest; + _configurationPath = Path.Combine(artifacts, configuration.Name + ".configuration.json"); + _logPath = Path.Combine(artifacts, configuration.Name + ".stdout.jsonl"); + _errorPath = Path.Combine(artifacts, configuration.Name + ".stderr.log"); + _process = new Process + { + StartInfo = new ProcessStartInfo(Environment.GetEnvironmentVariable("DOTNET_HOST_PATH") ?? "dotnet") + { + WorkingDirectory = binaryDirectory, + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + ArgumentList = { Path.Combine(binaryDirectory, "Orleans.Runtime.Tests.dll"), "--silo", _configurationPath }, + }, + }; + _pending[0] = _ready; + if (configuration.SiloProcessorCount > 0) + { + _process.StartInfo.Environment["DOTNET_PROCESSOR_COUNT"] = + configuration.SiloProcessorCount.ToString(System.Globalization.CultureInfo.InvariantCulture); + } + + if (configuration.GCConserveMemory > 0) + { + _process.StartInfo.Environment["DOTNET_GCConserveMemory"] = + configuration.GCConserveMemory.ToString(System.Globalization.CultureInfo.InvariantCulture); + } + } + + public NodeConfiguration Configuration { get; } + public NodeSnapshot Last { get; private set; } = null!; + public NodeSnapshot Initial { get; private set; } = null!; + public bool Stopped { get; private set; } + public bool ForcedKill { get; private set; } + public double ShutdownMilliseconds { get; private set; } + + public async Task Start() + { + await File.WriteAllTextAsync(_configurationPath, JsonSerializer.Serialize(Configuration)); + Assert.True(_process.Start(), "Failed to launch silo OS process."); + _started = true; + _output = PumpOutput(); + _error = PumpErrors(); + var ready = await _ready.Task.WaitAsync(TimeSpan.FromSeconds(100), TestContext.Current.CancellationToken); + Assert.Null(ready.Error); + Last = ready.Snapshot!; + Initial = Last; + Assert.Equal(_process.Id, Last.Identity.ProcessId); + Assert.Equal(_manifest.Runtime, Last.Identity.Runtime); + Assert.Equal(_manifest.SourceRevision, Last.Identity.SourceRevision); + Assert.Equal(_manifest.AssemblyName, Last.Identity.AssemblyName); + Assert.Equal(_manifest.AssemblyVersion, Last.Identity.AssemblyVersion); + Assert.Equal(_manifest.Sha256, Last.Identity.Sha256, ignoreCase: true); + Assert.Contains(_manifest.SourceRevision, Last.Identity.InformationalVersion, StringComparison.OrdinalIgnoreCase); + Assert.Equal(_manifest.Runtime == "New", Last.Identity.HasDissemination); + Assert.Equal(Configuration.Enabled, Last.Enabled); + Assert.Equal(Configuration.Enabled, Last.NamespaceEnabled); + if (Configuration.SiloProcessorCount > 0) + { + Assert.Equal(Configuration.SiloProcessorCount, Last.ProcessorCount); + } + var actualDirectory = Path.GetDirectoryName(Path.GetFullPath(Last.Identity.AssemblyPath)); + Assert.Equal(Path.GetFullPath(_process.StartInfo.WorkingDirectory), actualDirectory); + foreach (var name in new[] { "Orleans.Runtime", "Orleans.Core", "Orleans.Core.Abstractions", "Orleans.Serialization" }) + { + Assert.Contains(name, Last.Identity.Assemblies.Keys); + } + + foreach (var (name, proof) in Last.Identity.Assemblies) + { + Assert.True(_manifest.Assemblies.TryGetValue(name, out var expected), $"Loaded unexpected assembly {name}."); + Assert.Equal(expected!.AssemblyVersion, proof.Version); + Assert.Equal(expected.Sha256, proof.Sha256, ignoreCase: true); + Assert.Equal(Path.GetFullPath(_process.StartInfo.WorkingDirectory), Path.GetDirectoryName(Path.GetFullPath(proof.Path))); + } + } + + public Task Send(string operation, string? peer = null) => + Send(TestContext.Current.CancellationToken, operation, peer); + + public async Task Send( + CancellationToken cancellationToken, string operation, string? peer = null, OpenLoopPlan? openLoop = null) + { + await _commandLock.WaitAsync(cancellationToken); + try + { + var command = new Command(Interlocked.Increment(ref _commandId), operation, peer, openLoop); + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _pending[command.Id] = completed; + await _process.StandardInput.WriteLineAsync(JsonSerializer.Serialize(command).AsMemory(), cancellationToken); + await _process.StandardInput.FlushAsync(cancellationToken); + var response = await completed.Task.WaitAsync(TimeSpan.FromSeconds(70), cancellationToken); + Assert.True(response.Error is null, $"{Configuration.Name}/{operation}: {response.Error}; {_errorPath}"); + Last = response.Snapshot!; + return Last; + } + finally + { + _commandLock.Release(); + } + } + + public async Task Stop() + { + if (Stopped || !_started) + { + return; + } + + var clock = Stopwatch.StartNew(); + try + { + if (!_process.HasExited) + { + // Cleanup has its own bound, independent of a cancelled xUnit test token. + await _process.StandardInput.WriteLineAsync(JsonSerializer.Serialize(new Command(-2, "stop"))); + await _process.StandardInput.FlushAsync(); + await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(35)); + } + + await Task.WhenAll(_output, _error).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, _process.ExitCode); + } + catch + { + if (!_process.HasExited) + { + ForcedKill = true; + _process.Kill(entireProcessTree: true); + await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(10)); + } + + throw; + } + finally + { + ShutdownMilliseconds = clock.Elapsed.TotalMilliseconds; + Stopped = true; + } + } + + private async Task PumpOutput() + { + await using var log = File.CreateText(_logPath); + while (await _process.StandardOutput.ReadLineAsync() is { } line) + { + await log.WriteLineAsync(line); + await log.FlushAsync(); + if (!line.StartsWith("HARNESS ", StringComparison.Ordinal)) + { + continue; + } + + var response = JsonSerializer.Deserialize(line["HARNESS ".Length..])!; + if (response.Snapshot is not null) + { + Last = response.Snapshot; + } + + if (_pending.TryRemove(response.Id, out var completed)) + { + completed.TrySetResult(response); + } + } + + foreach (var completion in _pending.Values) + { + completion.TrySetException(new InvalidOperationException($"Silo stdout closed; inspect {_errorPath}.")); + } + } + + private async Task PumpErrors() + { + await using var log = File.CreateText(_errorPath); + while (await _process.StandardError.ReadLineAsync() is { } line) + { + await log.WriteLineAsync(line); + await log.FlushAsync(); + } + } + + public async ValueTask DisposeAsync() + { + if (_disposed) + { + return; + } + + _disposed = true; + try + { + await Stop(); + } + finally + { + _process.Dispose(); + _commandLock.Dispose(); + } + } +} diff --git a/test/Dissemination.PerformanceHarness/Tests/ScalingComparison.cs b/test/Dissemination.PerformanceHarness/Tests/ScalingComparison.cs new file mode 100644 index 00000000000..16027824412 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Tests/ScalingComparison.cs @@ -0,0 +1,145 @@ +using System.Globalization; +using System.Text; +using System.Text.Json; + +namespace Orleans.Dissemination.PerformanceHarness; + +internal sealed record ScalingComparison(ScalingPair[] Pairs, int IncompletePairs) +{ + public static ScalingComparison Create(string json) + { + var samples = JsonSerializer.Deserialize(json) + ?? throw new InvalidOperationException("Missing scaling samples."); + var pairs = new List(); + var incomplete = 0; + foreach (var group in samples + .Where(sample => sample.RuntimePath is "CurrentDisabled" or "CurrentEnabledSupported") + .GroupBy(sample => (sample.Size, sample.Scenario, sample.Repetition))) + { + if (group.Count() == 1) + { + incomplete++; + continue; + } + + var disabled = group.Single(sample => sample.RuntimePath == "CurrentDisabled"); + var enabled = group.Single(sample => sample.RuntimePath == "CurrentEnabledSupported"); + Validate(disabled); + Validate(enabled); + if (disabled.Iterations != enabled.Iterations + || disabled.Workload != enabled.Workload + || disabled.OfferedPublications != enabled.OfferedPublications + || disabled.Runtime.SourceRevision != enabled.Runtime.SourceRevision + || disabled.Runtime.Sha256 != enabled.Runtime.Sha256 + || disabled.Environment.GetRawText() != enabled.Environment.GetRawText()) + { + throw new InvalidOperationException($"Mismatched binary, workload, or environment for {group.Key}."); + } + + pairs.Add(new( + disabled.Size, disabled.Scenario, disabled.Repetition, disabled.Iterations, disabled.OfferedPublications, + disabled.Workload, + disabled.Runtime.SourceRevision, disabled.Runtime.Sha256, disabled.Environment, + ScalingCost.From(disabled), ScalingCost.From(enabled))); + } + + return new(pairs.ToArray(), incomplete); + } + + private static void Validate(ScalingSample sample) + { + if (sample.Size < 3 || sample.LiveSilos != sample.Size || sample.OfferedPublications <= 0 + || sample.Runtime is null || string.IsNullOrEmpty(sample.Runtime.SourceRevision) || string.IsNullOrEmpty(sample.Runtime.Sha256) + || sample.Environment.ValueKind != JsonValueKind.Object + || sample.Iterations < 3 || sample.ConvergenceMilliseconds is null + || sample.TotalRpcs <= 0 || sample.SerializedBytesSent <= 0 || sample.AllocatedBytes <= 0 + || sample.CpuMilliseconds < 0 || sample.ConvergenceMilliseconds.Any(value => !double.IsFinite(value) || value < 0)) + { + throw new InvalidOperationException($"Incomplete scaling measurement for {sample.RuntimePath}, {sample.Size} silos."); + } + if (sample.Workload == "ClosedLoop") + { + if (sample.ConvergenceMilliseconds.Length != sample.Iterations || sample.OpenLoop is not null) + { + throw new InvalidOperationException("Closed-loop measurements require one convergence latency per round."); + } + } + else if (sample.Workload is "OpenLoopSynchronized" or "OpenLoopStaggered" + && sample.OpenLoop is { } openLoop) + { + if (sample.Scenario != "stable" || sample.Size > 32 || sample.Iterations > 30 + || sample.OfferedPublications != sample.Size * sample.Iterations + || sample.ConvergenceMilliseconds.Length != 0 || openLoop.ProducerRateHz != 1 + || openLoop.DurationSeconds != sample.Iterations || openLoop.MissedPeriods != 0 || openLoop.Overruns != 0 + || openLoop.FinalLatestLatencyUpperBoundMilliseconds.Length != sample.Size + || openLoop.FirstObservedAgeUpperBoundMilliseconds.Length == 0 + || openLoop.PublicationMilliseconds.Length != sample.OfferedPublications + || openLoop.ActualPublicationPeriodsMilliseconds.Length != sample.Size * (sample.Iterations - 1) + || openLoop.ObservedPublicationPairs < sample.Size * sample.Size || openLoop.UnobservedPublicationPairs < 0 + || openLoop.ObservedPublicationPairs + openLoop.UnobservedPublicationPairs != sample.Size * sample.OfferedPublications + || !double.IsFinite(openLoop.MaximumStartLatenessMilliseconds) + || openLoop.MaximumStartLatenessMilliseconds is < 0 or >= 1000 + || openLoop.FinalLatestLatencyUpperBoundMilliseconds.Concat(openLoop.FirstObservedAgeUpperBoundMilliseconds) + .Any(value => !double.IsFinite(value) || value < 0 || value > (sample.Iterations + 10) * 1000)) + { + throw new InvalidOperationException("Incomplete or out-of-bounds 1 Hz open-loop measurement."); + } + } + else + { + throw new InvalidOperationException($"Missing or unsupported workload measurement: {sample.Workload}."); + } + } + + public string ToMarkdown() + { + var result = new StringBuilder(); + result.AppendLine("### Dissemination disabled vs enabled"); + result.AppendLine("Enabled mode explicitly opts into the subsystem and both namespaces, retaining the candidate's production dissemination tuning defaults."); + result.AppendLine("Each cell is disabled -> enabled. Costs per offered publication include background and control traffic. Single-host, resource-shared measurements; raw totals, environment and per-node data are in the artifacts."); + result.AppendLine("ClosedLoop latency is per-round convergence. OpenLoop latency is the polling upper bound for each producer's final value reaching all processes; cadence, observed intermediate values and their age upper bounds are in the raw open-loop artifacts."); + result.AppendLine(); + result.AppendLine("| Silos / scenario / workload / repetition | RPCs / publication | KiB / publication | Allocated KiB / publication | CPU ms / publication | Median ms | P95 ms |"); + result.AppendLine("|---|---:|---:|---:|---:|---:|---:|"); + foreach (var pair in Pairs) + { + result.AppendLine(FormattableString.Invariant( + $"| {pair.Size} / {pair.Scenario} / {pair.Workload} / {pair.Repetition} | {pair.Disabled.RpcsPerPublication:F2} -> {pair.Enabled.RpcsPerPublication:F2} | {pair.Disabled.SerializedBytesPerPublication / 1024:F2} -> {pair.Enabled.SerializedBytesPerPublication / 1024:F2} | {pair.Disabled.AllocatedBytesPerPublication / 1024:F2} -> {pair.Enabled.AllocatedBytesPerPublication / 1024:F2} | {pair.Disabled.CpuMillisecondsPerPublication:F2} -> {pair.Enabled.CpuMillisecondsPerPublication:F2} | {pair.Disabled.MedianConvergenceMilliseconds:F2} -> {pair.Enabled.MedianConvergenceMilliseconds:F2} | {pair.Disabled.P95ConvergenceMilliseconds:F2} -> {pair.Enabled.P95ConvergenceMilliseconds:F2} |")); + } + + result.AppendLine(); + result.AppendLine($"Incomplete pairs: {IncompletePairs.ToString(CultureInfo.InvariantCulture)}."); + return result.ToString(); + } +} + +internal sealed record ScalingSample( + string RuntimePath, int Size, int LiveSilos, string Scenario, int Repetition, int Iterations, int OfferedPublications, + ScalingBinary Runtime, JsonElement Environment, long TotalRpcs, double SerializedBytesSent, + long AllocatedBytes, double CpuMilliseconds, long RetainedManagedBytes, double[] ConvergenceMilliseconds, + string Workload = "ClosedLoop", OpenLoopSummary? OpenLoop = null); + +internal sealed record ScalingBinary(string SourceRevision, string Sha256); + +internal sealed record ScalingPair( + int Size, string Scenario, int Repetition, int Iterations, int OfferedPublications, + string Workload, + string SourceRevision, string Sha256, JsonElement Environment, ScalingCost Disabled, ScalingCost Enabled); + +internal sealed record ScalingCost( + long TotalRpcs, double SerializedBytesSent, long AllocatedBytes, double CpuMilliseconds, long RetainedManagedBytes, + double RpcsPerPublication, double SerializedBytesPerPublication, double AllocatedBytesPerPublication, + double CpuMillisecondsPerPublication, double MedianConvergenceMilliseconds, double P95ConvergenceMilliseconds) +{ + public static ScalingCost From(ScalingSample sample) + { + var ordered = (sample.Workload == "ClosedLoop" ? sample.ConvergenceMilliseconds + : sample.OpenLoop!.FinalLatestLatencyUpperBoundMilliseconds).Order().ToArray(); + var median = (ordered[(ordered.Length - 1) / 2] + ordered[ordered.Length / 2]) / 2; + return new( + sample.TotalRpcs, sample.SerializedBytesSent, sample.AllocatedBytes, sample.CpuMilliseconds, sample.RetainedManagedBytes, + (double)sample.TotalRpcs / sample.OfferedPublications, sample.SerializedBytesSent / sample.OfferedPublications, + (double)sample.AllocatedBytes / sample.OfferedPublications, sample.CpuMilliseconds / sample.OfferedPublications, + median, ordered[(int)Math.Ceiling(0.95 * ordered.Length) - 1]); + } +} diff --git a/test/Dissemination.PerformanceHarness/Tests/ScalingTests.cs b/test/Dissemination.PerformanceHarness/Tests/ScalingTests.cs new file mode 100644 index 00000000000..f653401dae3 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/Tests/ScalingTests.cs @@ -0,0 +1,416 @@ +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Text.Json; +using Xunit; + +namespace Orleans.Dissemination.PerformanceHarness; + +public sealed class ScalingTests +{ + [Fact] + [Trait("Category", "DisseminationScale")] + public async Task Scaling_SelectedRuntimePathsConverge() + { + Assert.Equal("true", Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_MANUAL_RUN")); + var sizes = (Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_SIZES") ?? "4,8") + .Split(',').Select(value => int.Parse(value, System.Globalization.CultureInfo.InvariantCulture)).ToArray(); + var iterations = Setting("ITERATIONS", 3, 3, 200); + var repetitions = Setting("REPETITIONS", 2, 1, 5); + Assert.All(sizes, size => Assert.InRange(size, 3, 128)); + Assert.InRange(sizes.Length, 1, 6); + Assert.Equal(sizes.Length, sizes.Distinct().Count()); + var scenarios = (Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_SCENARIOS") ?? "stable,churn,partition").Split(','); + Assert.InRange(scenarios.Length, 1, 3); + Assert.Equal(scenarios.Length, scenarios.Distinct().Count()); + Assert.All(scenarios, scenario => Assert.Contains(scenario, new[] { "stable", "churn", "partition" })); + var paths = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_RUNTIME_PATHS") ?? "All"; + Assert.Contains(paths, new[] { "All", "Current" }); + var workload = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_WORKLOAD") ?? "ClosedLoop"; + Assert.Contains(workload, new[] { "ClosedLoop", "OpenLoopSynchronized", "OpenLoopStaggered" }); + if (workload != "ClosedLoop") + { + Assert.Equal(new[] { "stable" }, scenarios); + Assert.InRange(iterations, 3, 30); + Assert.All(sizes, size => Assert.InRange(size, 3, 32)); + } + + var records = new List(); + var output = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_RESULTS")!; + RuntimeVariant[] variants = + [ + new("OriginalLegacy", "Old", Enabled: false), + new("CurrentDisabled", "New", Enabled: false), + new("CurrentEnabledSupported", "New", Enabled: true), + ]; + if (paths == "Current") + { + variants = variants[1..]; + } + + foreach (var size in sizes) + { + foreach (var scenario in scenarios) + { + for (var repetition = 0; repetition < repetitions; repetition++) + { + // Reverse the whole sequence so every off/on pair changes order. + for (var order = 0; order < variants.Length; order++) + { + var variant = variants[ExecutionIndex(repetition, order, variants.Length)]; + var record = await Measure(size, scenario, variant, iterations, repetition, order, workload); + records.Add(record); + var json = JsonSerializer.Serialize(records, new JsonSerializerOptions { WriteIndented = true }); + await File.WriteAllTextAsync(Path.Combine(output, "cost-results.json"), json, TestContext.Current.CancellationToken); + await File.WriteAllTextAsync(Path.Combine(output, "comparisons.json"), + JsonSerializer.Serialize(ScalingComparison.Create(json), new JsonSerializerOptions { WriteIndented = true }), + TestContext.Current.CancellationToken); + } + } + } + } + + var report = ScalingComparison.Create(JsonSerializer.Serialize(records)); + Assert.Equal(0, report.IncompletePairs); + Assert.Equal(sizes.Length * scenarios.Length * repetitions, report.Pairs.Length); + if (Environment.GetEnvironmentVariable("GITHUB_STEP_SUMMARY") is { Length: > 0 } summary) + { + await File.AppendAllTextAsync(summary, report.ToMarkdown(), TestContext.Current.CancellationToken); + } + } + + private static async Task Measure( + int size, string scenario, RuntimeVariant variant, int iterations, int repetition, int order, string workload) + { + var enabled = variant.Enabled; + var siloProcessorCount = Setting("SILO_PROCESSOR_COUNT", 0, 0, 64); + var gcConserveMemory = Setting("GC_CONSERVE_MEMORY", 0, 0, 9); + var bootstrap = Stopwatch.StartNew(); + await using var cluster = new ProcessCluster( + $"scale-{size}-{scenario}-{workload}-{variant.Name}-{repetition}", siloProcessorCount, gcConserveMemory); + for (var index = 0; index < size; index++) + { + await cluster.Start(variant.Runtime, enabled); + } + + await cluster.Stabilize(); + await cluster.AssertControlRpcs(allPairs: false); + await cluster.PublishAndConverge(); + await cluster.PublishAndConverge(); + if (enabled) + { + await ConfirmSupport(cluster); + } + + await cluster.Save("warmup.json", await Task.WhenAll(cluster.Active.Select(node => node.Send("measure")))); + // Socket EventCounters arrive in one-second buckets. Every path has the same two-second margins. + await Task.Delay(TimeSpan.FromSeconds(2), TestContext.Current.CancellationToken); + var start = (await Task.WhenAll(cluster.Active.Select(node => node.Send("measure")))) + .ToDictionary(snapshot => snapshot.Identity.ProcessId); + Assert.Equal(size, start.Count); + Assert.All(start.Values, snapshot => Assert.Equal(size, snapshot.ActiveMembers.Length)); + var bootstrapMilliseconds = bootstrap.Elapsed.TotalMilliseconds; + var startedAt = DateTimeOffset.UtcNow; + var clock = Stopwatch.StartNew(); + var latencies = new List(); + var publicationLatencies = new List(); + var convergenceWaits = new List(); + var recoveryLatencies = new List(); + var offeredPublications = 0; + var restarts = 0; + var partitions = 0; + OpenLoopSummary? openLoop = null; + if (workload != "ClosedLoop") + { + openLoop = await cluster.MeasureOpenLoop(workload, iterations); + offeredPublications = size * iterations; + } + for (var iteration = 0; workload == "ClosedLoop" && iteration < iterations; iteration++) + { + if (scenario == "churn" && iteration > 0 && iteration % 2 == 0) + { + var victim = cluster.Active[iteration % size]; + await victim.Stop(); + // Reuse the endpoint to exercise a real new silo incarnation and stale-generation cleanup. + await cluster.Start(variant.Runtime, enabled, victim.Configuration.SiloPort); + await cluster.Stabilize(); + if (enabled) + { + // Bootstrap/fallback work stays inside the churn window, but offered rounds use supported peers. + await ConfirmSupport(cluster); + } + + restarts++; + } + else if (scenario == "partition" && iteration % 3 == 1) + { + var victim = cluster.Active[iteration % size]; + await victim.Send("partition"); + partitions++; + foreach (var source in cluster.Active.Where(node => !ReferenceEquals(node, victim))) + { + await source.Send("publish"); + offeredPublications++; + } + + var recovery = Stopwatch.StartNew(); + // Establish delivery paths before offering the next sample, including on the one-way legacy path. + await cluster.HealPartition(victim); + await PublishRound(); + recoveryLatencies.Add(recovery.Elapsed.TotalMilliseconds); + continue; + } + + await PublishRound(); + } + + async Task PublishRound() + { + latencies.Add(await cluster.PublishAndConverge()); + publicationLatencies.Add(cluster.LastPublicationMilliseconds); + convergenceWaits.Add(cluster.LastConvergenceWaitMilliseconds); + offeredPublications += size; + } + + await Task.Delay(TimeSpan.FromSeconds(2), TestContext.Current.CancellationToken); + await Task.WhenAll(cluster.Active.Select(node => node.Send("measure"))); + var elapsed = clock.Elapsed.TotalMilliseconds; + var finishedAt = DateTimeOffset.UtcNow; + var finish = cluster.All.Select(node => node.Last).ToArray(); + var retained = await Task.WhenAll(cluster.Active.Select(node => node.Send("retained-memory"))); + var retainedByPid = retained.ToDictionary(snapshot => snapshot.Identity.ProcessId, snapshot => snapshot.ManagedHeapBytes); + Assert.Equal(scenario == "churn" ? (iterations - 1) / 2 : 0, restarts); + Assert.Equal(scenario == "partition" ? (iterations + 1) / 3 : 0, partitions); + if (enabled) + { + Assert.All(start.Values, snapshot => Assert.Empty(snapshot.UnconfirmedPeers)); + Assert.All(cluster.Active, node => Assert.Empty(node.Last.UnconfirmedPeers)); + } + + var nodes = finish.Select(snapshot => + { + start.TryGetValue(snapshot.Identity.ProcessId, out var baseline); + var allMessages = Delta(snapshot, baseline, "orleans-messaging-sent-messages-size|"); + var requestMessages = Delta(snapshot, baseline, "orleans-messaging-sent-messages-size|", "MessageDirection=Request"); + var oneWayMessages = Delta(snapshot, baseline, "orleans-messaging-sent-messages-size|", "MessageDirection=OneWay"); + var received = Delta(snapshot, baseline, "orleans-messaging-received-messages-size|"); + var topology = snapshot.TopologyMembers.Contains(snapshot.Address) + ? snapshot + : baseline ?? cluster.All.Single(node => node.Last.Identity.ProcessId == snapshot.Identity.ProcessId).Initial; + var index = Array.IndexOf(topology.TopologyMembers, snapshot.Address); + return new NodeCost( + snapshot.Identity.ProcessId, + snapshot.Address, + index >= 0 ? index : null, + index >= 0 && (topology.AggregationTree ? index <= topology.Fanout : index < topology.Fanout), + topology.OriginatorTargets, + topology.ForwardingTargets, + topology.Fanout, + topology.FanoutSource, + requestMessages.Count + oneWayMessages.Count, + allMessages.Count, + allMessages.Sum, + received.Count, + received.Sum, + snapshot.TransportBytesWritten - (baseline?.TransportBytesWritten ?? 0), + snapshot.TransportBytesRead - (baseline?.TransportBytesRead ?? 0), + snapshot.SocketBytesSent - (baseline?.SocketBytesSent ?? 0), + snapshot.SocketBytesReceived - (baseline?.SocketBytesReceived ?? 0), + snapshot.SocketCounterSamples - (baseline?.SocketCounterSamples ?? 0), + snapshot.CpuMilliseconds - (baseline?.CpuMilliseconds ?? 0), + snapshot.AllocatedBytes - (baseline?.AllocatedBytes ?? 0), + retainedByPid.GetValueOrDefault(snapshot.Identity.ProcessId), + snapshot.WorkingSetBytes, + snapshot.PrivateBytes, + Delta(snapshot, baseline, "orleans-dissemination-bytes-sent|").Sum, + Delta(snapshot, baseline, "orleans-dissemination-broadcast-sent|").Sum, + Delta(snapshot, baseline, "orleans-dissemination-anti-entropy-exchanges|", "direction=out").Sum, + Delta(snapshot, baseline, "orleans-dissemination-broadcast-sent|", "namespace=load").Sum, + Delta(snapshot, baseline, "orleans-dissemination-values-sent|", "namespace=load").Sum); + }).ToArray(); + Assert.All(nodes, node => + { + Assert.True(node.SentMessages > 0, "Orleans message histogram evidence is missing."); + Assert.True(node.SerializedBytesSent > 0, "Serialized byte evidence is missing."); + Assert.True(node.SocketSamples > 0 && node.SocketBytesSent > 0, "OS socket byte evidence is missing."); + }); + if (!enabled) + { + Assert.All(nodes, node => + { + Assert.Equal(0, node.DisseminationBroadcasts); + Assert.Equal(0, node.AntiEntropyExchanges); + }); + } + else + { + Assert.True(nodes.Sum(node => node.DisseminationBroadcasts) > 0, + "The supported-enabled cost window did not exercise any dissemination broadcasts."); + } + + var record = new + { + Schema = 2, + RuntimePath = variant.Name, + Runtime = finish[0].Identity, + Size = size, + LiveSilos = start.Count, + Scenario = scenario, + Workload = workload, + OpenLoop = openLoop, + Enabled = enabled, + Iterations = iterations, + Repetition = repetition, + ExecutionOrder = order, + OfferedPublications = offeredPublications, + RestartedProcesses = restarts, + InjectedPartitions = partitions, + WindowMilliseconds = elapsed, + WindowStartedAtUtc = startedAt, + WindowFinishedAtUtc = finishedAt, + ConvergenceMilliseconds = latencies, + PublicationMilliseconds = publicationLatencies, + ConvergenceAfterPublicationMilliseconds = convergenceWaits, + PartitionRecoveryMilliseconds = recoveryLatencies, + TotalRpcs = nodes.Sum(node => node.Rpcs), + TotalMessages = nodes.Sum(node => node.SentMessages), + LoadBroadcastRequests = nodes.Sum(node => node.LoadBroadcastRequests), + LoadValueTransmissions = nodes.Sum(node => node.LoadValueTransmissions), + LoadBroadcastRequestsPerRound = openLoop is null ? nodes.Sum(node => node.LoadBroadcastRequests) / iterations : (double?)null, + LoadBroadcastRequestsPerSecond = nodes.Sum(node => node.LoadBroadcastRequests) / (elapsed / 1000), + SerializedBytesSent = nodes.Sum(node => node.SerializedBytesSent), + SocketBytesSent = nodes.Sum(node => node.SocketBytesSent), + TransportBytesSubmitted = nodes.Sum(node => node.TransportBytesSubmitted), + CpuMilliseconds = nodes.Sum(node => node.CpuMilliseconds), + AllocatedBytes = nodes.Sum(node => node.AllocatedBytes), + RetainedManagedBytes = nodes.Sum(node => node.RetainedManagedBytes), + BootstrapMilliseconds = bootstrapMilliseconds, + BootstrapCpuMilliseconds = start.Values.Sum(snapshot => snapshot.CpuMilliseconds), + BootstrapAllocatedBytes = start.Values.Sum(snapshot => snapshot.AllocatedBytes), + Nodes = nodes, + Environment = new + { + RuntimeInformation.OSDescription, + RuntimeInformation.FrameworkDescription, + System.Environment.ProcessorCount, + SiloProcessorCount = siloProcessorCount, + GCConserveMemory = gcConserveMemory, + WorkerProcessorCounts = start.Values.Select(snapshot => snapshot.ProcessorCount).Distinct().Order().ToArray(), + WorkerServerGC = start.Values.Select(snapshot => snapshot.ServerGC).Distinct().Order().ToArray(), + }, + Methodology = new + { + WarmupRounds = 2, + SettlingMarginSeconds = 2, + AutomaticLoadPublisherPaused = true, + ProducerCadence = workload == "ClosedLoop" ? "controller publication/convergence rounds" : "worker-local 1 Hz monotonic schedule", + OpenLoopTiming = "5-second common-start arming budget; 3..30-second stable publication window; 10-second completion/latest-state drain; missed periods and overruns fail without catch-up", + Latency = workload == "ClosedLoop" ? "per-round end-to-end exact convergence" : "first-observed ages and per-producer final-latest convergence are polling upper bounds using same-host UTC timestamps", + ProductionDisseminationCadence = variant.Runtime == "New" && enabled, + ExplicitDisseminationOptOut = variant.Runtime == "New" && !enabled, + LoadNamespaceSupportConfirmedBeforeOfferedRounds = enabled, + IncludesUnconfirmedBootstrapTraffic = enabled && scenario == "churn", + LivenessFailureDetectionEnabled = false, + StorageProvider = "locked local JSON file; no external-service network", + Window = "after warmup/capability discovery, through offered load, churn/restarts and final 2-second drain; GC retained-size probe is outside CPU/allocation window", + RpcDefinition = "sent Orleans message-size observations tagged MessageDirection=Request OR MessageDirection=OneWay; not response messages", + SerializedBytes = "complete serialized Orleans messages (headers and bodies), not dissemination payload counters", + SocketBytes = "System.Net.Sockets bytes-sent/bytes-received EventCounters, 1-second sampling; cumulative values differenced across the window (incremental counters also supported); excludes TCP/IP headers and retransmits", + TransportBytes = "bytes advanced at the real connection transport pipe, including handshake; may include buffered bytes lost when a partition aborts a connection", + RetainedMemory = "full GC live managed bytes on surviving nodes only; not a peak; RSS/private bytes and retired-node snapshots also retained", + Includes = "all silo work during the window, including control/snapshot/instrumentation overhead and legacy fallback; restart startup/shutdown costs included for churn", + Observation = "identical state polling on selected paths; detailed counters only at window boundaries", + Topology = "per-node actual tree at end, or start for a retired node; raw command snapshots preserve intervening topology changes; role is not time-weighted", + Excludes = "controller process CPU/allocation, storage disk byte accounting, cross-machine latency, NIC/TCP wire overhead, production storage/placement/grain work", + PartitionComparison = "retire partitioned connections at both endpoints, then confirm healed connections with bidirectional acknowledged control RPCs before the same single publication round on all paths; teardown and readiness remain inside partition recovery time and total cost", + OriginalBaselineAttribution = "original-to-current disabled deltas include all intervening source changes, including lifecycle prerequisites and load notification/tombstone/dedup changes; only current off/on isolates enabling the subsystem on the same binary", + Interpretation = "separate original-legacy, current-explicitly-disabled and current-explicitly-enabled-supported measurements; enabled mode explicitly opts into the subsystem and both namespaces, retaining candidate production dissemination tuning defaults", + }, + }; + await cluster.Save("window-start.json", start); + await cluster.Save("window-finish.json", finish); + await cluster.Save("cost.json", record); + return record; + } + + internal static int ExecutionIndex(int repetition, int order, int count) => + repetition % 2 == 0 ? order : count - 1 - order; + + private static Task ConfirmSupport(ProcessCluster cluster) + { + var timeout = TimeSpan.FromSeconds(90); + foreach (var node in cluster.Active) + { + Assert.True(node.Last.AntiEntropyPeerCount > 0); + Assert.True(node.Last.AntiEntropyIntervalMilliseconds > 0); + var rounds = Math.Ceiling((cluster.Active.Length - 1.0) / node.Last.AntiEntropyPeerCount); + var rotationBudget = TimeSpan.FromMilliseconds(2 * rounds * node.Last.AntiEntropyIntervalMilliseconds) + + TimeSpan.FromSeconds(60); + if (rotationBudget > timeout) + { + timeout = rotationBudget; + } + } + + return cluster.Eventually("all enabled peers confirm support before offered publication rounds", async () => + { + var snapshots = await Task.WhenAll(cluster.Active.Select(node => node.Send("snapshot"))); + return snapshots.All(snapshot => snapshot.UnconfirmedPeers.Length == 0); + }, timeout); + } + + private sealed record RuntimeVariant(string Name, string Runtime, bool Enabled); + + private static MetricValue Delta(NodeSnapshot value, NodeSnapshot? baseline, string prefix, string? tag = null) + { + var entries = value.Metrics.Where(pair => pair.Key.StartsWith(prefix, StringComparison.Ordinal) + && (tag is null || pair.Key.Contains(tag, StringComparison.Ordinal))); + long count = 0; + double sum = 0; + foreach (var (key, metric) in entries) + { + var previous = baseline?.Metrics.GetValueOrDefault(key); + count += metric.Count - (previous?.Count ?? 0); + sum += metric.Sum - (previous?.Sum ?? 0); + } + + return new(count, sum); + } + + private static int Setting(string name, int defaultValue, int minimum, int maximum) + { + var value = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_" + name); + var result = value is null ? defaultValue : int.Parse(value, System.Globalization.CultureInfo.InvariantCulture); + Assert.InRange(result, minimum, maximum); + return result; + } + + private sealed record NodeCost( + int ProcessId, + string Address, + int? TreeRank, + bool UpperTree, + string[] OriginatorTargets, + string[] ForwardingTargets, + int Fanout, + string? FanoutSource, + long Rpcs, + long SentMessages, + double SerializedBytesSent, + long ReceivedMessages, + double SerializedBytesReceived, + long TransportBytesSubmitted, + long TransportBytesRead, + double SocketBytesSent, + double SocketBytesReceived, + long SocketSamples, + double CpuMilliseconds, + long AllocatedBytes, + long RetainedManagedBytes, + long WorkingSetBytes, + long PrivateBytes, + double DisseminationPayloadBytes, + double DisseminationBroadcasts, + double AntiEntropyExchanges, + double LoadBroadcastRequests, + double LoadValueTransmissions); +} diff --git a/test/Dissemination.PerformanceHarness/methodology.json b/test/Dissemination.PerformanceHarness/methodology.json new file mode 100644 index 00000000000..620b983a0e9 --- /dev/null +++ b/test/Dissemination.PerformanceHarness/methodology.json @@ -0,0 +1,84 @@ +{ + "schema": 1, + "purpose": "Manually selected, real-process measurements of original legacy and candidate dissemination off/on costs.", + "entrypoint": ".github/scripts/dissemination-performance.ps1", + "workflow": ".github/workflows/dissemination-performance.yml", + "trigger": "workflow_dispatch", + "extractedFrom": "3fc0a4610cd2eb5feef27e958fcca59658aaae47", + "baseline": { + "repository": "dotnet/orleans", + "commit": "6739589254b746a8790cf53524e6abe372bb53d4" + }, + "selection": { + "candidate": "Explicit repository/ref inputs resolve to a commit in a separate checkout. Both worker checkouts receive only this tooling revision's Silo and Shared files.", + "build": "Each worker resolves runtime project references, SDK selection and package settings from its own checkout. The controller builds from the tooling checkout with shared protocol/instrumentation and MTP/xUnit.", + "identity": "selection.json preserves tooling/candidate/original commits and build dotnet --info. Each OS process checks its PID, source revision, assembly version, informational commit, dissemination presence, loaded Orleans DLL hashes and binary-directory paths against its published manifest.", + "comparison": "Candidate off/on pairs use one published binary directory and identical process counts, workload and environment. Original-to-candidate deltas include every intervening runtime change." + }, + "controls": { + "defaults": { "sizes": [4, 8], "iterations": 3, "repetitions": 2, "runtimePaths": "All", "scenarios": ["stable", "churn", "partition"], "workload": "ClosedLoop" }, + "sizes": "One to six distinct counts, each 3..128", + "iterations": "3..200", + "openLoop": "Workload OpenLoopSynchronized or OpenLoopStaggered requires stable only, 3..32 processes, and Iterations 3..30 as duration seconds at 1 Hz per producer.", + "repetitions": "1..5; even repetitions run original/off/on, odd repetitions run on/off/original; Current selects off/on", + "scenarios": "Distinct stable, churn and partition selections", + "siloProcessorCount": "0..64; 0 inherits the runner processor count", + "gcConserveMemory": "0..9; equal process environment for every path", + "memory": "Linux/Windows available-memory planning guard: 128 MiB per requested silo plus 1 GiB controller headroom; host-resources.json records the calculation. Job/process deadlines bound the run independently.", + "largeCurve": "For an explicitly requested large stable curve, use 8,16,32,64,100 with Current, two repetitions, one processor per silo and GCConserveMemory=9 on a sufficiently provisioned host." + }, + "measurement": { + "test": "Scaling_SelectedRuntimePathsConverge", + "paths": ["OriginalLegacy", "CurrentDisabled", "CurrentEnabledSupported"], + "enablement": "Enabled mode explicitly opts into the subsystem and both namespaces; disabled mode explicitly opts out. Both retain the candidate's production dissemination tuning defaults for overlay, repair, coalescing and namespace cadence.", + "warmup": "Two production publication rounds and complete enabled-peer support confirmation precede the cost window. Preparation time, CPU and allocation totals are reported separately.", + "publication": "Dispose the automatic DeploymentLoadPublisher timer on its scheduler while retaining the field reference for original-runtime lifecycle shutdown. Invoke the production PublishStatistics method once per active node, then require exact load dictionary inventory and values using IncludeFields=true.", + "stable": "Repeat equal closed-loop publication/convergence rounds.", + "openLoopStable": "Arm each worker five seconds before a shared UTC epoch. Worker-local monotonic deadlines offer one production publication per second independently of controller polling. Synchronized producers share the boundary; staggered producers use index/count-second phases. The complete workload is bounded to 30 seconds and 32 processes; missed periods and publication overruns fail while preserving their counts and partial timestamps. A ten-second post-window deadline covers completion and exact latest-state convergence. Observed values must match actually published versions and fields; intermediate unobserved pairs are counted separately.", + "openLoopLatency": "First-observed peer ages and per-producer final-latest convergence are 100 ms polling upper bounds derived from same-host UTC timestamps. Planned/actual publication timestamps, durations and periods remain in raw artifacts. Comparisons keep work pattern/duration/rate/process count equal and distinguish these upper bounds from per-round closed-loop latency.", + "fanoutCapability": "For aggregation routing, read the optional public int Overlay.AggregationFanOutFactor and apply the runtime clamp 1..max(1,members). Absent capability selects the legacy 3fc membership fanout formula and labels that source explicitly; an incompatible property shape fails with a runtime-adapter diagnostic. Membership routing retains its selector.", + "churn": "Gracefully stop and restart one process at its original endpoint every other round. Confirm membership and supported enabled peers before the next offered round. Replacement startup, shutdown, discovery and fallback costs remain in the measured window.", + "partition": "Abort real inbound/outbound silo connections and reject new admissions at one endpoint. Publish once on each remaining node. Recovery drains middleware and close tasks at both endpoints, including unidentified handshakes, then confirms bidirectional acknowledged control RPCs before a single publication per active node. Teardown/readiness/retries remain inside recovery time and total cost, preserving the #11243 barrier.", + "window": "Identical two-second socket-sampling margins; costs include background runtime, control RPCs and instrumentation. The post-window full-GC retained-memory probe is separate from CPU/allocation costs.", + "membership": "Exclusive-file-lease JSON provider with atomic replacement, ETags and incarnation-aware rows. Cleanup removes non-active rows whose runtime EffectiveUpdateTime precedes the cutoff, accounting for startup, heartbeat and suspect votes. Liveness detection is disabled so the transport partition retains stable membership.", + "metrics": [ + "Request plus one-way Orleans RPC counts; all messages including responses reported separately", + "Load broadcast requests and logical value transmissions reported separately, with requests per round and per second", + "Complete serialized Orleans message bytes, including headers and bodies", + "Process-wide System.Net.Sockets sent/received counters and separate transport-pipe submitted/read bytes", + "Process CPU, allocated bytes, retained managed bytes, working set and private bytes", + "Publication latency, residual convergence wait, end-to-end convergence and partition recovery time", + "Per-process topology, originator/forwarding targets, tree rank, upper-tree role and namespace/anti-entropy counters" + ], + "reports": "cost-results.json keeps raw samples. comparisons.json and the job summary pair candidate off/on by size/scenario/repetition, checking equal live process count, binary/revision, environment, iterations and offered publications. Costs normalize per offered publication; median/P95 convergence and incomplete pairs remain explicit." + }, + "interpretation": { + "environment": "Loopback TCP processes share one runner's CPU, memory and local storage. ClosedLoop uses a convergence barrier after each round; optional OpenLoop fixes each producer at 1 Hz with independent state polling.", + "timing": "100 ms state polling and one-second socket sampling introduce boundary uncertainty; raw UTC snapshots and separate publication/wait durations preserve that context.", + "socketBytes": "Socket counters measure application bytes; transport counters include submitted buffered bytes, including bytes abandoned during connection abort.", + "memory": "Retained memory is a post-GC live-size sample for surviving processes. Artifacts also retain retired-process snapshots and per-process RSS/private-memory samples.", + "costScope": "The window covers silo processes. Controller CPU/allocation, storage disk bytes, TCP/IP headers, retransmission, NIC overhead, cross-machine latency and application-grain work require separate experiments.", + "attribution": "The candidate off/on pair isolates enablement on the same binaries. Original-to-candidate comparisons reflect all source changes between the selected revisions." + }, + "bounds": { + "startupSeconds": 90, + "controllerStartupSeconds": 100, + "siloCommandSeconds": 60, + "controllerCommandSeconds": 70, + "convergenceSeconds": 60, + "partitionRecoverySeconds": 60, + "capabilityDiscovery": "max(90 seconds, two complete peer rotations at actual repair cadence plus 60 seconds)", + "membershipLockSeconds": 10, + "shutdownSeconds": 35, + "processLifetimeMinutes": 45, + "workflowMinutes": 90, + "failure": "Preserve partial samples, logs, snapshots, binlogs and TRX. Cleanup kills only owned child processes which exceeded their shutdown deadline and reports that as a failure." + }, + "artifacts": { + "root": "Artifacts/DisseminationPerformance", + "results": "results/-: selection, invocation, methodology, host resources, TRX, comparisons, raw costs and per-cluster process snapshots/stdout/stderr. Open-loop runs also preserve plans, producer publication reports, first observations and summary JSON.", + "binaries": "bin/{Old,New,Runner} and bin/selection.json for explicit -SkipBuild replay", + "paths": "Artifact roots are single named directories under Artifacts; existing links/reparse points are rejected. Every invocation writes a fresh results directory.", + "retentionDays": { "results": 30, "binaries": 7 } + } +} diff --git a/test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj b/test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj index 7c50a8e91a8..d3ae84f89ad 100644 --- a/test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj +++ b/test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj @@ -9,6 +9,13 @@ true + + + + + + +