From e0385b047d7bd622319e9f6c4c00184a1294eea8 Mon Sep 17 00:00:00 2001 From: Nick Banks Date: Mon, 26 Feb 2024 20:00:14 -0500 Subject: [PATCH] Process Full Latency Output and Encode in Json (#4157) --- scripts/secnetperf-helpers.psm1 | 67 ++++++++++++++++++++++++++++----- scripts/secnetperf.ps1 | 9 +++-- 2 files changed, 63 insertions(+), 13 deletions(-) diff --git a/scripts/secnetperf-helpers.psm1 b/scripts/secnetperf-helpers.psm1 index 9a87f5f914..82eb63e5d6 100644 --- a/scripts/secnetperf-helpers.psm1 +++ b/scripts/secnetperf-helpers.psm1 @@ -282,6 +282,7 @@ function Start-LocalTest { $pinfo.FileName = $FullPath $pinfo.Arguments = $FullArgs } else { + # We use bash to execute the test so we can collect core dumps. $pinfo.FileName = "bash" $pinfo.Arguments = "-c `"ulimit -c unlimited && LSAN_OPTIONS=report_objects=1 ASAN_OPTIONS=disable_coredump=0:abort_on_error=1 UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 $FullPath $FullArgs && echo ''`"" $pinfo.WorkingDirectory = $OutputDir @@ -300,7 +301,9 @@ function Wait-LocalTest { param ($Process, $OutputDir, $testKernel, $TimeoutMs) $StdOut = $Process.StandardOutput.ReadToEndAsync() $StdError = $Process.StandardError.ReadToEndAsync() + # Wait for the process to exit. if (!$Process.WaitForExit($TimeoutMs)) { + # We timed out waiting for completion. Collect a dump of the current state. if ($testKernel) { Collect-LiveKD $OutputDir "client" } Collect-LocalDump $Process $OutputDir try { $Process.Kill() } catch { } @@ -311,6 +314,7 @@ function Wait-LocalTest { } catch {} throw "secnetperf: Client timed out!" } + # Verify the process cleanly exitted. if ($Process.ExitCode -ne 0) { try { [System.Threading.Tasks.Task]::WaitAll(@($StdOut, $StdError)) @@ -319,6 +323,7 @@ function Wait-LocalTest { } catch {} throw "secnetperf: Nonzero exit code: $($Process.ExitCode)" } + # Wait for the output streams to flush. [System.Threading.Tasks.Task]::WaitAll(@($StdOut, $StdError)) $consoleTxt = $StdOut.Result.Trim() if ($consoleTxt.Length -eq 0) { @@ -377,20 +382,64 @@ function Get-TestOutput { } } +function Get-LatencyOutput { + param ($FilePath) + # The data is in the format of: + # + # Value Percentile TotalCount 1/(1-Percentile) + # + # 93.000 0.000000 1 1.00 + # 118.000 0.100000 18203 1.11 + # 120.000 0.200000 29488 1.25 + # ... ... ... ... + # 9847.000 0.999992 139772 131072.00 + # 41151.000 0.999993 139773 145635.56 + # 41151.000 1.000000 139773 inf + ##[Mean = 142.875, StdDeviation = 137.790] + ##[Max = 41151.000, Total count = 139773] + ##[Buckets = 6, SubBuckets = 2048] + + $contents = Get-Content $FilePath + Remove-Item $FilePath + + # Parse through the data and extract the Value and Percentile columns and + # convert to an array. Ignore the trailing data. + $values = @() + $percentiles = @() + foreach ($line in $contents) { + if ($line -match "^\s*(\d+\.\d+)\s+(\d+\.\d+)") { + $values += [int]$matches[1] + $percentiles += [double]$matches[2] + } + } + + return [pscustomobject]@{ + Values = $values + Percentiles = $percentiles + } +} + # Invokes secnetperf with the given arguments for both TCP and QUIC. function Invoke-Secnetperf { param ($Session, $RemoteName, $RemoteDir, $SecNetPerfPath, $LogProfile, $TestId, $ExeArgs, $io, $Filter) + $values = @(@(), @()) + $latency = $null + $extraOutput = $null + $hasFailures = $false + $tcpSupported = ($io -ne "xdp" -and $io -ne "qtip" -and $io -ne "wsk") ? 1 : 0 $metric = "throughput" if ($exeArgs.Contains("plat:1")) { $metric = "latency" + $latency = @(@(), @()) + $extraOutput = Repo-Path "latency.txt" + if (!$isWindows) { + chmod +rw "$extraOutput" + } } elseif ($exeArgs.Contains("prate:1")) { $metric = "hps" } - $values = @(@(), @()) - $hasFailures = $false - $tcpSupported = ($io -ne "xdp" -and $io -ne "qtip" -and $io -ne "wsk") ? 1 : 0 for ($tcp = 0; $tcp -le $tcpSupported; $tcp++) { # Set up all the parameters and paths for running the test. @@ -413,12 +462,8 @@ function Invoke-Secnetperf { $serverArgs += " -stats:1" $clientArgs += " -pconn:1" } - if ($testId.Contains("rps")) { - $latencyDir = Repo-Path "latency.txt" - if (!$isWindows) { - chmod +rw "$latencyDir" - } - $clientArgs += " -extraOutputFile:$latencyDir" + if ($extraOutput) { + $clientArgs += " -extraOutputFile:$extraOutput" } if (!(Check-TestFilter $clientArgs $Filter)) { @@ -473,6 +518,9 @@ function Invoke-Secnetperf { $rawOutput = Wait-LocalTest $process $artifactDir ($io -eq "wsk") 30000 Write-Host $rawOutput $values[$tcp] += Get-TestOutput $rawOutput $metric + if ($extraOutput) { + $latency[$tcp] += Get-LatencyOutput $extraOutput + } $rawOutput | Add-Content $clientOut $successCount++ } catch { @@ -523,6 +571,7 @@ function Invoke-Secnetperf { return [pscustomobject]@{ Metric = $metric Values = $values + Latency = $latency HasFailures = $hasFailures } } diff --git a/scripts/secnetperf.ps1 b/scripts/secnetperf.ps1 index 917187afc0..926cea3b79 100644 --- a/scripts/secnetperf.ps1 +++ b/scripts/secnetperf.ps1 @@ -284,6 +284,7 @@ VALUES ("$TestId-tcp-$tcp", "$MsQuicCommit", $env, $env, $item, NULL, "$io", "$t "@ } } elseif ($Test.Metric -eq "latency") { + $json["$testId-$transport-lat"] = $Test.Latency[$tcp] # Test.Values[...] is a flattened 1D array of the form: [ first run + RPS, second run + RPS, third run + RPS..... ], ie. if each run has 8 values + RPS, then the array has 27 elements (8*3 + 3) for ($offset = 0; $offset -lt $Test.Values[$tcp].Length; $offset += 9) { $SQL += @" @@ -325,10 +326,10 @@ Write-Host "Tests complete!" } catch { } # Save the test results (sql and json). - Write-Host "`Writing test-results-$environment-$plat-$os-$arch-$tls-$io.sql" - $SQL | Set-Content -Path "test-results-$environment-$plat-$os-$arch-$tls-$io.sql" - Write-Host "`Writing json-test-results-$environment-$plat-$os-$arch-$tls-$io.json" - $json | ConvertTo-Json | Set-Content -Path "json-test-results-$environment-$plat-$os-$arch-$tls-$io.json" + Write-Host "`Writing test-results-$environment-$os-$arch-$tls-$io.sql" + $SQL | Set-Content -Path "test-results-$environment-$os-$arch-$tls-$io.sql" + Write-Host "`Writing json-test-results-$environment-$os-$arch-$tls-$io.json" + $json | ConvertTo-Json -Depth 4 | Set-Content -Path "json-test-results-$environment-$os-$arch-$tls-$io.json" } # Clear out any exit codes from previous commands.