Process Full Latency Output and Encode in Json (#4157)

microsoft · Feb 27, 2024 · e0385b0 · e0385b0
1 parent e309741
commit e0385b0
Show file tree

Hide file tree

Showing 2 changed files with 63 additions and 13 deletions.
diff --git a/scripts/secnetperf-helpers.psm1 b/scripts/secnetperf-helpers.psm1
@@ -282,6 +282,7 @@ function Start-LocalTest {
         $pinfo.FileName = $FullPath
         $pinfo.Arguments = $FullArgs
     } else {
+        # We use bash to execute the test so we can collect core dumps.
         $pinfo.FileName = "bash"
         $pinfo.Arguments = "-c `"ulimit -c unlimited && LSAN_OPTIONS=report_objects=1 ASAN_OPTIONS=disable_coredump=0:abort_on_error=1 UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 $FullPath $FullArgs && echo ''`""
         $pinfo.WorkingDirectory = $OutputDir
@@ -300,7 +301,9 @@ function Wait-LocalTest {
     param ($Process, $OutputDir, $testKernel, $TimeoutMs)
     $StdOut = $Process.StandardOutput.ReadToEndAsync()
     $StdError = $Process.StandardError.ReadToEndAsync()
+    # Wait for the process to exit.
     if (!$Process.WaitForExit($TimeoutMs)) {
+        # We timed out waiting for completion. Collect a dump of the current state.
         if ($testKernel) { Collect-LiveKD $OutputDir "client" }
         Collect-LocalDump $Process $OutputDir
         try { $Process.Kill() } catch { }
@@ -311,6 +314,7 @@ function Wait-LocalTest {
         } catch {}
         throw "secnetperf: Client timed out!"
     }
+    # Verify the process cleanly exitted.
     if ($Process.ExitCode -ne 0) {
         try {
             [System.Threading.Tasks.Task]::WaitAll(@($StdOut, $StdError))
@@ -319,6 +323,7 @@ function Wait-LocalTest {
         } catch {}
         throw "secnetperf: Nonzero exit code: $($Process.ExitCode)"
     }
+    # Wait for the output streams to flush.
     [System.Threading.Tasks.Task]::WaitAll(@($StdOut, $StdError))
     $consoleTxt = $StdOut.Result.Trim()
     if ($consoleTxt.Length -eq 0) {
@@ -377,20 +382,64 @@ function Get-TestOutput {
     }
 }
 
+function Get-LatencyOutput {
+    param ($FilePath)
+    # The data is in the format of:
+    #
+    #       Value   Percentile   TotalCount 1/(1-Percentile)
+    #
+    #      93.000     0.000000            1         1.00
+    #     118.000     0.100000        18203         1.11
+    #     120.000     0.200000        29488         1.25
+    #         ...          ...          ...          ...
+    #    9847.000     0.999992       139772    131072.00
+    #   41151.000     0.999993       139773    145635.56
+    #   41151.000     1.000000       139773          inf
+    ##[Mean    =      142.875, StdDeviation   =      137.790]
+    ##[Max     =    41151.000, Total count    =       139773]
+    ##[Buckets =            6, SubBuckets     =         2048]
+
+    $contents = Get-Content $FilePath
+    Remove-Item $FilePath
+
+    # Parse through the data and extract the Value and Percentile columns and
+    # convert to an array. Ignore the trailing data.
+    $values = @()
+    $percentiles = @()
+    foreach ($line in $contents) {
+        if ($line -match "^\s*(\d+\.\d+)\s+(\d+\.\d+)") {
+            $values += [int]$matches[1]
+            $percentiles += [double]$matches[2]
+        }
+    }
+
+    return [pscustomobject]@{
+        Values = $values
+        Percentiles = $percentiles
+    }
+}
+
 # Invokes secnetperf with the given arguments for both TCP and QUIC.
 function Invoke-Secnetperf {
     param ($Session, $RemoteName, $RemoteDir, $SecNetPerfPath, $LogProfile, $TestId, $ExeArgs, $io, $Filter)
 
+    $values = @(@(), @())
+    $latency = $null
+    $extraOutput = $null
+    $hasFailures = $false
+    $tcpSupported = ($io -ne "xdp" -and $io -ne "qtip" -and $io -ne "wsk") ? 1 : 0
     $metric = "throughput"
     if ($exeArgs.Contains("plat:1")) {
         $metric = "latency"
+        $latency = @(@(), @())
+        $extraOutput = Repo-Path "latency.txt"
+        if (!$isWindows) {
+            chmod +rw "$extraOutput"
+        }
     } elseif ($exeArgs.Contains("prate:1")) {
         $metric = "hps"
     }
 
-    $values = @(@(), @())
-    $hasFailures = $false
-    $tcpSupported = ($io -ne "xdp" -and $io -ne "qtip" -and $io -ne "wsk") ? 1 : 0
     for ($tcp = 0; $tcp -le $tcpSupported; $tcp++) {
 
     # Set up all the parameters and paths for running the test.
@@ -413,12 +462,8 @@ function Invoke-Secnetperf {
         $serverArgs += " -stats:1"
         $clientArgs += " -pconn:1"
     }
-    if ($testId.Contains("rps")) {
-        $latencyDir = Repo-Path "latency.txt"
-        if (!$isWindows) {
-            chmod +rw "$latencyDir"
-        }
-        $clientArgs += " -extraOutputFile:$latencyDir"
+    if ($extraOutput) {
+        $clientArgs += " -extraOutputFile:$extraOutput"
     }
 
     if (!(Check-TestFilter $clientArgs $Filter)) {
@@ -473,6 +518,9 @@ function Invoke-Secnetperf {
             $rawOutput = Wait-LocalTest $process $artifactDir ($io -eq "wsk") 30000
             Write-Host $rawOutput
             $values[$tcp] += Get-TestOutput $rawOutput $metric
+            if ($extraOutput) {
+                $latency[$tcp] += Get-LatencyOutput $extraOutput
+            }
             $rawOutput | Add-Content $clientOut
             $successCount++
         } catch {
@@ -523,6 +571,7 @@ function Invoke-Secnetperf {
     return [pscustomobject]@{
         Metric = $metric
         Values = $values
+        Latency = $latency
         HasFailures = $hasFailures
     }
 }
diff --git a/scripts/secnetperf.ps1 b/scripts/secnetperf.ps1
@@ -284,6 +284,7 @@ VALUES ("$TestId-tcp-$tcp", "$MsQuicCommit", $env, $env, $item, NULL, "$io", "$t
 "@
             }
         } elseif ($Test.Metric -eq "latency") {
+            $json["$testId-$transport-lat"] = $Test.Latency[$tcp]
             # Test.Values[...] is a flattened 1D array of the form: [ first run + RPS, second run + RPS, third run + RPS..... ], ie. if each run has 8 values + RPS, then the array has 27 elements (8*3 + 3)
             for ($offset = 0; $offset -lt $Test.Values[$tcp].Length; $offset += 9) {
                 $SQL += @"
@@ -325,10 +326,10 @@ Write-Host "Tests complete!"
     } catch { }
 
     # Save the test results (sql and json).
-    Write-Host "`Writing test-results-$environment-$plat-$os-$arch-$tls-$io.sql"
-    $SQL | Set-Content -Path "test-results-$environment-$plat-$os-$arch-$tls-$io.sql"
-    Write-Host "`Writing json-test-results-$environment-$plat-$os-$arch-$tls-$io.json"
-    $json | ConvertTo-Json | Set-Content -Path "json-test-results-$environment-$plat-$os-$arch-$tls-$io.json"
+    Write-Host "`Writing test-results-$environment-$os-$arch-$tls-$io.sql"
+    $SQL | Set-Content -Path "test-results-$environment-$os-$arch-$tls-$io.sql"
+    Write-Host "`Writing json-test-results-$environment-$os-$arch-$tls-$io.json"
+    $json | ConvertTo-Json -Depth 4 | Set-Content -Path "json-test-results-$environment-$os-$arch-$tls-$io.json"
 }
 
 # Clear out any exit codes from previous commands.