From 70362c337d80ac7d88cd9eac984e7971c7a6a472 Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Tue, 7 Mar 2023 14:36:58 -0800 Subject: [PATCH 1/6] rerun flag --- .../stress-testing/deploy-stress-tests.ps1 | 2 ++ .../stress-testing/redepploy-failed-tests.ps1 | 36 +++++++++++++++++++ .../stress-test-deployment-lib.ps1 | 1 + 3 files changed, 39 insertions(+) create mode 100644 eng/common/scripts/stress-testing/redepploy-failed-tests.ps1 diff --git a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 index bbf1d1d42533..706b148a811d 100644 --- a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 +++ b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 @@ -25,6 +25,8 @@ param( # Renders chart templates locally without deployment [Parameter(Mandatory=$False)][switch]$Template, + [Parameter(Mandatory=$False)][switch]$RerunFailedJobs, + # Matrix generation parameters [Parameter(Mandatory=$False)][string]$MatrixFileName, [Parameter(Mandatory=$False)][string]$MatrixSelection, diff --git a/eng/common/scripts/stress-testing/redepploy-failed-tests.ps1 b/eng/common/scripts/stress-testing/redepploy-failed-tests.ps1 new file mode 100644 index 000000000000..d68a24f731f7 --- /dev/null +++ b/eng/common/scripts/stress-testing/redepploy-failed-tests.ps1 @@ -0,0 +1,36 @@ +# Set a default parameter set here so we can call this script without requiring -Login and -Subscription, +# but if it IS called with either of those, then both parameters need to be required. Not defining a +# default parameter set makes Login/Subscription required all the time. +[CmdletBinding(DefaultParameterSetName = 'Default')] +param( + [string]$SearchDirectory, + [hashtable]$Filters, + [string]$Environment, + [string]$Repository, + [switch]$PushImages, + [string]$ClusterGroup, + [string]$DeployId, + [switch]$Login, + [string]$Subscription, + + # Optional namespace override, otherwise the shell user or chart annotation will be used + [string]$Namespace, + + # Override remote stress-test-addons with local on-disk addons for development + [System.IO.FileInfo]$LocalAddonsPath, + + # Matrix generation parameters + [Parameter(Mandatory=$False)][string]$MatrixFileName, + [Parameter(Mandatory=$False)][string]$MatrixSelection, + [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, + [Parameter(Mandatory=$False)][array]$MatrixFilters, + [Parameter(Mandatory=$False)][array]$MatrixReplace, + [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters +) + +. $PSScriptRoot/stress-test-deployment-lib.ps1 + + + +CheckDependencies +DeployStressTests @PSBoundParameters diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 index bafbf77a945c..beaacb034951 100644 --- a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 +++ b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 @@ -98,6 +98,7 @@ function DeployStressTests( })] [System.IO.FileInfo]$LocalAddonsPath, [Parameter(Mandatory=$False)][switch]$Template, + [Parameter(Mandatory=$False)][switch]$RerunFailedJobs, [Parameter(Mandatory=$False)][string]$MatrixFileName, [Parameter(Mandatory=$False)][string]$MatrixSelection = "sparse", [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, From 3ce7ea6c0724f304ff2082632a972c2131ee39e3 Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Thu, 16 Mar 2023 15:01:50 -0700 Subject: [PATCH 2/6] rerun failed stress test --- .../stress-testing/redepploy-failed-tests.ps1 | 36 ------------ .../stress-test-deployment-lib.ps1 | 57 +++++++++++++++++-- 2 files changed, 52 insertions(+), 41 deletions(-) delete mode 100644 eng/common/scripts/stress-testing/redepploy-failed-tests.ps1 diff --git a/eng/common/scripts/stress-testing/redepploy-failed-tests.ps1 b/eng/common/scripts/stress-testing/redepploy-failed-tests.ps1 deleted file mode 100644 index d68a24f731f7..000000000000 --- a/eng/common/scripts/stress-testing/redepploy-failed-tests.ps1 +++ /dev/null @@ -1,36 +0,0 @@ -# Set a default parameter set here so we can call this script without requiring -Login and -Subscription, -# but if it IS called with either of those, then both parameters need to be required. Not defining a -# default parameter set makes Login/Subscription required all the time. -[CmdletBinding(DefaultParameterSetName = 'Default')] -param( - [string]$SearchDirectory, - [hashtable]$Filters, - [string]$Environment, - [string]$Repository, - [switch]$PushImages, - [string]$ClusterGroup, - [string]$DeployId, - [switch]$Login, - [string]$Subscription, - - # Optional namespace override, otherwise the shell user or chart annotation will be used - [string]$Namespace, - - # Override remote stress-test-addons with local on-disk addons for development - [System.IO.FileInfo]$LocalAddonsPath, - - # Matrix generation parameters - [Parameter(Mandatory=$False)][string]$MatrixFileName, - [Parameter(Mandatory=$False)][string]$MatrixSelection, - [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, - [Parameter(Mandatory=$False)][array]$MatrixFilters, - [Parameter(Mandatory=$False)][array]$MatrixReplace, - [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters -) - -. $PSScriptRoot/stress-test-deployment-lib.ps1 - - - -CheckDependencies -DeployStressTests @PSBoundParameters diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 index beaacb034951..2e96e60b72b9 100644 --- a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 +++ b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 @@ -216,9 +216,56 @@ function DeployStressPackage( if ($LASTEXITCODE) {exit $LASTEXITCODE} $dockerBuildConfigs = @() - + $genValFile = Join-Path $pkg.Directory "generatedValues.yaml" $genVal = Get-Content $genValFile -Raw | ConvertFrom-Yaml -Ordered + $releaseName = $pkg.ReleaseName + if ($RerunFailedJobs) { + $pods = kubectl get pods -n $pkg.namespace -o json | ConvertFrom-Json + + # Get all jobs within this helm release + $helmResources = helm status -n $pkg.Namespace $pkg.ReleaseName --show-resources + $discoveredJob = $False + $jobs = @() + foreach ($helmResource in $helmResources) { + if ($discoveredJob -and $helmResource -match "==>") {break} + if ($discoveredJob) { + $jobs += ($helmResource -split '\s+')[0] | where{($_ -ne "NAME") -and ($_)} + } + if ($helmResource -match "==> v1/Job") { + $discoveredJob = $True + } + } + + $failedJobsScenario = @() + $revision = 0 + foreach ($job in $jobs) { + $jobRevision = [int]$job.split('-')[-1] + if ($jobRevision -gt $revision) { + $revision = $jobRevision + } + + $podPhase = kubectl describe jobs -n $pkg.Namespace $job | Select-String "0 Failed" + if ([System.String]::IsNullOrEmpty($podPhase)) { + $failedJobsScenario += $job.split("-$($pkg.ReleaseName)")[0] + } + } + + $releaseName = "$($pkg.ReleaseName)-$revision-retry" + + $genValRerun = @{"scenarios"=@()} + foreach ($failedScenario in $failedJobsScenario) { + $failedScenarioObject = $genVal.scenarios | Where {$_.Scenario -eq $failedScenario} + $genValRerun.scenarios += $failedScenarioObject + } + + if (!$genValRerun.scenarios.length) { + Write-Host "There are no failed jobs to rerun." + return + } + $genVal = $genValRerun + } + if (Test-Path $genValFile) { $scenarios = $genVal.Scenarios foreach ($scenario in $scenarios) { @@ -302,12 +349,12 @@ function DeployStressPackage( $genVal | ConvertTo-Yaml | Out-File -FilePath $genValFile } - Write-Host "Installing or upgrading stress test $($pkg.ReleaseName) from $($pkg.Directory)" + Write-Host "Installing or upgrading stress test $releaseName from $($pkg.Directory)" $generatedConfigPath = Join-Path $pkg.Directory generatedValues.yaml $subCommand = $Template ? "template" : "upgrade" $installFlag = $Template ? "" : "--install" - $helmCommandArg = "helm", $subCommand, $pkg.ReleaseName, $pkg.Directory, "-n", $pkg.Namespace, $installFlag, "--set", "stress-test-addons.env=$environment", "--values", $generatedConfigPath + $helmCommandArg = "helm", $subCommand, $releaseName, $pkg.Directory, "-n", $pkg.Namespace, $installFlag, "--set", "stress-test-addons.env=$environment", "--values", $generatedConfigPath $result = (Run @helmCommandArg) 2>&1 | Write-Host @@ -323,7 +370,7 @@ function DeployStressPackage( # Issues like 'UPGRADE FAILED: another operation (install/upgrade/rollback) is in progress' # can be the result of cancelled `upgrade` operations (e.g. ctrl-c). # See https://github.com/helm/helm/issues/4558 - Write-Warning "The issue may be fixable by first running 'helm rollback -n $($pkg.Namespace) $($pkg.ReleaseName)'" + Write-Warning "The issue may be fixable by first running 'helm rollback -n $($pkg.Namespace) $releaseName'" return } } @@ -334,7 +381,7 @@ function DeployStressPackage( if(!$Template) { $helmReleaseConfig = RunOrExitOnFailure kubectl get secrets ` -n $pkg.Namespace ` - -l "status=deployed,name=$($pkg.ReleaseName)" ` + -l "status=deployed,name=$releaseName" ` -o jsonpath='{.items[0].metadata.name}' Run kubectl label secret -n $pkg.Namespace --overwrite $helmReleaseConfig deployId=$deployId } From 304d376313518d3ff8e8da6305ce8ce0a3df6edc Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Thu, 16 Mar 2023 16:09:17 -0700 Subject: [PATCH 3/6] naming & commenting --- .../stress-testing/deploy-stress-tests.ps1 | 2 +- .../stress-test-deployment-lib.ps1 | 33 +++++++++++++++---- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 index 706b148a811d..bc028f26aa10 100644 --- a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 +++ b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 @@ -25,7 +25,7 @@ param( # Renders chart templates locally without deployment [Parameter(Mandatory=$False)][switch]$Template, - [Parameter(Mandatory=$False)][switch]$RerunFailedJobs, + [Parameter(Mandatory=$False)][switch]$RetryFailedTests, # Matrix generation parameters [Parameter(Mandatory=$False)][string]$MatrixFileName, diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 index 2e96e60b72b9..ae6efb05300d 100644 --- a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 +++ b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 @@ -98,7 +98,7 @@ function DeployStressTests( })] [System.IO.FileInfo]$LocalAddonsPath, [Parameter(Mandatory=$False)][switch]$Template, - [Parameter(Mandatory=$False)][switch]$RerunFailedJobs, + [Parameter(Mandatory=$False)][switch]$RetryFailedTests, [Parameter(Mandatory=$False)][string]$MatrixFileName, [Parameter(Mandatory=$False)][string]$MatrixSelection = "sparse", [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, @@ -220,11 +220,30 @@ function DeployStressPackage( $genValFile = Join-Path $pkg.Directory "generatedValues.yaml" $genVal = Get-Content $genValFile -Raw | ConvertFrom-Yaml -Ordered $releaseName = $pkg.ReleaseName - if ($RerunFailedJobs) { + if ($RetryFailedTests) { $pods = kubectl get pods -n $pkg.namespace -o json | ConvertFrom-Json # Get all jobs within this helm release $helmResources = helm status -n $pkg.Namespace $pkg.ReleaseName --show-resources + # -----Example output----- + # NAME: + # LAST DEPLOYED: Mon Jan 01 12:12:12 2020 + # NAMESPACE: + # STATUS: deployed + # REVISION: 10 + # RESOURCES: + # ==> v1alpha1/Schedule + # NAME AGE + # 5h5m + # 5h5m + + # ==> v1/SecretProviderClass + # 7d4h + + # ==> v1/Job + # NAME COMPLETIONS DURATION AGE + # 0/1 5h5m 5h5m + # 0/1 5h5m 5h5m $discoveredJob = $False $jobs = @() foreach ($helmResource in $helmResources) { @@ -253,17 +272,17 @@ function DeployStressPackage( $releaseName = "$($pkg.ReleaseName)-$revision-retry" - $genValRerun = @{"scenarios"=@()} + $genValRetry = @{"scenarios"=@()} foreach ($failedScenario in $failedJobsScenario) { $failedScenarioObject = $genVal.scenarios | Where {$_.Scenario -eq $failedScenario} - $genValRerun.scenarios += $failedScenarioObject + $genValRetry.scenarios += $failedScenarioObject } - if (!$genValRerun.scenarios.length) { - Write-Host "There are no failed jobs to rerun." + if (!$genValRetry.scenarios.length) { + Write-Host "There are no failed pods to retry." return } - $genVal = $genValRerun + $genVal = $genValRetry } if (Test-Path $genValFile) { From f804bcd91e8ed216f9e716482ccdbecfbd588e93 Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Thu, 23 Mar 2023 11:31:25 -0700 Subject: [PATCH 4/6] update --- .../stress-test-deployment-lib.ps1 | 128 +++++++++--------- 1 file changed, 66 insertions(+), 62 deletions(-) diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 index ae6efb05300d..4acdebadefe6 100644 --- a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 +++ b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 @@ -221,68 +221,7 @@ function DeployStressPackage( $genVal = Get-Content $genValFile -Raw | ConvertFrom-Yaml -Ordered $releaseName = $pkg.ReleaseName if ($RetryFailedTests) { - $pods = kubectl get pods -n $pkg.namespace -o json | ConvertFrom-Json - - # Get all jobs within this helm release - $helmResources = helm status -n $pkg.Namespace $pkg.ReleaseName --show-resources - # -----Example output----- - # NAME: - # LAST DEPLOYED: Mon Jan 01 12:12:12 2020 - # NAMESPACE: - # STATUS: deployed - # REVISION: 10 - # RESOURCES: - # ==> v1alpha1/Schedule - # NAME AGE - # 5h5m - # 5h5m - - # ==> v1/SecretProviderClass - # 7d4h - - # ==> v1/Job - # NAME COMPLETIONS DURATION AGE - # 0/1 5h5m 5h5m - # 0/1 5h5m 5h5m - $discoveredJob = $False - $jobs = @() - foreach ($helmResource in $helmResources) { - if ($discoveredJob -and $helmResource -match "==>") {break} - if ($discoveredJob) { - $jobs += ($helmResource -split '\s+')[0] | where{($_ -ne "NAME") -and ($_)} - } - if ($helmResource -match "==> v1/Job") { - $discoveredJob = $True - } - } - - $failedJobsScenario = @() - $revision = 0 - foreach ($job in $jobs) { - $jobRevision = [int]$job.split('-')[-1] - if ($jobRevision -gt $revision) { - $revision = $jobRevision - } - - $podPhase = kubectl describe jobs -n $pkg.Namespace $job | Select-String "0 Failed" - if ([System.String]::IsNullOrEmpty($podPhase)) { - $failedJobsScenario += $job.split("-$($pkg.ReleaseName)")[0] - } - } - - $releaseName = "$($pkg.ReleaseName)-$revision-retry" - - $genValRetry = @{"scenarios"=@()} - foreach ($failedScenario in $failedJobsScenario) { - $failedScenarioObject = $genVal.scenarios | Where {$_.Scenario -eq $failedScenario} - $genValRetry.scenarios += $failedScenarioObject - } - - if (!$genValRetry.scenarios.length) { - Write-Host "There are no failed pods to retry." - return - } - $genVal = $genValRetry + runRetryTests $pkg ([ref]$releaseName) ([ref]$genVal) } if (Test-Path $genValFile) { @@ -442,3 +381,68 @@ function CheckDependencies() } } + +function runRetryTests ($pkg, [ref]$releaseName, [ref]$genVal) { + $pods = kubectl get pods -n $pkg.namespace -o json | ConvertFrom-Json + + # Get all jobs within this helm release + $helmResources = helm status -n $pkg.Namespace $pkg.ReleaseName --show-resources + # -----Example output----- + # NAME: + # LAST DEPLOYED: Mon Jan 01 12:12:12 2020 + # NAMESPACE: + # STATUS: deployed + # REVISION: 10 + # RESOURCES: + # ==> v1alpha1/Schedule + # NAME AGE + # 5h5m + # 5h5m + + # ==> v1/SecretProviderClass + # 7d4h + + # ==> v1/Job + # NAME COMPLETIONS DURATION AGE + # 0/1 5h5m 5h5m + # 0/1 5h5m 5h5m + $discoveredJob = $False + $jobs = @() + foreach ($helmResource in $helmResources) { + if ($discoveredJob -and $helmResource -match "==>") {break} + if ($discoveredJob) { + $jobs += ($helmResource -split '\s+')[0] | Where-Object {($_ -ne "NAME") -and ($_)} + } + if ($helmResource -match "==> v1/Job") { + $discoveredJob = $True + } + } + + $failedJobsScenario = @() + $revision = 0 + foreach ($job in $jobs) { + $jobRevision = [int]$job.split('-')[-1] + if ($jobRevision -gt $revision) { + $revision = $jobRevision + } + + $podPhase = kubectl describe jobs -n $pkg.Namespace $job | Select-String "0 Failed" + if ([System.String]::IsNullOrEmpty($podPhase)) { + $failedJobsScenario += $job.split("-$($pkg.ReleaseName)")[0] + } + } + + $releaseName = "$($pkg.ReleaseName)-$revision-retry" + + $genValRetry = @{"scenarios"=@()} + foreach ($failedScenario in $failedJobsScenario) { + $failedScenarioObject = $genVal.value.scenarios | Where {$_.Scenario -eq $failedScenario} + $genValRetry.scenarios += $failedScenarioObject + } + + if (!$genValRetry.scenarios.length) { + Write-Host "There are no failed pods to retry." + return + } + $genVal.value = $genValRetry +} From 7078f5cb869e94fd318cf6ef2ea8378be4b994b5 Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Thu, 23 Mar 2023 14:40:21 -0700 Subject: [PATCH 5/6] function and var renaming for better readability --- .../stress-test-deployment-lib.ps1 | 27 ++++++++++--------- eng/common/scripts/stress-testing/test.ps1 | 16 +++++++++++ 2 files changed, 30 insertions(+), 13 deletions(-) create mode 100644 eng/common/scripts/stress-testing/test.ps1 diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 index 4acdebadefe6..1f7bc2d3970f 100644 --- a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 +++ b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 @@ -217,15 +217,15 @@ function DeployStressPackage( $dockerBuildConfigs = @() - $genValFile = Join-Path $pkg.Directory "generatedValues.yaml" - $genVal = Get-Content $genValFile -Raw | ConvertFrom-Yaml -Ordered + $generatedHelmValuesFilePath = Join-Path $pkg.Directory "generatedValues.yaml" + $generatedHelmValues = Get-Content $generatedHelmValuesFilePath -Raw | ConvertFrom-Yaml -Ordered $releaseName = $pkg.ReleaseName if ($RetryFailedTests) { - runRetryTests $pkg ([ref]$releaseName) ([ref]$genVal) + $releaseName, $generatedHelmValues = generateRetryTestsHelmValues $pkg $releaseName $generatedHelmValues } - if (Test-Path $genValFile) { - $scenarios = $genVal.Scenarios + if (Test-Path $generatedHelmValuesFilePath) { + $scenarios = $generatedHelmValues.Scenarios foreach ($scenario in $scenarios) { if ("image" -in $scenario.keys) { $dockerFilePath = Join-Path $pkg.Directory $scenario.image @@ -292,7 +292,7 @@ function DeployStressPackage( } } } - $genVal.scenarios = @( foreach ($scenario in $genVal.scenarios) { + $generatedHelmValues.scenarios = @( foreach ($scenario in $generatedHelmValues.scenarios) { $dockerPath = if ("image" -notin $scenario) { $dockerFilePath } else { @@ -304,7 +304,7 @@ function DeployStressPackage( $scenario } ) - $genVal | ConvertTo-Yaml | Out-File -FilePath $genValFile + $generatedHelmValues | ConvertTo-Yaml | Out-File -FilePath $generatedHelmValuesFilePath } Write-Host "Installing or upgrading stress test $releaseName from $($pkg.Directory)" @@ -382,7 +382,7 @@ function CheckDependencies() } -function runRetryTests ($pkg, [ref]$releaseName, [ref]$genVal) { +function generateRetryTestsHelmValues ($pkg, $releaseName, $generatedHelmValues) { $pods = kubectl get pods -n $pkg.namespace -o json | ConvertFrom-Json # Get all jobs within this helm release @@ -434,15 +434,16 @@ function runRetryTests ($pkg, [ref]$releaseName, [ref]$genVal) { $releaseName = "$($pkg.ReleaseName)-$revision-retry" - $genValRetry = @{"scenarios"=@()} + $retryTestsHelmVal = @{"scenarios"=@()} foreach ($failedScenario in $failedJobsScenario) { - $failedScenarioObject = $genVal.value.scenarios | Where {$_.Scenario -eq $failedScenario} - $genValRetry.scenarios += $failedScenarioObject + $failedScenarioObject = $generatedHelmValues.scenarios | Where {$_.Scenario -eq $failedScenario} + $retryTestsHelmVal.scenarios += $failedScenarioObject } - if (!$genValRetry.scenarios.length) { + if (!$retryTestsHelmVal.scenarios.length) { Write-Host "There are no failed pods to retry." return } - $genVal.value = $genValRetry + $generatedHelmValues = $retryTestsHelmVal + return $releaseName, $generatedHelmValues } diff --git a/eng/common/scripts/stress-testing/test.ps1 b/eng/common/scripts/stress-testing/test.ps1 new file mode 100644 index 000000000000..3e1d35fb1512 --- /dev/null +++ b/eng/common/scripts/stress-testing/test.ps1 @@ -0,0 +1,16 @@ + +function determineRetryTests ([ref] $val, [ref]$val2) { + $val.value += 1 + $val2.value += 2 + Write-Host "val $($val.value)" + Write-Host "val $($val2.value)" +} + +$v = 0 +$v2 = 0 +determineRetryTests ([ref]$v) ([ref]$v2) + +Write-host "v $v" +Write-Host "v2 $v2" + + From bdc486e2eebc102480d6f6fdadc19ae4abcf8bcd Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Mon, 27 Mar 2023 11:21:58 -0700 Subject: [PATCH 6/6] readability & exit on error --- .../stress-test-deployment-lib.ps1 | 17 ++++++++++------- eng/common/scripts/stress-testing/test.ps1 | 16 ---------------- 2 files changed, 10 insertions(+), 23 deletions(-) delete mode 100644 eng/common/scripts/stress-testing/test.ps1 diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 index 1f7bc2d3970f..a7d45ae66f27 100644 --- a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 +++ b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 @@ -383,10 +383,12 @@ function CheckDependencies() } function generateRetryTestsHelmValues ($pkg, $releaseName, $generatedHelmValues) { - $pods = kubectl get pods -n $pkg.namespace -o json | ConvertFrom-Json + $podOutput = RunOrExitOnFailure kubectl get pods -n $pkg.namespace -o json + $pods = $podOutput | ConvertFrom-Json # Get all jobs within this helm release - $helmResources = helm status -n $pkg.Namespace $pkg.ReleaseName --show-resources + + $helmStatusOutput = RunOrExitOnFailure helm status -n $pkg.Namespace $pkg.ReleaseName --show-resources # -----Example output----- # NAME: # LAST DEPLOYED: Mon Jan 01 12:12:12 2020 @@ -408,12 +410,12 @@ function generateRetryTestsHelmValues ($pkg, $releaseName, $generatedHelmValues) # 0/1 5h5m 5h5m $discoveredJob = $False $jobs = @() - foreach ($helmResource in $helmResources) { - if ($discoveredJob -and $helmResource -match "==>") {break} + foreach ($line in $helmStatusOutput) { + if ($discoveredJob -and $line -match "==>") {break} if ($discoveredJob) { - $jobs += ($helmResource -split '\s+')[0] | Where-Object {($_ -ne "NAME") -and ($_)} + $jobs += ($line -split '\s+')[0] | Where-Object {($_ -ne "NAME") -and ($_)} } - if ($helmResource -match "==> v1/Job") { + if ($line -match "==> v1/Job") { $discoveredJob = $True } } @@ -426,7 +428,8 @@ function generateRetryTestsHelmValues ($pkg, $releaseName, $generatedHelmValues) $revision = $jobRevision } - $podPhase = kubectl describe jobs -n $pkg.Namespace $job | Select-String "0 Failed" + $jobOutput = RunOrExitOnFailure kubectl describe jobs -n $pkg.Namespace $job + $podPhase = $jobOutput | Select-String "0 Failed" if ([System.String]::IsNullOrEmpty($podPhase)) { $failedJobsScenario += $job.split("-$($pkg.ReleaseName)")[0] } diff --git a/eng/common/scripts/stress-testing/test.ps1 b/eng/common/scripts/stress-testing/test.ps1 deleted file mode 100644 index 3e1d35fb1512..000000000000 --- a/eng/common/scripts/stress-testing/test.ps1 +++ /dev/null @@ -1,16 +0,0 @@ - -function determineRetryTests ([ref] $val, [ref]$val2) { - $val.value += 1 - $val2.value += 2 - Write-Host "val $($val.value)" - Write-Host "val $($val2.value)" -} - -$v = 0 -$v2 = 0 -determineRetryTests ([ref]$v) ([ref]$v2) - -Write-host "v $v" -Write-Host "v2 $v2" - -