From b94e9e9921c16d757730d55628a0ecf60359a2ed Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Tue, 13 Feb 2024 19:20:06 -0500 Subject: [PATCH 1/4] Support resolving environment variable references in matrix config --- doc/common/matrix_generator.md | 24 +++ ...ob-matrix-functions.modification.tests.ps1 | 120 +++++++++-- .../tests/job-matrix-functions.tests.ps1 | 6 +- .../job-matrix/job-matrix-functions.ps1 | 195 ++++++++++-------- 4 files changed, 238 insertions(+), 107 deletions(-) diff --git a/doc/common/matrix_generator.md b/doc/common/matrix_generator.md index 8405982d01e..1f8a87680cf 100644 --- a/doc/common/matrix_generator.md +++ b/doc/common/matrix_generator.md @@ -17,6 +17,7 @@ * [all](#all) * [sparse](#sparse) * [include/exclude](#includeexclude) + * [Environment Variable References](#environment-variable-references) * [Generated display name](#generated-display-name) * [Filters](#filters) * [Replace/Modify/Append](#replacemodifyappend-values) @@ -512,6 +513,29 @@ will be generated, but the full matrix of both include and exclude will be proce Excludes are processed first, so includes can be used to forcefully add specific combinations to the matrix, regardless of exclusions. +### Environment Variable References + +The matrix config supports values that reference environment variables and resolves them at +matrix generation time. This is useful especially in pipeline scenarios where we want to reference +common values that are defined elsewhere in the environment and/or pipeline config. + +Prefix a matrix value with `env:` to trigger an environment variable +lookup. If the variable does not exist, then the value will resolve to empty string. + +For example: + +``` yaml +{ + "net461_macOS1015": { + "framework": "net461", + "operatingSystem": "env:OperatingSystem" + } +} +``` + +Matrix filters and replace parameters evaluate before environment variables are resolved. Matrix +display name renames and display name filters evaluate after variables are resolved. + ### Generated display name In the matrix job output that azure pipelines consumes, the format is a map of maps. For example: diff --git a/eng/common-tests/matrix-generator/tests/job-matrix-functions.modification.tests.ps1 b/eng/common-tests/matrix-generator/tests/job-matrix-functions.modification.tests.ps1 index e0daf43dbce..9f6b5f4f719 100644 --- a/eng/common-tests/matrix-generator/tests/job-matrix-functions.modification.tests.ps1 +++ b/eng/common-tests/matrix-generator/tests/job-matrix-functions.modification.tests.ps1 @@ -2,7 +2,7 @@ Import-Module Pester BeforeAll { - . $PSScriptRoot/../../../common/scripts/job-matrix/job-matrix-functions.ps1 + . $PSScriptRoot/../../../common/scripts/job-matrix/job-matrix-functions.ps1 function CompareMatrices([Array]$matrix, [Array]$expected) { $matrix.Length | Should -Be $expected.Length @@ -33,17 +33,17 @@ Describe "Platform Matrix nonSparse" -Tag "UnitTest", "nonsparse" { } It "Should process nonSparse parameters" { - $parameters, $nonSparse = ProcessNonSparseParameters $config.matrixParameters "testField1","testField3" + $parameters, $nonSparse = ProcessNonSparseParameters $config.matrixParameters "testField1", "testField3" $parameters.Count | Should -Be 1 $parameters[0].Name | Should -Be "testField2" - $parameters[0].Value | Should -Be 1,2,3 + $parameters[0].Value | Should -Be 1, 2, 3 $nonSparse.Count | Should -Be 2 $nonSparse[0].Name | Should -Be "testField1" - $nonSparse[0].Value | Should -Be 1,2 + $nonSparse[0].Value | Should -Be 1, 2 $nonSparse[1].Name | Should -Be "testField3" - $nonSparse[1].Value | Should -Be 1,2,3,4 + $nonSparse[1].Value | Should -Be 1, 2, 3, 4 $parameters, $nonSparse = ProcessNonSparseParameters $config.matrixParameters "testField3" $parameters.Count | Should -Be 2 @@ -51,7 +51,7 @@ Describe "Platform Matrix nonSparse" -Tag "UnitTest", "nonsparse" { $nonSparse.Count | Should -Be 1 $nonSparse[0].Name | Should -Be "testField3" - $nonSparse[0].Value | Should -Be 1,2,3,4 + $nonSparse[0].Value | Should -Be 1, 2, 3, 4 } It "Should ignore nonSparse with all selection" { @@ -77,10 +77,10 @@ Describe "Platform Matrix nonSparse" -Tag "UnitTest", "nonsparse" { '@ $config = GetMatrixConfigFromJson $matrixJson - $matrix = GenerateMatrix $config "all" -nonSparseParameters "testField3","testField4" + $matrix = GenerateMatrix $config "all" -nonSparseParameters "testField3", "testField4" $matrix.Length | Should -Be 16 - $matrix = GenerateMatrix $config "sparse" -nonSparseParameters "testField3","testField4" + $matrix = GenerateMatrix $config "sparse" -nonSparseParameters "testField3", "testField4" $matrix.Length | Should -Be 8 } @@ -125,13 +125,13 @@ Describe "Platform Matrix nonSparse" -Tag "UnitTest", "nonsparse" { } } -# This test is currently disabled (it doesn't have "UnitTest" tag) as it fails +# This test is currently disabled (it doesn't have "UnitTest" tag) as it fails # in test "Should generate a sparse matrix where the entire base matrix is imported" on line: # # $matrix = GenerateMatrix $importConfig "sparse" # # with message: -# +# # ParameterBindingArgumentTransformationException: Cannot process argument transformation on parameter 'parameters'. Cannot convert the "System.Collections.Hashtable" value of type "System.Collections.Hashtable" to type "MatrixParameter". # # See full build failure: @@ -392,11 +392,11 @@ Describe "Platform Matrix Import" -Tag "import" { Describe "Platform Matrix Replace" -Tag "UnitTest", "replace" { It "Should parse replacement syntax" -TestCases @( - @{ query = 'foo=bar/baz'; key = '^foo$'; value = '^bar$'; replace = 'baz' }, - @{ query = 'foo=\/p:bar/\/p:baz'; key = '^foo$'; value = '^\/p:bar$'; replace = '/p:baz' }, - @{ query = 'f\=o\/o=\/p:b\=ar/\/p:b\=az'; key = '^f\=o\/o$'; value = '^\/p:b\=ar$'; replace = '/p:b=az' }, - @{ query = 'foo=bar/'; key = '^foo$'; value = '^bar$'; replace = '' }, - @{ query = 'foo=/baz'; key = '^foo$'; value = '^$'; replace = 'baz' } + @{ query = 'foo=bar/baz'; key = '^foo$'; value = '^bar$'; replace = 'baz' }, + @{ query = 'foo=\/p:bar/\/p:baz'; key = '^foo$'; value = '^\/p:bar$'; replace = '/p:baz' }, + @{ query = 'f\=o\/o=\/p:b\=ar/\/p:b\=az'; key = '^f\=o\/o$'; value = '^\/p:b\=ar$'; replace = '/p:b=az' }, + @{ query = 'foo=bar/'; key = '^foo$'; value = '^bar$'; replace = '' }, + @{ query = 'foo=/baz'; key = '^foo$'; value = '^$'; replace = 'baz' } ) { $parsed = ParseReplacement $query $parsed.key | Should -Be $key @@ -582,5 +582,95 @@ Describe "Platform Matrix Replace" -Tag "UnitTest", "replace" { $matrix[2].parameters.Baz | Should -Be "importedBaz" $matrix[2].parameters.replaceme | Should -Be "replaceme" } +} + +Describe "Platform Matrix Environment Variables" -Tag "UnitTest", "envvar" { + It "Should parse environment variable reference syntax" { + $matrixJson = @' +{ + "matrix": { + "foo": "bar", + "envReference": ["env:TestMatrixEnvReference", "env:TestMatrixEnvReference2", "noref"] + } +} +'@ + + [System.Environment]::SetEnvironmentVariable("TestMatrixEnvReference", "") + [array]$matrix = GenerateMatrix (GetMatrixConfigFromJson $matrixJson) "sparse" + $matrix.Length | Should -Be 3 + $matrix[0].name | Should -Be "bar" + $matrix[0].parameters.envReference | Should -Be "" + + [System.Environment]::SetEnvironmentVariable("TestMatrixEnvReference", "replaced") + [System.Environment]::SetEnvironmentVariable("TestMatrixEnvReference2", "replaced2") + [array]$replacedMatrix = GenerateMatrix (GetMatrixConfigFromJson $matrixJson) "sparse" + $replacedMatrix.Length | Should -Be 3 + $replacedMatrix[0].name | Should -Be "bar_replaced" + $replacedMatrix[0].parameters.envReference | Should -Be "replaced" + $replacedMatrix[1].name | Should -Be "bar_replaced2" + $replacedMatrix[1].parameters.envReference | Should -Be "replaced2" + $replacedMatrix[2].name | Should -Be "bar_noref" + $replacedMatrix[2].parameters.envReference | Should -Be "noref" + } + + It "Should support filter/replace with variable reference syntax" { + $matrixJson = @' +{ + "displayNames": { + "env:replaceme": "env:TestMatrixEnvReference" + }, + "matrix": { + "foo": "bar", + "envReference": "env:replaceme" + } +} +'@ + + [System.Environment]::SetEnvironmentVariable("TestMatrixEnvReference", "replaced") + + [array]$replacedMatrix = GenerateMatrix ` + -config (GetMatrixConfigFromJson $matrixJson) ` + -selectFromMatrixType "sparse" ` + -replace @("envReference=env:replaceme/env:TestMatrixEnvReference") + $replacedMatrix.Length | Should -Be 1 + $replacedMatrix[0].name | Should -Be "bar_replaced" + $replacedMatrix[0].parameters.envReference | Should -Be "replaced" + + # Don't filter out by replaced values, but by original references + [System.Environment]::SetEnvironmentVariable("replaceme", "filter_replaced") + [array]$replacedMatrix = GenerateMatrix ` + -config (GetMatrixConfigFromJson $matrixJson) ` + -selectFromMatrixType "sparse" ` + -filter @("envReference=env:replaceme") + $replacedMatrix.Length | Should -Be 1 + $replacedMatrix[0].name | Should -Be "bar_filter_replaced" + $replacedMatrix[0].parameters.envReference | Should -Be "filter_replaced" + } + It "Should support display name and display name filter with variable reference syntax" { + $matrixJson = @' +{ + "displayNames": { + "replaced": "display" + }, + "matrix": { + "foo": "bar", + "envReference": "env:TestMatrixEnvReference" + } } +'@ + + [System.Environment]::SetEnvironmentVariable("TestMatrixEnvReference", "replaced") + [array]$replacedMatrix = GenerateMatrix (GetMatrixConfigFromJson $matrixJson) "sparse" + $replacedMatrix.Length | Should -Be 1 + $replacedMatrix[0].name | Should -Be "bar_display" + $replacedMatrix[0].parameters.envReference | Should -Be "replaced" + + [System.Environment]::SetEnvironmentVariable("TestMatrixEnvReference", "replaced") + [array]$replacedMatrix = GenerateMatrix ` + -config (GetMatrixConfigFromJson $matrixJson) ` + -selectFromMatrixType "sparse" ` + -displayNameFilter "doesnotexist" + $replacedMatrix | Should -BeNullOrEmpty + } +} \ No newline at end of file diff --git a/eng/common-tests/matrix-generator/tests/job-matrix-functions.tests.ps1 b/eng/common-tests/matrix-generator/tests/job-matrix-functions.tests.ps1 index a29202b3a43..5a01b3e990a 100644 --- a/eng/common-tests/matrix-generator/tests/job-matrix-functions.tests.ps1 +++ b/eng/common-tests/matrix-generator/tests/job-matrix-functions.tests.ps1 @@ -228,17 +228,17 @@ Describe "Matrix-Reverse-Lookup" -Tag "UnitTest", "lookup" { @{ index = 1; expected = @(0,0,0,1) } @{ index = 2; expected = @(0,0,0,2) } @{ index = 3; expected = @(0,0,0,3) } - + @{ index = 4; expected = @(0,0,1,0) } @{ index = 5; expected = @(0,0,1,1) } @{ index = 6; expected = @(0,0,1,2) } @{ index = 7; expected = @(0,0,1,3) } - + @{ index = 8; expected = @(0,1,0,0) } @{ index = 9; expected = @(0,1,0,1) } @{ index = 10; expected = @(0,1,0,2) } @{ index = 11; expected = @(0,1,0,3) } - + @{ index = 12; expected = @(0,1,1,0) } @{ index = 13; expected = @(0,1,1,1) } @{ index = 14; expected = @(0,1,1,2) } diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 index fa8a1da2d09..983baaf0a91 100644 --- a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -18,8 +18,7 @@ class MatrixParameter { [System.Object]$Value [System.Object]$Name - Set($value, [String]$keyRegex = '') - { + Set($value, [String]$keyRegex = '') { if ($this.Value -is [PSCustomObject]) { $set = $false foreach ($prop in $this.Value.PSObject.Properties) { @@ -32,44 +31,48 @@ class MatrixParameter { if (!$set) { throw "Property `"$keyRegex`" does not exist for MatrixParameter." } - } else { + } + else { $this.Value = $value } } - [System.Object]Flatten() - { + [System.Object]Flatten() { if ($this.Value -is [PSCustomObject]) { return $this.Value.PSObject.Properties | ForEach-Object { [MatrixParameter]::new($_.Name, $_.Value) } - } elseif ($this.Value -is [Array]) { + } + elseif ($this.Value -is [Array]) { return $this.Value | ForEach-Object { [MatrixParameter]::new($this.Name, $_) } - } else { + } + else { return $this } } - [Int]Length() - { + [Int]Length() { if ($this.Value -is [PSCustomObject]) { return ($this.Value.PSObject.Properties | Measure-Object).Count - } elseif ($this.Value -is [Array]) { + } + elseif ($this.Value -is [Array]) { return $this.Value.Length - } else { + } + else { return 1 } } - [String]CreateDisplayName([Hashtable]$displayNamesLookup) - { + [String]CreateDisplayName([Hashtable]$displayNamesLookup) { if ($null -eq $this.Value) { $displayName = "" - } elseif ($this.Value -is [PSCustomObject]) { + } + elseif ($this.Value -is [PSCustomObject]) { $displayName = $this.Name - } else { + } + else { $displayName = $this.Value.ToString() } @@ -96,13 +99,15 @@ function GenerateMatrix( [Array]$nonSparseParameters = @() ) { $matrixParameters, $importedMatrix, $combinedDisplayNameLookup = ` - ProcessImport $config.matrixParameters $selectFromMatrixType $nonSparseParameters $config.displayNamesLookup + ProcessImport $config.matrixParameters $selectFromMatrixType $nonSparseParameters $config.displayNamesLookup if ($selectFromMatrixType -eq "sparse") { $matrix = GenerateSparseMatrix $matrixParameters $config.displayNamesLookup $nonSparseParameters - } elseif ($selectFromMatrixType -eq "all") { + } + elseif ($selectFromMatrixType -eq "all") { $matrix = GenerateFullMatrix $matrixParameters $config.displayNamesLookup - } else { - throw "Matrix generator not implemented for selectFromMatrixType: $($platform.selectFromMatrixType)" + } + else { + throw "Matrix generator not implemented for selectFromMatrixType: '$selectFromMatrixType'" } # Combine with imported after matrix generation, since a sparse selection should result in a full combination of the @@ -119,6 +124,7 @@ function GenerateMatrix( $matrix = FilterMatrix $matrix $filters $matrix = ProcessReplace $matrix $replace $combinedDisplayNameLookup + $matrix = ProcessEnvironmentVariableReferences $matrix $combinedDisplayNameLookup $matrix = FilterMatrixDisplayName $matrix $displayNameFilter return $matrix } @@ -137,7 +143,8 @@ function ProcessNonSparseParameters( foreach ($param in $parameters) { if ($param.Name -in $nonSparseParameters) { $nonSparse += $param - } else { + } + else { $sparse += $param } } @@ -186,39 +193,37 @@ function ParseFilter([string]$filter) { $key = $matches[1] $regex = $matches[2] return $key, $regex - } else { + } + else { throw "Invalid filter: `"${filter}`", expected = format" } } -function GetMatrixConfigFromFile([String] $config) -{ - [MatrixConfig]$config = try{ +function GetMatrixConfigFromFile([String] $config) { + [MatrixConfig]$config = try { GetMatrixConfigFromJson $config - } catch { + } + catch { GetMatrixConfigFromYaml $config } return $config } -function GetMatrixConfigFromYaml([String] $yamlConfig) -{ +function GetMatrixConfigFromYaml([String] $yamlConfig) { Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module # ConvertTo then from json is to make sure the nested values are in PSCustomObject [MatrixConfig]$config = ConvertFrom-Yaml $yamlConfig -Ordered | ConvertTo-Json -Depth 100 | ConvertFrom-Json return GetMatrixConfig $config } -function GetMatrixConfigFromJson([String]$jsonConfig) -{ +function GetMatrixConfigFromJson([String]$jsonConfig) { [MatrixConfig]$config = $jsonConfig | ConvertFrom-Json return GetMatrixConfig $config } # Importing the JSON as PSCustomObject preserves key ordering, # whereas ConvertFrom-Json -AsHashtable does not -function GetMatrixConfig([MatrixConfig]$config) -{ +function GetMatrixConfig([MatrixConfig]$config) { $config.matrixParameters = @() $config.displayNamesLookup = @{} $include = [MatrixParameter[]]@() @@ -233,10 +238,10 @@ function GetMatrixConfig([MatrixConfig]$config) $config.matrixParameters = PsObjectToMatrixParameterArray $config.matrix } foreach ($includeMatrix in $config.include) { - $include += ,@(PsObjectToMatrixParameterArray $includeMatrix) + $include += , @(PsObjectToMatrixParameterArray $includeMatrix) } foreach ($excludeMatrix in $config.exclude) { - $exclude += ,@(PsObjectToMatrixParameterArray $excludeMatrix) + $exclude += , @(PsObjectToMatrixParameterArray $excludeMatrix) } $config.include = $include @@ -245,8 +250,7 @@ function GetMatrixConfig([MatrixConfig]$config) return $config } -function PsObjectToMatrixParameterArray([PSCustomObject]$obj) -{ +function PsObjectToMatrixParameterArray([PSCustomObject]$obj) { if ($obj -eq $null) { return $null } @@ -255,8 +259,7 @@ function PsObjectToMatrixParameterArray([PSCustomObject]$obj) } } -function ProcessExcludes([Array]$matrix, [Array]$excludes) -{ +function ProcessExcludes([Array]$matrix, [Array]$excludes) { $deleteKey = "%DELETE%" $exclusionMatrix = @() @@ -277,8 +280,7 @@ function ProcessExcludes([Array]$matrix, [Array]$excludes) return $matrix | Where-Object { !$_.parameters.Contains($deleteKey) } } -function ProcessIncludes([MatrixConfig]$config, [Array]$matrix) -{ +function ProcessIncludes([MatrixConfig]$config, [Array]$matrix) { $inclusionMatrix = @() foreach ($inclusion in $config.include) { $full = GenerateFullMatrix $inclusion $config.displayNamesLookup @@ -301,7 +303,8 @@ function ParseReplacement([String]$replacement) { } if (!$escaped -and $c -in $operators) { $idx++ - } else { + } + else { $parsed[$idx] += $c } $escaped = $c -eq '\' @@ -314,15 +317,14 @@ function ParseReplacement([String]$replacement) { $replace = $parsed[2] -replace "\\([$($operators -join '')])", '$1' return @{ - "key" = '^' + $parsed[0] + '$' + "key" = '^' + $parsed[0] + '$' # Force full matches only. - "value" = '^' + $parsed[1] + '$' + "value" = '^' + $parsed[1] + '$' "replace" = $replace } } -function ProcessReplace -{ +function ProcessReplace { param( [Array]$matrix, [Array]$replacements, @@ -365,18 +367,46 @@ function ProcessReplace return $replaceMatrix } -function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Array]$nonSparseParameters, [Hashtable]$displayNamesLookup) -{ +function ProcessEnvironmentVariableReferences([array]$matrix, $displayNamesLookup) { + $updatedMatrix = @() + + foreach ($element in $matrix) { + $updated = [MatrixParameter[]]@() + + foreach ($perm in $element._permutation) { + # Iterate nested permutations or run once for singular values (int, string, bool) + foreach ($flattened in $perm.Flatten()) { + if ($flattened.Value?.GetType() -eq "".GetType() -and $flattened.Value.StartsWith("env:")) { + $envKey = $flattened.Value.Replace("env:", "") + $value = [System.Environment]::GetEnvironmentVariable($envKey) ?? "" + if (!$value) { + Write-Warning "Environment variable `"$envKey`" was not found or is empty." + } + $flattened.Set($value, $flattened.Name) + } + } + + $updated += $perm + } + + $updatedMatrix += CreateMatrixCombinationScalar $updated $displayNamesLookup + } + + return $updatedMatrix +} + +function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Array]$nonSparseParameters, [Hashtable]$displayNamesLookup) { $importPath = "" $matrix = $matrix | ForEach-Object { if ($_.Name -ne $IMPORT_KEYWORD) { return $_ - } else { + } + else { $importPath = $_.Value } } if ((!$matrix -and !$importPath) -or !$importPath) { - return $matrix, @(), @{} + return $matrix, @(), $displayNamesLookup } if (!(Test-Path $importPath)) { @@ -385,9 +415,9 @@ function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Array]$n } $importedMatrixConfig = GetMatrixConfigFromFile (Get-Content -Raw $importPath) $importedMatrix = GenerateMatrix ` - -config $importedMatrixConfig ` - -selectFromMatrixType $selection ` - -nonSparseParameters $nonSparseParameters + -config $importedMatrixConfig ` + -selectFromMatrixType $selection ` + -nonSparseParameters $nonSparseParameters $combinedDisplayNameLookup = $importedMatrixConfig.displayNamesLookup foreach ($lookup in $displayNamesLookup.GetEnumerator()) { @@ -397,8 +427,7 @@ function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Array]$n return $matrix, $importedMatrix, $combinedDisplayNameLookup } -function CombineMatrices([Array]$matrix1, [Array]$matrix2, [Hashtable]$displayNamesLookup = @{}) -{ +function CombineMatrices([Array]$matrix1, [Array]$matrix2, [Hashtable]$displayNamesLookup = @{}) { $combined = @() if (!$matrix1) { return $matrix2 @@ -416,8 +445,7 @@ function CombineMatrices([Array]$matrix1, [Array]$matrix2, [Hashtable]$displayNa return $combined } -function MatrixElementMatch([System.Collections.Specialized.OrderedDictionary]$source, [System.Collections.Specialized.OrderedDictionary]$target) -{ +function MatrixElementMatch([System.Collections.Specialized.OrderedDictionary]$source, [System.Collections.Specialized.OrderedDictionary]$target) { if ($target.Count -eq 0) { return $false } @@ -439,8 +467,7 @@ function CloneOrderedDictionary([System.Collections.Specialized.OrderedDictionar return $newDictionary } -function SerializePipelineMatrix([Array]$matrix) -{ +function SerializePipelineMatrix([Array]$matrix) { $pipelineMatrix = [Ordered]@{} foreach ($entry in $matrix) { if ($pipelineMatrix.Contains($entry.Name)) { @@ -455,7 +482,7 @@ function SerializePipelineMatrix([Array]$matrix) return @{ compressed = $pipelineMatrix | ConvertTo-Json -Compress ; - pretty = $pipelineMatrix | ConvertTo-Json; + pretty = $pipelineMatrix | ConvertTo-Json; } } @@ -482,8 +509,7 @@ function GenerateSparseMatrix( return $sparseMatrix } -function GetSparseMatrixIndexes([Array]$dimensions) -{ +function GetSparseMatrixIndexes([Array]$dimensions) { $size = ($dimensions | Measure-Object -Maximum).Maximum $indexes = @() @@ -498,10 +524,10 @@ function GetSparseMatrixIndexes([Array]$dimensions) for ($j = 0; $j -lt $dimensions.Length; $j++) { $idx += $i % $dimensions[$j] } - $indexes += ,$idx + $indexes += , $idx } - return ,$indexes + return , $indexes } function GenerateFullMatrix( @@ -519,8 +545,7 @@ function GenerateFullMatrix( return $matrix } -function CreateMatrixCombinationScalar([MatrixParameter[]]$permutation, [Hashtable]$displayNamesLookup = @{}) -{ +function CreateMatrixCombinationScalar([MatrixParameter[]]$permutation, [Hashtable]$displayNamesLookup = @{}) { $names = @() $flattenedParameters = [Ordered]@{} @@ -551,15 +576,14 @@ function CreateMatrixCombinationScalar([MatrixParameter[]]$permutation, [Hashtab } return @{ - name = $name - parameters = $flattenedParameters + name = $name + parameters = $flattenedParameters # Keep the original permutation around in case we need to re-process this entry when transforming the matrix _permutation = $permutation } } -function InitializeMatrix -{ +function InitializeMatrix { param( [MatrixParameter[]]$parameters, [Hashtable]$displayNamesLookup, @@ -581,8 +605,7 @@ function InitializeMatrix } } -function GetMatrixDimensions([MatrixParameter[]]$parameters) -{ +function GetMatrixDimensions([MatrixParameter[]]$parameters) { $dimensions = @() foreach ($param in $parameters) { $dimensions += $param.Length() @@ -591,8 +614,7 @@ function GetMatrixDimensions([MatrixParameter[]]$parameters) return $dimensions } -function SetNdMatrixElement -{ +function SetNdMatrixElement { param( $element, [ValidateNotNullOrEmpty()] @@ -611,8 +633,7 @@ function SetNdMatrixElement $matrix[$arrayIndex] = $element } -function GetNdMatrixArrayIndex -{ +function GetNdMatrixArrayIndex { param( [ValidateNotNullOrEmpty()] [Array]$idx, @@ -627,20 +648,19 @@ function GetNdMatrixArrayIndex $stride = 1 # Commented out does lookup with wrap handling # $index = $idx[$idx.Length-1] % $dimensions[$idx.Length-1] - $index = $idx[$idx.Length-1] + $index = $idx[$idx.Length - 1] - for ($i = $dimensions.Length-1; $i -ge 1; $i--) { + for ($i = $dimensions.Length - 1; $i -ge 1; $i--) { $stride *= $dimensions[$i] # Commented out does lookup with wrap handling # $index += ($idx[$i-1] % $dimensions[$i-1]) * $stride - $index += $idx[$i-1] * $stride + $index += $idx[$i - 1] * $stride } return $index } -function GetNdMatrixElement -{ +function GetNdMatrixElement { param( [ValidateNotNullOrEmpty()] [Array]$idx, @@ -654,8 +674,7 @@ function GetNdMatrixElement return $matrix[$arrayIndex] } -function GetNdMatrixIndex -{ +function GetNdMatrixIndex { param( [int]$index, [ValidateNotNullOrEmpty()] @@ -665,12 +684,12 @@ function GetNdMatrixIndex $matrixIndex = @() $stride = 1 - for ($i = $dimensions.Length-1; $i -ge 1; $i--) { + for ($i = $dimensions.Length - 1; $i -ge 1; $i--) { $stride *= $dimensions[$i] - $page = [math]::floor($index / $stride) % $dimensions[$i-1] - $matrixIndex = ,$page + $matrixIndex + $page = [math]::floor($index / $stride) % $dimensions[$i - 1] + $matrixIndex = , $page + $matrixIndex } - $col = $index % $dimensions[$dimensions.Length-1] + $col = $index % $dimensions[$dimensions.Length - 1] $matrixIndex += $col return $matrixIndex @@ -680,8 +699,7 @@ function GetNdMatrixIndex # The below functions are non-dynamic examples that # # help explain the above N-dimensional algorithm # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -function Get4dMatrixElement([Array]$idx, [Array]$matrix, [Array]$dimensions) -{ +function Get4dMatrixElement([Array]$idx, [Array]$matrix, [Array]$dimensions) { $stride1 = $idx[0] * $dimensions[1] * $dimensions[2] * $dimensions[3] $stride2 = $idx[1] * $dimensions[2] * $dimensions[3] $stride3 = $idx[2] * $dimensions[3] @@ -690,8 +708,7 @@ function Get4dMatrixElement([Array]$idx, [Array]$matrix, [Array]$dimensions) return $matrix[$stride1 + $stride2 + $stride3 + $stride4] } -function Get4dMatrixIndex([int]$index, [Array]$dimensions) -{ +function Get4dMatrixIndex([int]$index, [Array]$dimensions) { $stride1 = $dimensions[3] $stride2 = $dimensions[2] $stride3 = $dimensions[1] From 5b9cb4a13caaccf7b9a795543143d1c8d685c0b5 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Tue, 13 Feb 2024 20:32:54 -0500 Subject: [PATCH 2/4] Improve type and null handling --- .../job-matrix-functions.modification.tests.ps1 | 14 +++++++++++--- .../scripts/job-matrix/job-matrix-functions.ps1 | 10 ++++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/eng/common-tests/matrix-generator/tests/job-matrix-functions.modification.tests.ps1 b/eng/common-tests/matrix-generator/tests/job-matrix-functions.modification.tests.ps1 index 9f6b5f4f719..3e1e27a453c 100644 --- a/eng/common-tests/matrix-generator/tests/job-matrix-functions.modification.tests.ps1 +++ b/eng/common-tests/matrix-generator/tests/job-matrix-functions.modification.tests.ps1 @@ -591,26 +591,34 @@ Describe "Platform Matrix Environment Variables" -Tag "UnitTest", "envvar" { "matrix": { "foo": "bar", "envReference": ["env:TestMatrixEnvReference", "env:TestMatrixEnvReference2", "noref"] - } + }, + "include": [ + { + "foo": "bar", + "envReference": "env:TestMatrixEnvReference" + } + ] } '@ [System.Environment]::SetEnvironmentVariable("TestMatrixEnvReference", "") [array]$matrix = GenerateMatrix (GetMatrixConfigFromJson $matrixJson) "sparse" - $matrix.Length | Should -Be 3 + $matrix.Length | Should -Be 4 $matrix[0].name | Should -Be "bar" $matrix[0].parameters.envReference | Should -Be "" [System.Environment]::SetEnvironmentVariable("TestMatrixEnvReference", "replaced") [System.Environment]::SetEnvironmentVariable("TestMatrixEnvReference2", "replaced2") [array]$replacedMatrix = GenerateMatrix (GetMatrixConfigFromJson $matrixJson) "sparse" - $replacedMatrix.Length | Should -Be 3 + $replacedMatrix.Length | Should -Be 4 $replacedMatrix[0].name | Should -Be "bar_replaced" $replacedMatrix[0].parameters.envReference | Should -Be "replaced" $replacedMatrix[1].name | Should -Be "bar_replaced2" $replacedMatrix[1].parameters.envReference | Should -Be "replaced2" $replacedMatrix[2].name | Should -Be "bar_noref" $replacedMatrix[2].parameters.envReference | Should -Be "noref" + $replacedMatrix[3].name | Should -Be "bar_replaced" + $replacedMatrix[3].parameters.envReference | Should -Be "replaced" } It "Should support filter/replace with variable reference syntax" { diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 index 983baaf0a91..c11a318b9bd 100644 --- a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -76,7 +76,7 @@ class MatrixParameter { $displayName = $this.Value.ToString() } - if ($displayNamesLookup.ContainsKey($displayName)) { + if ($displayNamesLookup -and $displayNamesLookup.ContainsKey($displayName)) { $displayName = $displayNamesLookup[$displayName] } @@ -339,6 +339,9 @@ function ProcessReplace { foreach ($element in $matrix) { $replacement = [MatrixParameter[]]@() + if (!$element -or $element.Count -eq 0) { + continue + } foreach ($perm in $element._permutation) { $replace = $perm @@ -372,11 +375,14 @@ function ProcessEnvironmentVariableReferences([array]$matrix, $displayNamesLooku foreach ($element in $matrix) { $updated = [MatrixParameter[]]@() + if (!$element -or $element.Count -eq 0) { + continue + } foreach ($perm in $element._permutation) { # Iterate nested permutations or run once for singular values (int, string, bool) foreach ($flattened in $perm.Flatten()) { - if ($flattened.Value?.GetType() -eq "".GetType() -and $flattened.Value.StartsWith("env:")) { + if ($flattened.Value -is [string] -and $flattened.Value.StartsWith("env:")) { $envKey = $flattened.Value.Replace("env:", "") $value = [System.Environment]::GetEnvironmentVariable($envKey) ?? "" if (!$value) { From a309fafd57de5266fb60b25417cff664741ebf82 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Thu, 15 Feb 2024 14:28:38 -0500 Subject: [PATCH 3/4] Fix reference bug --- eng/common/scripts/job-matrix/job-matrix-functions.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 index c11a318b9bd..693cf489c24 100644 --- a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -388,7 +388,7 @@ function ProcessEnvironmentVariableReferences([array]$matrix, $displayNamesLooku if (!$value) { Write-Warning "Environment variable `"$envKey`" was not found or is empty." } - $flattened.Set($value, $flattened.Name) + $perm.Set($value, $flattened.Name) } } From c5b029a64250e53189da02e9fb412551d4e5816c Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Thu, 15 Feb 2024 17:45:04 -0500 Subject: [PATCH 4/4] Change behavior on missing env vars to throw --- .../tests/job-matrix-functions.modification.tests.ps1 | 7 ++----- eng/common/scripts/job-matrix/job-matrix-functions.ps1 | 8 ++++++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/eng/common-tests/matrix-generator/tests/job-matrix-functions.modification.tests.ps1 b/eng/common-tests/matrix-generator/tests/job-matrix-functions.modification.tests.ps1 index 3e1e27a453c..f4d8e3c71aa 100644 --- a/eng/common-tests/matrix-generator/tests/job-matrix-functions.modification.tests.ps1 +++ b/eng/common-tests/matrix-generator/tests/job-matrix-functions.modification.tests.ps1 @@ -602,10 +602,7 @@ Describe "Platform Matrix Environment Variables" -Tag "UnitTest", "envvar" { '@ [System.Environment]::SetEnvironmentVariable("TestMatrixEnvReference", "") - [array]$matrix = GenerateMatrix (GetMatrixConfigFromJson $matrixJson) "sparse" - $matrix.Length | Should -Be 4 - $matrix[0].name | Should -Be "bar" - $matrix[0].parameters.envReference | Should -Be "" + { GenerateMatrix (GetMatrixConfigFromJson $matrixJson) "sparse" } | Should -Throw [System.Environment]::SetEnvironmentVariable("TestMatrixEnvReference", "replaced") [System.Environment]::SetEnvironmentVariable("TestMatrixEnvReference2", "replaced2") @@ -681,4 +678,4 @@ Describe "Platform Matrix Environment Variables" -Tag "UnitTest", "envvar" { -displayNameFilter "doesnotexist" $replacedMatrix | Should -BeNullOrEmpty } -} \ No newline at end of file +} diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 index 693cf489c24..d70ea6d87b6 100644 --- a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -372,6 +372,7 @@ function ProcessReplace { function ProcessEnvironmentVariableReferences([array]$matrix, $displayNamesLookup) { $updatedMatrix = @() + $missingEnvVars = @{} foreach ($element in $matrix) { $updated = [MatrixParameter[]]@() @@ -384,9 +385,9 @@ function ProcessEnvironmentVariableReferences([array]$matrix, $displayNamesLooku foreach ($flattened in $perm.Flatten()) { if ($flattened.Value -is [string] -and $flattened.Value.StartsWith("env:")) { $envKey = $flattened.Value.Replace("env:", "") - $value = [System.Environment]::GetEnvironmentVariable($envKey) ?? "" + $value = [System.Environment]::GetEnvironmentVariable($envKey) if (!$value) { - Write-Warning "Environment variable `"$envKey`" was not found or is empty." + $missingEnvVars[$envKey] = $true } $perm.Set($value, $flattened.Name) } @@ -398,6 +399,9 @@ function ProcessEnvironmentVariableReferences([array]$matrix, $displayNamesLooku $updatedMatrix += CreateMatrixCombinationScalar $updated $displayNamesLookup } + if ($missingEnvVars.Count -gt 0) { + throw "Environment variables '$($missingEnvVars.Keys -join ", ")' were empty or not found." + } return $updatedMatrix }