From 4ebf5316921d81b7218b6bd47e50aaee5fbb5303 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Wed, 10 Jul 2024 14:05:55 -0700 Subject: [PATCH 1/9] original draft of the adjustments. definitely going to be some discussion on the serviceInput parameter. probably want to go to a separate script entirely? add missing line --- eng/common/scripts/Generate-PR-Diff.ps1 | 2 +- eng/common/scripts/Package-Properties.ps1 | 14 +++++++++++++ .../scripts/Save-Package-Properties.ps1 | 21 ++++++++++++++----- eng/common/scripts/common.ps1 | 1 + 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/eng/common/scripts/Generate-PR-Diff.ps1 b/eng/common/scripts/Generate-PR-Diff.ps1 index 8239b219c..d84f9e15c 100644 --- a/eng/common/scripts/Generate-PR-Diff.ps1 +++ b/eng/common/scripts/Generate-PR-Diff.ps1 @@ -45,7 +45,7 @@ $changedServices = Get-ChangedServices -ChangedFiles $changedFiles $result = [PSCustomObject]@{ "ChangedFiles" = $changedFiles "ChangedServices" = $changedServices - "PRNumber" = $env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER + "PRNumber" = if ($env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER) { $env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER } else { "-1" } } $result | ConvertTo-Json | Out-File $ArtifactName diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index 606998c8e..013188504 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -105,6 +105,20 @@ function Get-PkgProperties return $null } +function Get-PrPkgProperties([string]$InputDiffJson) { + $pkgPropsResult = @() + + Write-Host "In Get-PRPkgProperties function" + + if ($GetPRPackageInfoFromRepoFn -and (Test-Path "Function:$GetPRPackageInfoFromRepoFn")) + { + Write-Host "Attempting to invoke the language one" + $pkgPropsResult = &$GetPRPackageInfoFromRepoFn -InputDiffJson $InputDiffJson + } + + return $pkgPropsResult +} + # Takes ServiceName and Repo Root Directory # Returns important properties for each package in the specified service, or entire repo if the serviceName is not specified # Returns a Table of service key to array values of PS Object with properties @ { pkgName, pkgVersion, pkgDirectoryPath, pkgReadMePath, pkgChangeLogPath } diff --git a/eng/common/scripts/Save-Package-Properties.ps1 b/eng/common/scripts/Save-Package-Properties.ps1 index 0e8ce26aa..faf6640f6 100644 --- a/eng/common/scripts/Save-Package-Properties.ps1 +++ b/eng/common/scripts/Save-Package-Properties.ps1 @@ -14,8 +14,8 @@ In cases of collisions where track 2 packages (IsNewSdk = true) have the same filename as track 1 packages (e.g. same artifact name or package name), the track 2 package properties will be written. -.PARAMETER serviceDirectory -Service directory in which to search for packages +.PARAMETER serviceInput +Service directory in which to search for packages, or file path ending in diff.json. .PARAMETER outDirectory Output location (generally a package artifact directory in DevOps) for JSON @@ -33,7 +33,7 @@ Verison property in that file. [CmdletBinding()] Param ( [Parameter(Mandatory=$True)] - [string] $serviceDirectory, + [string] $serviceInput, [Parameter(Mandatory=$True)] [string] $outDirectory, [switch] $addDevVersion @@ -92,7 +92,18 @@ function GetRelativePath($path) { } $exportedPaths = @{} -$allPackageProperties = Get-AllPkgProperties $serviceDirectory + +$allPackageProperties = @() + +if ($serviceInput.endswith("diff.json")) { + $allPackageProperties = Get-PrPkgProperties $serviceInput +} +else { + $allPackageProperties = Get-AllPkgProperties $serviceInput +} + +Write-Host $allPackageProperties + if ($allPackageProperties) { if (-not (Test-Path -Path $outDirectory)) @@ -137,6 +148,6 @@ if ($allPackageProperties) } else { - Write-Error "Package properties are not available for service directory $($serviceDirectory)" + Write-Error "Package properties are not available for service directory $($serviceInput)" exit 1 } diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index 0025d1c6f..d53c3b165 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -42,6 +42,7 @@ if (!(Get-Variable -Name "LanguageDisplayName" -ValueOnly -ErrorAction "Ignore") # Transformed Functions # Expected to be set in eng/scripts/Language-Settings.ps1 $GetPackageInfoFromRepoFn = "Get-${Language}-PackageInfoFromRepo" +$GetPRPackageInfoFromRepoFn = "Get-${Language}-PRPackageInfoFromRepo" $GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile" $PublishGithubIODocsFn = "Publish-${Language}-GithubIODocs" $UpdateDocsMsPackagesFn = "Update-${Language}-DocsMsPackages" From 608914c5884deda218db47e4f104d03f299e92b4 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Mon, 15 Jul 2024 14:57:54 -0700 Subject: [PATCH 2/9] admendments as suggested by Wes --- eng/common/scripts/Package-Properties.ps1 | 30 ++++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index 013188504..a9915101c 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -105,18 +105,36 @@ function Get-PkgProperties return $null } +function Get-PackageFolderFromPath($file) { + $parts = $file -split "/" + + if ($parts.Length -lt 3) { + return "" + } + return [System.IO.Path]::Combine($parts[0], $parts[1], $parts[2]) +} + +# Takes an input diff json file generated from eng/common/scripts/Generate-PR-Diff.ps1 +# On PR builds, this function should be utilized instead of Get-AllPkgProperties. function Get-PrPkgProperties([string]$InputDiffJson) { - $pkgPropsResult = @() + $packagesWithChanges = @() - Write-Host "In Get-PRPkgProperties function" + $allPackageProperties = Get-AllPkgProperties + $diff = Get-Content $InputDiffJson | ConvertFrom-Json + $targetedFiles = $diff.ChangedFiles - if ($GetPRPackageInfoFromRepoFn -and (Test-Path "Function:$GetPRPackageInfoFromRepoFn")) + foreach($pkg in $allPackageProperties) { - Write-Host "Attempting to invoke the language one" - $pkgPropsResult = &$GetPRPackageInfoFromRepoFn -InputDiffJson $InputDiffJson + foreach($file in $targetedFiles) + { + $pkgFolder = Get-PackageFolderFromPath($file) + if ($pkgFolder.EndsWith($pkg.ArtifactName)) { + $packagesWithChanges += $pkg + } + } } - return $pkgPropsResult + return $packagesWithChanges } # Takes ServiceName and Repo Root Directory From e54fda658812055f9b6c26c4aa3248baaff69ef3 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Mon, 15 Jul 2024 15:01:13 -0700 Subject: [PATCH 3/9] remove new function ref in common.ps1 --- eng/common/scripts/common.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index d53c3b165..0025d1c6f 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -42,7 +42,6 @@ if (!(Get-Variable -Name "LanguageDisplayName" -ValueOnly -ErrorAction "Ignore") # Transformed Functions # Expected to be set in eng/scripts/Language-Settings.ps1 $GetPackageInfoFromRepoFn = "Get-${Language}-PackageInfoFromRepo" -$GetPRPackageInfoFromRepoFn = "Get-${Language}-PRPackageInfoFromRepo" $GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile" $PublishGithubIODocsFn = "Publish-${Language}-GithubIODocs" $UpdateDocsMsPackagesFn = "Update-${Language}-DocsMsPackages" From 300781626799665afbff4002ceca9cab738c3f09 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Tue, 16 Jul 2024 10:52:06 -0700 Subject: [PATCH 4/9] remove debug statements --- eng/common/scripts/Save-Package-Properties.ps1 | 2 -- 1 file changed, 2 deletions(-) diff --git a/eng/common/scripts/Save-Package-Properties.ps1 b/eng/common/scripts/Save-Package-Properties.ps1 index faf6640f6..8fea77067 100644 --- a/eng/common/scripts/Save-Package-Properties.ps1 +++ b/eng/common/scripts/Save-Package-Properties.ps1 @@ -102,8 +102,6 @@ else { $allPackageProperties = Get-AllPkgProperties $serviceInput } -Write-Host $allPackageProperties - if ($allPackageProperties) { if (-not (Test-Path -Path $outDirectory)) From 9dd84fd8f2649891538454aca6b776b79279ee18 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Tue, 16 Jul 2024 11:34:02 -0700 Subject: [PATCH 5/9] adjustments to call Save-Package-Properties from other eng/common calls --- .../pipelines/templates/steps/daily-dev-build-variable.yml | 2 +- eng/common/scripts/Verify-RestApiSpecLocation.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml b/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml index 37efd0bd0..ed34d8e64 100644 --- a/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml +++ b/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml @@ -9,7 +9,7 @@ steps: inputs: filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 arguments: > - -ServiceDirectory ${{parameters.ServiceDirectory}} + -ServiceInput ${{parameters.ServiceDirectory}} -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo pwsh: true workingDirectory: $(Pipeline.Workspace) diff --git a/eng/common/scripts/Verify-RestApiSpecLocation.ps1 b/eng/common/scripts/Verify-RestApiSpecLocation.ps1 index 8a420bd76..f5bb1cef5 100644 --- a/eng/common/scripts/Verify-RestApiSpecLocation.ps1 +++ b/eng/common/scripts/Verify-RestApiSpecLocation.ps1 @@ -169,7 +169,7 @@ function Verify-PackageVersion() { if (-not (Test-Path -Path $PackageInfoDirectory)) { # Call Save-Package-Properties.ps1 script to generate package info json files $savePropertiesScriptPath = Join-Path -Path $PSScriptRoot "Save-Package-Properties.ps1" - & $savePropertiesScriptPath -serviceDirectory $ServiceDirectory -outDirectory $PackageInfoDirectory + & $savePropertiesScriptPath -serviceInput $ServiceDirectory -outDirectory $PackageInfoDirectory } } $pkgPropPath = Join-Path -Path $PackageInfoDirectory "$PackageName.json" From 342bc466add6636fe04b27069103965eb567ac46 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Wed, 17 Jul 2024 12:11:21 -0700 Subject: [PATCH 6/9] just check for file under pkg folder --- .../steps/daily-dev-build-variable.yml | 2 +- eng/common/scripts/Package-Properties.ps1 | 18 ++++-------------- eng/common/scripts/Save-Package-Properties.ps1 | 17 ++++++++++------- .../scripts/Verify-RestApiSpecLocation.ps1 | 2 +- 4 files changed, 16 insertions(+), 23 deletions(-) diff --git a/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml b/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml index ed34d8e64..37efd0bd0 100644 --- a/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml +++ b/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml @@ -9,7 +9,7 @@ steps: inputs: filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 arguments: > - -ServiceInput ${{parameters.ServiceDirectory}} + -ServiceDirectory ${{parameters.ServiceDirectory}} -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo pwsh: true workingDirectory: $(Pipeline.Workspace) diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index a9915101c..cec8898d5 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -105,21 +105,10 @@ function Get-PkgProperties return $null } -function Get-PackageFolderFromPath($file) { - $parts = $file -split "/" - - if ($parts.Length -lt 3) { - return "" - } - return [System.IO.Path]::Combine($parts[0], $parts[1], $parts[2]) -} - -# Takes an input diff json file generated from eng/common/scripts/Generate-PR-Diff.ps1 -# On PR builds, this function should be utilized instead of Get-AllPkgProperties. function Get-PrPkgProperties([string]$InputDiffJson) { $packagesWithChanges = @() - $allPackageProperties = Get-AllPkgProperties + $allPackageProperties = Get-AllPkgProperties -ServiceDirectory media $diff = Get-Content $InputDiffJson | ConvertFrom-Json $targetedFiles = $diff.ChangedFiles @@ -127,8 +116,9 @@ function Get-PrPkgProperties([string]$InputDiffJson) { { foreach($file in $targetedFiles) { - $pkgFolder = Get-PackageFolderFromPath($file) - if ($pkgFolder.EndsWith($pkg.ArtifactName)) { + $filePath = Resolve-Path (Join-Path $RepoRoot $file) + $shouldInclude = $filePath -like "$($pkg.DirectoryPath)*" + if ($shouldInclude) { $packagesWithChanges += $pkg } } diff --git a/eng/common/scripts/Save-Package-Properties.ps1 b/eng/common/scripts/Save-Package-Properties.ps1 index 8fea77067..b27567f03 100644 --- a/eng/common/scripts/Save-Package-Properties.ps1 +++ b/eng/common/scripts/Save-Package-Properties.ps1 @@ -14,8 +14,11 @@ In cases of collisions where track 2 packages (IsNewSdk = true) have the same filename as track 1 packages (e.g. same artifact name or package name), the track 2 package properties will be written. -.PARAMETER serviceInput -Service directory in which to search for packages, or file path ending in diff.json. +.PARAMETER serviceDirectory +Service directory in which to search for packages. + +.PARAMETER prDiff +A file path leading to a file generated from Generate-PR-Diff.json. This parameter takes precedence over serviceDirectory, do not provide both. .PARAMETER outDirectory Output location (generally a package artifact directory in DevOps) for JSON @@ -32,10 +35,10 @@ Verison property in that file. [CmdletBinding()] Param ( - [Parameter(Mandatory=$True)] - [string] $serviceInput, + [string] $serviceDirectory, [Parameter(Mandatory=$True)] [string] $outDirectory, + [string] $prDiff, [switch] $addDevVersion ) @@ -95,11 +98,11 @@ $exportedPaths = @{} $allPackageProperties = @() -if ($serviceInput.endswith("diff.json")) { - $allPackageProperties = Get-PrPkgProperties $serviceInput +if ($prDiff) { + $allPackageProperties = Get-PrPkgProperties $prDiff } else { - $allPackageProperties = Get-AllPkgProperties $serviceInput + $allPackageProperties = Get-AllPkgProperties $serviceDirectory } if ($allPackageProperties) diff --git a/eng/common/scripts/Verify-RestApiSpecLocation.ps1 b/eng/common/scripts/Verify-RestApiSpecLocation.ps1 index f5bb1cef5..8a420bd76 100644 --- a/eng/common/scripts/Verify-RestApiSpecLocation.ps1 +++ b/eng/common/scripts/Verify-RestApiSpecLocation.ps1 @@ -169,7 +169,7 @@ function Verify-PackageVersion() { if (-not (Test-Path -Path $PackageInfoDirectory)) { # Call Save-Package-Properties.ps1 script to generate package info json files $savePropertiesScriptPath = Join-Path -Path $PSScriptRoot "Save-Package-Properties.ps1" - & $savePropertiesScriptPath -serviceInput $ServiceDirectory -outDirectory $PackageInfoDirectory + & $savePropertiesScriptPath -serviceDirectory $ServiceDirectory -outDirectory $PackageInfoDirectory } } $pkgPropPath = Join-Path -Path $PackageInfoDirectory "$PackageName.json" From 1fefead66aea6920061b822fecdf276e9d67c446 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Wed, 17 Jul 2024 14:10:28 -0700 Subject: [PATCH 7/9] a debug value was left behind. remove it --- eng/common/scripts/Package-Properties.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index cec8898d5..040d012ed 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -108,7 +108,7 @@ function Get-PkgProperties function Get-PrPkgProperties([string]$InputDiffJson) { $packagesWithChanges = @() - $allPackageProperties = Get-AllPkgProperties -ServiceDirectory media + $allPackageProperties = Get-AllPkgProperties $diff = Get-Content $InputDiffJson | ConvertFrom-Json $targetedFiles = $diff.ChangedFiles From c5bd839bd0841b8306072c276b6dc1a62a727306 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Wed, 17 Jul 2024 14:14:06 -0700 Subject: [PATCH 8/9] Update eng/common/scripts/Save-Package-Properties.ps1 Co-authored-by: Wes Haggard --- eng/common/scripts/Save-Package-Properties.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/Save-Package-Properties.ps1 b/eng/common/scripts/Save-Package-Properties.ps1 index b27567f03..c234d8b28 100644 --- a/eng/common/scripts/Save-Package-Properties.ps1 +++ b/eng/common/scripts/Save-Package-Properties.ps1 @@ -149,6 +149,6 @@ if ($allPackageProperties) } else { - Write-Error "Package properties are not available for service directory $($serviceInput)" + Write-Error "Package properties are not available for service directory $serviceDirectory or $prdiff" exit 1 } From d6482c8c6135370e4215b1de21b7e63f1f066331 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Wed, 17 Jul 2024 14:19:56 -0700 Subject: [PATCH 9/9] resolve the path always --- eng/common/scripts/Package-Properties.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index 040d012ed..0e703a9d4 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -114,10 +114,12 @@ function Get-PrPkgProperties([string]$InputDiffJson) { foreach($pkg in $allPackageProperties) { + $pkgDirectory = Resolve-Path "$($pkg.DirectoryPath)" + foreach($file in $targetedFiles) { $filePath = Resolve-Path (Join-Path $RepoRoot $file) - $shouldInclude = $filePath -like "$($pkg.DirectoryPath)*" + $shouldInclude = $filePath -like "$pkgDirectory*" if ($shouldInclude) { $packagesWithChanges += $pkg }