From 16fd7b2de162251c6e2397ea20eaf654c159a515 Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Thu, 9 Dec 2021 10:10:52 -0500 Subject: [PATCH 01/21] Automate filing issues for any external package dependency and deprecation --- eng/pipelines/weekly_automation.yml | 15 ++++ eng/scripts/check-external-dependency.ps1 | 100 ++++++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 eng/scripts/check-external-dependency.ps1 diff --git a/eng/pipelines/weekly_automation.yml b/eng/pipelines/weekly_automation.yml index d9e4e18596ca..2bd553d4ed53 100644 --- a/eng/pipelines/weekly_automation.yml +++ b/eng/pipelines/weekly_automation.yml @@ -29,3 +29,18 @@ jobs: PRTitle: "Automatic rush update --full" PRBody: "This is an automatic PR generated weekly with changes from running the command rush update --full" PushArgs: "-f" + + - script: | + rush unlink & gitclean -xdf + displayName: "Rush Unlink" + + - task: PowerShell@2 + displayName: 'Check Dependency warnings and File GitHub Issues' + inputs: + targetType: filePath + filePath: "eng/scripts/check-external-dependency.ps1" + arguments: > + -RepoName: azure-sdk-for-js + -RepoOwner azure + -AuthToken "$(azuresdk-github-pat)" + pwsh: true diff --git a/eng/scripts/check-external-dependency.ps1 b/eng/scripts/check-external-dependency.ps1 new file mode 100644 index 000000000000..d6398bf7bcd2 --- /dev/null +++ b/eng/scripts/check-external-dependency.ps1 @@ -0,0 +1,100 @@ +[CmdletBinding(SupportsShouldProcess = $true)] +param ( + [Parameter(Mandatory = $true)] + [string]$AuthToken, + + [Parameter(Mandatory = $true)] + [string]$RepoOwner, + + [Parameter(Mandatory = $true)] + [string]$RepoName +) + + +$dependencyUpgradeLabel = "dependency-upgrade-required" +$dependencyRegex = "^\+\s(?[\S]*)\s(?[\S]*)\s\((?[0-9\.a-b]*).*\)\s?(?deprecated)?" +$RepoRoot = Resolve-Path -Path "${PSScriptRoot}/../.." +$rushFile = join-Path -Path $RepoRoot "rush.json" +$commonConfigFile = Join-path -Path $RepoRoot "common" "config" "rush" "common-versions.json" + +$EngCommonScriptsPath = Join-Path (Resolve-Path "${PSScriptRoot}/..") "common" "scripts" +Test-Path $EngCommonScriptsPath +. (Join-Path $EngCommonScriptsPath common.ps1) + + +# Check and return if an isue already exists to upgrade the package +function Get-GithubIssue($PackageName, $IsDeprecated) { + $issueTitle = "Dependency package $PackageName has a new version available" + $issues = Get-GitHubIssues -RepoOwner $RepoOwner -RepoName $RepoName -CreatedBy "praveenkuttappan" -Labels "dependency-upgrade-required" -AuthToken $AuthToken + if ($issues) { + foreach ($issue in $issues) { + if ($issue.title -eq $issueTitle) { + return $issue + } + } + } + return $issues +} + +# Create new issue if none exists for the package. or update current one if an issue exists +function Set-GitHubIssue($Package) { + $pkgName = $Package.Name + $issueTitle = "Dependency package $pkgName has a new version available" + $issueDesc = "We have identified a dependency on $pkgName ($($Package.OldVersion)). " + + if ($Package.IsDeprecated) { + $issueDesc = "Version $($Package.OldVersion) of $pkgName has been deprecated.`n" + } + $issueDesc += "A new version ($($Package.NewVersion) is available now." + + + $issue = Get-GithubIssue -PackageName $Package.Name -IsDeprecated $Package.IsDeprecated + if ($issue) { + Update-GitHubIssue -RepoOwner $RepoOwner -RepoName $RepoName -AuthToken $AuthToken -IssueNumber $issue.number -Body $issueDesc + } + else { + $newIssue = New-GitHubIssue -RepoOwner $RepoOwner -RepoName $RepoName -AuthToken $AuthToken -Title $issueTitle -Description $issueDesc + Write-Host "Created issue $($newIssue.number) with title '$issueTitle'" + if ($newIssue) { + $out = Add-GitHubIssueLabels -RepoOwner $RepoOwner -RepoName $RepoName -AuthToken $AuthToken -Labels $dependencyUpgradeLabel -IssueNumber $newIssue.number + Write-Host "Addd label $dependencyUpgradeLabel to issue $($newIssue.number)" + } + } +} + + +# Update rush configuration files to alter settings +if ((Test-Path $rushFile) -and (Test-Path $commonConfigFile)) { + $rushJson = Get-Content -Path $rushFile | ConvertFrom-Json + $rushJson.pnpmOptions.strictPeerDependencies = $false + Set-Content -Path $rushFile -Value (ConvertTo-Json -InputObject $rushJson) + + $configJson = Get-Content -Path $commonConfigFile | ConvertFrom-Json + $configJson.implicitlyPreferredVersions = $true + Set-Content -Path $commonConfigFile -Value (ConvertTo-Json -InputObject $configJson) +} +else { + Write-Error "Failed to find $($rushFile) and/or $($commonConfigFile). Verify repo root parameter." + exit +} + +# Run rush update --full +rush update --full > rush.out +if (Test-Path "rush.out") { + $rushUpdateOutput = Get-Content -Path "rush.out" + foreach ($line in $rushUpdateOutput) { + if ($line -match $dependencyRegex -and !$matches['pkg'].StartsWith("@azure")) { + $p = New-Object PSObject -Property @{ + Name = $matches['pkg'] + OldVersion = [AzureEngSemanticVersion]::ParseVersionString($matches['version']) + NewVersion = [AzureEngSemanticVersion]::ParseVersionString($matches['newVersion']) + IsDeprecated = ($matches['deprecated'] -eq "deprecated") + } + Set-GitHubIssue -Package $p + } + } +} +else { + Write-Error "Failed to run rush update --full" + exit +} From 769babf6cb50774d0e50aa69bb989ff7abc1038d Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Fri, 10 Dec 2021 11:09:45 -0500 Subject: [PATCH 02/21] Revert temporary config changes --- eng/scripts/check-external-dependency.ps1 | 45 ++++++++++++++--------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/eng/scripts/check-external-dependency.ps1 b/eng/scripts/check-external-dependency.ps1 index d6398bf7bcd2..6c9919434f96 100644 --- a/eng/scripts/check-external-dependency.ps1 +++ b/eng/scripts/check-external-dependency.ps1 @@ -18,14 +18,13 @@ $rushFile = join-Path -Path $RepoRoot "rush.json" $commonConfigFile = Join-path -Path $RepoRoot "common" "config" "rush" "common-versions.json" $EngCommonScriptsPath = Join-Path (Resolve-Path "${PSScriptRoot}/..") "common" "scripts" -Test-Path $EngCommonScriptsPath . (Join-Path $EngCommonScriptsPath common.ps1) # Check and return if an isue already exists to upgrade the package function Get-GithubIssue($PackageName, $IsDeprecated) { $issueTitle = "Dependency package $PackageName has a new version available" - $issues = Get-GitHubIssues -RepoOwner $RepoOwner -RepoName $RepoName -CreatedBy "praveenkuttappan" -Labels "dependency-upgrade-required" -AuthToken $AuthToken + $issues = Get-GitHubIssues -RepoOwner $RepoOwner -RepoName $RepoName -CreatedBy "azure-sdk" -Labels "dependency-upgrade-required" -AuthToken $AuthToken if ($issues) { foreach ($issue in $issues) { if ($issue.title -eq $issueTitle) { @@ -78,23 +77,33 @@ else { exit } -# Run rush update --full -rush update --full > rush.out -if (Test-Path "rush.out") { - $rushUpdateOutput = Get-Content -Path "rush.out" - foreach ($line in $rushUpdateOutput) { - if ($line -match $dependencyRegex -and !$matches['pkg'].StartsWith("@azure")) { - $p = New-Object PSObject -Property @{ - Name = $matches['pkg'] - OldVersion = [AzureEngSemanticVersion]::ParseVersionString($matches['version']) - NewVersion = [AzureEngSemanticVersion]::ParseVersionString($matches['newVersion']) - IsDeprecated = ($matches['deprecated'] -eq "deprecated") + +try { + # Run rush update --full + rush update --full > rush.out + if (Test-Path "rush.out") { + $rushUpdateOutput = Get-Content -Path "rush.out" + foreach ($line in $rushUpdateOutput) { + if ($line -match $dependencyRegex -and !$matches['pkg'].StartsWith("@azure")) { + $p = New-Object PSObject -Property @{ + Name = $matches['pkg'] + OldVersion = [AzureEngSemanticVersion]::ParseVersionString($matches['version']) + NewVersion = [AzureEngSemanticVersion]::ParseVersionString($matches['newVersion']) + IsDeprecated = ($matches['deprecated'] -eq "deprecated") + } + Set-GitHubIssue -Package $p } - Set-GitHubIssue -Package $p } } + else { + Write-Error "Failed to run rush update --full" + exit 1 + } +} +catch { + Write-Error "Failed to check available new versions of external dependency." + exit 1 +} +finally { + rush unlink & gitclean -xdf } -else { - Write-Error "Failed to run rush update --full" - exit -} From 6ae09ffc79ddfea59ac19d28a2508a1ce61444bd Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Fri, 10 Dec 2021 11:22:55 -0500 Subject: [PATCH 03/21] Additional fixes --- eng/pipelines/weekly_automation.yml | 2 +- eng/scripts/check-external-dependency.ps1 | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/weekly_automation.yml b/eng/pipelines/weekly_automation.yml index 2bd553d4ed53..0806e58f6432 100644 --- a/eng/pipelines/weekly_automation.yml +++ b/eng/pipelines/weekly_automation.yml @@ -31,7 +31,7 @@ jobs: PushArgs: "-f" - script: | - rush unlink & gitclean -xdf + rush unlink & git clean -xdf displayName: "Rush Unlink" - task: PowerShell@2 diff --git a/eng/scripts/check-external-dependency.ps1 b/eng/scripts/check-external-dependency.ps1 index 6c9919434f96..ffdbfe0a3302 100644 --- a/eng/scripts/check-external-dependency.ps1 +++ b/eng/scripts/check-external-dependency.ps1 @@ -74,7 +74,7 @@ if ((Test-Path $rushFile) -and (Test-Path $commonConfigFile)) { } else { Write-Error "Failed to find $($rushFile) and/or $($commonConfigFile). Verify repo root parameter." - exit + exit 1 } @@ -105,5 +105,5 @@ catch { exit 1 } finally { - rush unlink & gitclean -xdf + rush unlink & git clean -xdf } From e4548b3e76cd20011257b0b7c2ab706a6bc409f9 Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Fri, 10 Dec 2021 13:38:47 -0500 Subject: [PATCH 04/21] Additional change as per review suggestion --- eng/scripts/check-external-dependency.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/scripts/check-external-dependency.ps1 b/eng/scripts/check-external-dependency.ps1 index ffdbfe0a3302..65844b1bf812 100644 --- a/eng/scripts/check-external-dependency.ps1 +++ b/eng/scripts/check-external-dependency.ps1 @@ -10,6 +10,7 @@ param ( [string]$RepoName ) +# This script will reset the repo and any changes in tracked files will be lost. $dependencyUpgradeLabel = "dependency-upgrade-required" $dependencyRegex = "^\+\s(?[\S]*)\s(?[\S]*)\s\((?[0-9\.a-b]*).*\)\s?(?deprecated)?" @@ -105,5 +106,6 @@ catch { exit 1 } finally { + git reset --hard rush unlink & git clean -xdf } From 531599b63a2b9060fe7a30e95109e7cbf128fdaa Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Fri, 10 Dec 2021 13:45:34 -0500 Subject: [PATCH 05/21] Fix pipeline errors --- eng/pipelines/weekly_automation.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/weekly_automation.yml b/eng/pipelines/weekly_automation.yml index 0806e58f6432..6d9e8bc1a057 100644 --- a/eng/pipelines/weekly_automation.yml +++ b/eng/pipelines/weekly_automation.yml @@ -35,12 +35,12 @@ jobs: displayName: "Rush Unlink" - task: PowerShell@2 - displayName: 'Check Dependency warnings and File GitHub Issues' - inputs: - targetType: filePath - filePath: "eng/scripts/check-external-dependency.ps1" - arguments: > - -RepoName: azure-sdk-for-js - -RepoOwner azure - -AuthToken "$(azuresdk-github-pat)" - pwsh: true + displayName: 'Check Dependency Warnings and File GitHub Issues' + inputs: + targetType: filePath + filePath: "eng/scripts/check-external-dependency.ps1" + arguments: > + -RepoName: azure-sdk-for-js + -RepoOwner azure + -AuthToken "$(azuresdk-github-pat)" + pwsh: true From 8580f6babe1c5d5b63afdb303db831555f82a6d9 Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Fri, 10 Dec 2021 14:04:00 -0500 Subject: [PATCH 06/21] Fix pipeline errors --- eng/pipelines/weekly_automation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/weekly_automation.yml b/eng/pipelines/weekly_automation.yml index 6d9e8bc1a057..11c4c1224fd2 100644 --- a/eng/pipelines/weekly_automation.yml +++ b/eng/pipelines/weekly_automation.yml @@ -40,7 +40,7 @@ jobs: targetType: filePath filePath: "eng/scripts/check-external-dependency.ps1" arguments: > - -RepoName: azure-sdk-for-js + -RepoName azure-sdk-for-js -RepoOwner azure -AuthToken "$(azuresdk-github-pat)" pwsh: true From 442612aeddd717f93af4fbebb726eb0f7239e987 Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Fri, 10 Dec 2021 14:11:51 -0500 Subject: [PATCH 07/21] Update path to script --- eng/pipelines/weekly_automation.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/weekly_automation.yml b/eng/pipelines/weekly_automation.yml index 11c4c1224fd2..ebf4152c4233 100644 --- a/eng/pipelines/weekly_automation.yml +++ b/eng/pipelines/weekly_automation.yml @@ -38,9 +38,10 @@ jobs: displayName: 'Check Dependency Warnings and File GitHub Issues' inputs: targetType: filePath - filePath: "eng/scripts/check-external-dependency.ps1" + filePath: $(Build.SourcesDirectory)/eng/scripts/check-external-dependency.ps1 arguments: > -RepoName azure-sdk-for-js -RepoOwner azure -AuthToken "$(azuresdk-github-pat)" + workingDirectory: $(Pipeline.Workspace) pwsh: true From e99416a2bff908d1f8ef34a78e66df468751eff5 Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Fri, 10 Dec 2021 14:18:53 -0500 Subject: [PATCH 08/21] Update path to script --- eng/pipelines/weekly_automation.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/eng/pipelines/weekly_automation.yml b/eng/pipelines/weekly_automation.yml index ebf4152c4233..3698c6029d38 100644 --- a/eng/pipelines/weekly_automation.yml +++ b/eng/pipelines/weekly_automation.yml @@ -30,6 +30,20 @@ jobs: PRBody: "This is an automatic PR generated weekly with changes from running the command rush update --full" PushArgs: "-f" + +- job: 'CheckDependency' + displayName: 'Check Package Dependency' + variables: + - template: /eng/pipelines/templates/variables/globals.yml + + pool: + name: azsdk-pool-mms-ubuntu-2004-general + vmImage: MMSUbuntu20.04 + + steps: + - template: /eng/pipelines/templates/steps/common.yml + - template: /eng/common/pipelines/templates/steps/set-default-branch.yml + - script: | rush unlink & git clean -xdf displayName: "Rush Unlink" From bf21a4190486b342860585ef400a41afe1a4e54d Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Fri, 10 Dec 2021 14:29:07 -0500 Subject: [PATCH 09/21] Logging changes --- eng/scripts/check-external-dependency.ps1 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eng/scripts/check-external-dependency.ps1 b/eng/scripts/check-external-dependency.ps1 index 65844b1bf812..6684d4be5da0 100644 --- a/eng/scripts/check-external-dependency.ps1 +++ b/eng/scripts/check-external-dependency.ps1 @@ -15,8 +15,11 @@ param ( $dependencyUpgradeLabel = "dependency-upgrade-required" $dependencyRegex = "^\+\s(?[\S]*)\s(?[\S]*)\s\((?[0-9\.a-b]*).*\)\s?(?deprecated)?" $RepoRoot = Resolve-Path -Path "${PSScriptRoot}/../.." +Write-Host "Repo root: $RepoRoot" $rushFile = join-Path -Path $RepoRoot "rush.json" +Write-Host "Path to rush.json: $rushFile" $commonConfigFile = Join-path -Path $RepoRoot "common" "config" "rush" "common-versions.json" +Write-Host "Path to common-versions.json: $commonConfigFile" $EngCommonScriptsPath = Join-Path (Resolve-Path "${PSScriptRoot}/..") "common" "scripts" . (Join-Path $EngCommonScriptsPath common.ps1) @@ -77,10 +80,11 @@ else { Write-Error "Failed to find $($rushFile) and/or $($commonConfigFile). Verify repo root parameter." exit 1 } - +Write-Host "Updated rush configuraion files" try { # Run rush update --full + Write-Host "Running rush update" rush update --full > rush.out if (Test-Path "rush.out") { $rushUpdateOutput = Get-Content -Path "rush.out" From 5cbb21fc9782fdafde144f003eb39aab6c2d3934 Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Fri, 10 Dec 2021 14:35:43 -0500 Subject: [PATCH 10/21] Logging changes --- eng/pipelines/weekly_automation.yml | 9 +++++++-- eng/scripts/check-external-dependency.ps1 | 4 ---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/weekly_automation.yml b/eng/pipelines/weekly_automation.yml index 3698c6029d38..194c4d3035fc 100644 --- a/eng/pipelines/weekly_automation.yml +++ b/eng/pipelines/weekly_automation.yml @@ -37,8 +37,8 @@ jobs: - template: /eng/pipelines/templates/variables/globals.yml pool: - name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 + name: azsdk-pool-mms-win-2019-general + vmImage: MMS2019 steps: - template: /eng/pipelines/templates/steps/common.yml @@ -59,3 +59,8 @@ jobs: -AuthToken "$(azuresdk-github-pat)" workingDirectory: $(Pipeline.Workspace) pwsh: true + + - script: | + git reset --hard + rush unlink & git clean -xdf + displayName: "Reset Changes" diff --git a/eng/scripts/check-external-dependency.ps1 b/eng/scripts/check-external-dependency.ps1 index 6684d4be5da0..74777b672cbf 100644 --- a/eng/scripts/check-external-dependency.ps1 +++ b/eng/scripts/check-external-dependency.ps1 @@ -109,7 +109,3 @@ catch { Write-Error "Failed to check available new versions of external dependency." exit 1 } -finally { - git reset --hard - rush unlink & git clean -xdf -} From 1c97629655090677960308080abe52109b46b5ac Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Fri, 10 Dec 2021 14:43:35 -0500 Subject: [PATCH 11/21] pipeline changs --- eng/pipelines/weekly_automation.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/eng/pipelines/weekly_automation.yml b/eng/pipelines/weekly_automation.yml index 194c4d3035fc..e6b148b16f68 100644 --- a/eng/pipelines/weekly_automation.yml +++ b/eng/pipelines/weekly_automation.yml @@ -41,9 +41,6 @@ jobs: vmImage: MMS2019 steps: - - template: /eng/pipelines/templates/steps/common.yml - - template: /eng/common/pipelines/templates/steps/set-default-branch.yml - - script: | rush unlink & git clean -xdf displayName: "Rush Unlink" From 5698d56acb7dd3172751de1dbf95ca38d4f60068 Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Fri, 10 Dec 2021 14:55:43 -0500 Subject: [PATCH 12/21] Update script to avoid create out file --- eng/pipelines/weekly_automation.yml | 1 - eng/scripts/check-external-dependency.ps1 | 27 +++++++++-------------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/eng/pipelines/weekly_automation.yml b/eng/pipelines/weekly_automation.yml index e6b148b16f68..f48277a11455 100644 --- a/eng/pipelines/weekly_automation.yml +++ b/eng/pipelines/weekly_automation.yml @@ -59,5 +59,4 @@ jobs: - script: | git reset --hard - rush unlink & git clean -xdf displayName: "Reset Changes" diff --git a/eng/scripts/check-external-dependency.ps1 b/eng/scripts/check-external-dependency.ps1 index 74777b672cbf..978961e46c88 100644 --- a/eng/scripts/check-external-dependency.ps1 +++ b/eng/scripts/check-external-dependency.ps1 @@ -85,25 +85,20 @@ Write-Host "Updated rush configuraion files" try { # Run rush update --full Write-Host "Running rush update" - rush update --full > rush.out - if (Test-Path "rush.out") { - $rushUpdateOutput = Get-Content -Path "rush.out" - foreach ($line in $rushUpdateOutput) { - if ($line -match $dependencyRegex -and !$matches['pkg'].StartsWith("@azure")) { - $p = New-Object PSObject -Property @{ - Name = $matches['pkg'] - OldVersion = [AzureEngSemanticVersion]::ParseVersionString($matches['version']) - NewVersion = [AzureEngSemanticVersion]::ParseVersionString($matches['newVersion']) - IsDeprecated = ($matches['deprecated'] -eq "deprecated") - } - Set-GitHubIssue -Package $p + $rushUpdateOutput = rush update --full + Write-Host "Parsing rush update out" + write-host $rushUpdateOutput + foreach ($line in $rushUpdateOutput) { + if ($line -match $dependencyRegex -and !$matches['pkg'].StartsWith("@azure")) { + $p = New-Object PSObject -Property @{ + Name = $matches['pkg'] + OldVersion = [AzureEngSemanticVersion]::ParseVersionString($matches['version']) + NewVersion = [AzureEngSemanticVersion]::ParseVersionString($matches['newVersion']) + IsDeprecated = ($matches['deprecated'] -eq "deprecated") } + Set-GitHubIssue -Package $p } } - else { - Write-Error "Failed to run rush update --full" - exit 1 - } } catch { Write-Error "Failed to check available new versions of external dependency." From 1f80c09e927d19c2a8d74f1112df5740cf02ec09 Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Fri, 10 Dec 2021 15:29:03 -0500 Subject: [PATCH 13/21] Update script to avoid create out file --- eng/pipelines/weekly_automation.yml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/eng/pipelines/weekly_automation.yml b/eng/pipelines/weekly_automation.yml index f48277a11455..8059f7db74bd 100644 --- a/eng/pipelines/weekly_automation.yml +++ b/eng/pipelines/weekly_automation.yml @@ -20,15 +20,6 @@ jobs: node common/scripts/install-run-rush.js update --full displayName: "Run Rush Update" - - template: /eng/common/pipelines/templates/steps/create-pull-request.yml - parameters: - RepoName: azure-sdk-for-js - BaseBranchName: $(DefaultBranch) - PRBranchName: automated-rush-update-full - CommitMsg: "Automatic rush update --full" - PRTitle: "Automatic rush update --full" - PRBody: "This is an automatic PR generated weekly with changes from running the command rush update --full" - PushArgs: "-f" - job: 'CheckDependency' @@ -37,10 +28,13 @@ jobs: - template: /eng/pipelines/templates/variables/globals.yml pool: - name: azsdk-pool-mms-win-2019-general - vmImage: MMS2019 + name: azsdk-pool-mms-ubuntu-2004-general + vmImage: MMSUbuntu20.04 steps: + - template: /eng/pipelines/templates/steps/common.yml + - template: /eng/common/pipelines/templates/steps/set-default-branch.yml + - script: | rush unlink & git clean -xdf displayName: "Rush Unlink" From e5db6dd1504cb62e59e55c374e33f954c6f4f290 Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Fri, 10 Dec 2021 15:34:57 -0500 Subject: [PATCH 14/21] Update script to avoid create out file --- eng/pipelines/weekly_automation.yml | 4 +++- eng/scripts/check-external-dependency.ps1 | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/weekly_automation.yml b/eng/pipelines/weekly_automation.yml index 8059f7db74bd..54dfb6555be8 100644 --- a/eng/pipelines/weekly_automation.yml +++ b/eng/pipelines/weekly_automation.yml @@ -36,7 +36,9 @@ jobs: - template: /eng/common/pipelines/templates/steps/set-default-branch.yml - script: | - rush unlink & git clean -xdf + node common/scripts/install-run-rush.js install + node common/scripts/install-run-rush.js unlink + git clean -xdf displayName: "Rush Unlink" - task: PowerShell@2 diff --git a/eng/scripts/check-external-dependency.ps1 b/eng/scripts/check-external-dependency.ps1 index 978961e46c88..5aa858155dd2 100644 --- a/eng/scripts/check-external-dependency.ps1 +++ b/eng/scripts/check-external-dependency.ps1 @@ -85,7 +85,7 @@ Write-Host "Updated rush configuraion files" try { # Run rush update --full Write-Host "Running rush update" - $rushUpdateOutput = rush update --full + $rushUpdateOutput = node common/scripts/install-run-rush.js update --full Write-Host "Parsing rush update out" write-host $rushUpdateOutput foreach ($line in $rushUpdateOutput) { From 256682a83e362ba08f5c30f8e032a792795852b1 Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Fri, 10 Dec 2021 15:42:14 -0500 Subject: [PATCH 15/21] Update script to avoid create out file --- eng/pipelines/weekly_automation.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/pipelines/weekly_automation.yml b/eng/pipelines/weekly_automation.yml index 54dfb6555be8..34e374f9074e 100644 --- a/eng/pipelines/weekly_automation.yml +++ b/eng/pipelines/weekly_automation.yml @@ -50,7 +50,6 @@ jobs: -RepoName azure-sdk-for-js -RepoOwner azure -AuthToken "$(azuresdk-github-pat)" - workingDirectory: $(Pipeline.Workspace) pwsh: true - script: | From cbc10fbcb481a801dca33563d3bf87240fc61c55 Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Fri, 10 Dec 2021 15:50:22 -0500 Subject: [PATCH 16/21] Update script to avoid create out file --- eng/scripts/check-external-dependency.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/scripts/check-external-dependency.ps1 b/eng/scripts/check-external-dependency.ps1 index 5aa858155dd2..089996f8f8c4 100644 --- a/eng/scripts/check-external-dependency.ps1 +++ b/eng/scripts/check-external-dependency.ps1 @@ -36,7 +36,7 @@ function Get-GithubIssue($PackageName, $IsDeprecated) { } } } - return $issues + return $null } # Create new issue if none exists for the package. or update current one if an issue exists From b8569867f9ce0005990a4f2161b84303222dfb6b Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Fri, 10 Dec 2021 16:02:53 -0500 Subject: [PATCH 17/21] Script changes --- eng/scripts/check-external-dependency.ps1 | 38 ++++++++++------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/eng/scripts/check-external-dependency.ps1 b/eng/scripts/check-external-dependency.ps1 index 089996f8f8c4..ece5009a8bd8 100644 --- a/eng/scripts/check-external-dependency.ps1 +++ b/eng/scripts/check-external-dependency.ps1 @@ -48,7 +48,7 @@ function Set-GitHubIssue($Package) { if ($Package.IsDeprecated) { $issueDesc = "Version $($Package.OldVersion) of $pkgName has been deprecated.`n" } - $issueDesc += "A new version ($($Package.NewVersion) is available now." + $issueDesc += "A new version ($($Package.NewVersion)) is available now." $issue = Get-GithubIssue -PackageName $Package.Name -IsDeprecated $Package.IsDeprecated @@ -60,7 +60,7 @@ function Set-GitHubIssue($Package) { Write-Host "Created issue $($newIssue.number) with title '$issueTitle'" if ($newIssue) { $out = Add-GitHubIssueLabels -RepoOwner $RepoOwner -RepoName $RepoName -AuthToken $AuthToken -Labels $dependencyUpgradeLabel -IssueNumber $newIssue.number - Write-Host "Addd label $dependencyUpgradeLabel to issue $($newIssue.number)" + Write-Host "Added label $dependencyUpgradeLabel to issue $($newIssue.number)" } } } @@ -81,26 +81,20 @@ else { exit 1 } Write-Host "Updated rush configuraion files" - -try { - # Run rush update --full - Write-Host "Running rush update" - $rushUpdateOutput = node common/scripts/install-run-rush.js update --full - Write-Host "Parsing rush update out" - write-host $rushUpdateOutput - foreach ($line in $rushUpdateOutput) { - if ($line -match $dependencyRegex -and !$matches['pkg'].StartsWith("@azure")) { - $p = New-Object PSObject -Property @{ - Name = $matches['pkg'] - OldVersion = [AzureEngSemanticVersion]::ParseVersionString($matches['version']) - NewVersion = [AzureEngSemanticVersion]::ParseVersionString($matches['newVersion']) - IsDeprecated = ($matches['deprecated'] -eq "deprecated") - } - Set-GitHubIssue -Package $p +# Run rush update --full +Write-Host "Running rush update" +$rushUpdateOutput = node common/scripts/install-run-rush.js update --full +Write-Host "Parsing rush update out" +write-host $rushUpdateOutput +foreach ($line in $rushUpdateOutput) { + if ($line -match $dependencyRegex -and !$matches['pkg'].StartsWith("@azure")) { + $p = New-Object PSObject -Property @{ + Name = $matches['pkg'] + OldVersion = [AzureEngSemanticVersion]::ParseVersionString($matches['version']) + NewVersion = [AzureEngSemanticVersion]::ParseVersionString($matches['newVersion']) + IsDeprecated = ($matches['deprecated'] -eq "deprecated") } + Set-GitHubIssue -Package $p } } -catch { - Write-Error "Failed to check available new versions of external dependency." - exit 1 -} +Write-Host "Verified and filed issues" From 5ea84a918d65bf61200ef1aa0bda3452d7f8fcd4 Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Fri, 10 Dec 2021 16:05:19 -0500 Subject: [PATCH 18/21] Script changes --- eng/scripts/check-external-dependency.ps1 | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/eng/scripts/check-external-dependency.ps1 b/eng/scripts/check-external-dependency.ps1 index ece5009a8bd8..c95aa29b05e5 100644 --- a/eng/scripts/check-external-dependency.ps1 +++ b/eng/scripts/check-external-dependency.ps1 @@ -43,7 +43,7 @@ function Get-GithubIssue($PackageName, $IsDeprecated) { function Set-GitHubIssue($Package) { $pkgName = $Package.Name $issueTitle = "Dependency package $pkgName has a new version available" - $issueDesc = "We have identified a dependency on $pkgName ($($Package.OldVersion)). " + $issueDesc = "We have identified a dependency on $pkgName ($($Package.OldVersion))." if ($Package.IsDeprecated) { $issueDesc = "Version $($Package.OldVersion) of $pkgName has been deprecated.`n" @@ -94,7 +94,10 @@ foreach ($line in $rushUpdateOutput) { NewVersion = [AzureEngSemanticVersion]::ParseVersionString($matches['newVersion']) IsDeprecated = ($matches['deprecated'] -eq "deprecated") } - Set-GitHubIssue -Package $p + + if ($null -ne $p.OldVersion -and $null -ne $p.NewVersion) { + Set-GitHubIssue -Package $p + } } } Write-Host "Verified and filed issues" From 7070b3aed0af4bb6152f795ce9798363dcc2586d Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Fri, 10 Dec 2021 16:24:19 -0500 Subject: [PATCH 19/21] Script changes --- eng/scripts/check-external-dependency.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/scripts/check-external-dependency.ps1 b/eng/scripts/check-external-dependency.ps1 index c95aa29b05e5..084103aeb512 100644 --- a/eng/scripts/check-external-dependency.ps1 +++ b/eng/scripts/check-external-dependency.ps1 @@ -53,7 +53,10 @@ function Set-GitHubIssue($Package) { $issue = Get-GithubIssue -PackageName $Package.Name -IsDeprecated $Package.IsDeprecated if ($issue) { - Update-GitHubIssue -RepoOwner $RepoOwner -RepoName $RepoName -AuthToken $AuthToken -IssueNumber $issue.number -Body $issueDesc + if ($issue.body -ne $issueDesc) { + $oldIssue = Update-GitHubIssue -RepoOwner $RepoOwner -RepoName $RepoName -AuthToken $AuthToken -IssueNumber $issue.number -Body $issueDesc + Write-Host "Updated existing issue $($oldIssue.number)" + } } else { $newIssue = New-GitHubIssue -RepoOwner $RepoOwner -RepoName $RepoName -AuthToken $AuthToken -Title $issueTitle -Description $issueDesc From 950192cf1233b194e7a1a5040a0efd985019afab Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Fri, 10 Dec 2021 17:18:21 -0500 Subject: [PATCH 20/21] Additional changes --- eng/scripts/check-external-dependency.ps1 | 30 +++++++++++------------ 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/eng/scripts/check-external-dependency.ps1 b/eng/scripts/check-external-dependency.ps1 index 084103aeb512..06828ef59320 100644 --- a/eng/scripts/check-external-dependency.ps1 +++ b/eng/scripts/check-external-dependency.ps1 @@ -24,16 +24,12 @@ Write-Host "Path to common-versions.json: $commonConfigFile" $EngCommonScriptsPath = Join-Path (Resolve-Path "${PSScriptRoot}/..") "common" "scripts" . (Join-Path $EngCommonScriptsPath common.ps1) - +$ghIssues = Get-GitHubIssues -RepoOwner $RepoOwner -RepoName $RepoName -CreatedBy "azure-sdk" -Labels "dependency-upgrade-required" -AuthToken $AuthToken # Check and return if an isue already exists to upgrade the package -function Get-GithubIssue($PackageName, $IsDeprecated) { - $issueTitle = "Dependency package $PackageName has a new version available" - $issues = Get-GitHubIssues -RepoOwner $RepoOwner -RepoName $RepoName -CreatedBy "azure-sdk" -Labels "dependency-upgrade-required" -AuthToken $AuthToken - if ($issues) { - foreach ($issue in $issues) { - if ($issue.title -eq $issueTitle) { - return $issue - } +function Get-GithubIssue($IssueTitle) { + foreach ($issue in $ghIssues) { + if ($issue.title -eq $IssueTitle) { + return $issue } } return $null @@ -43,22 +39,25 @@ function Get-GithubIssue($PackageName, $IsDeprecated) { function Set-GitHubIssue($Package) { $pkgName = $Package.Name $issueTitle = "Dependency package $pkgName has a new version available" - $issueDesc = "We have identified a dependency on $pkgName ($($Package.OldVersion))." + $issueDesc = "We have identified a dependency on $pkgName ($($Package.OldVersion)). " if ($Package.IsDeprecated) { - $issueDesc = "Version $($Package.OldVersion) of $pkgName has been deprecated.`n" + $issueDesc += "Version $($Package.OldVersion) of $pkgName has been deprecated.`n" } $issueDesc += "A new version ($($Package.NewVersion)) is available now." - - $issue = Get-GithubIssue -PackageName $Package.Name -IsDeprecated $Package.IsDeprecated + $issue = Get-GithubIssue -IssueTitle $issueTitle if ($issue) { if ($issue.body -ne $issueDesc) { $oldIssue = Update-GitHubIssue -RepoOwner $RepoOwner -RepoName $RepoName -AuthToken $AuthToken -IssueNumber $issue.number -Body $issueDesc - Write-Host "Updated existing issue $($oldIssue.number)" - } + Write-Host "Updated existing issue $($oldIssue.number)" + } + else { + Write-Host "Found existing issue for package $($Package.Name)" + } } else { + write-Host "Creating issue for $pkgName" $newIssue = New-GitHubIssue -RepoOwner $RepoOwner -RepoName $RepoName -AuthToken $AuthToken -Title $issueTitle -Description $issueDesc Write-Host "Created issue $($newIssue.number) with title '$issueTitle'" if ($newIssue) { @@ -100,6 +99,7 @@ foreach ($line in $rushUpdateOutput) { if ($null -ne $p.OldVersion -and $null -ne $p.NewVersion) { Set-GitHubIssue -Package $p + Start-Sleep -s 5 } } } From 2dee74e892f42b7c7470e4f72edcb30f090c5cb0 Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Fri, 10 Dec 2021 17:35:53 -0500 Subject: [PATCH 21/21] Revert temp changes --- eng/pipelines/weekly_automation.yml | 9 +++++++++ eng/scripts/check-external-dependency.ps1 | 5 +---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/weekly_automation.yml b/eng/pipelines/weekly_automation.yml index 34e374f9074e..60e94fd495c0 100644 --- a/eng/pipelines/weekly_automation.yml +++ b/eng/pipelines/weekly_automation.yml @@ -20,6 +20,15 @@ jobs: node common/scripts/install-run-rush.js update --full displayName: "Run Rush Update" + - template: /eng/common/pipelines/templates/steps/create-pull-request.yml + parameters: + RepoName: azure-sdk-for-js + BaseBranchName: $(DefaultBranch) + PRBranchName: automated-rush-update-full + CommitMsg: "Automatic rush update --full" + PRTitle: "Automatic rush update --full" + PRBody: "This is an automatic PR generated weekly with changes from running the command rush update --full" + PushArgs: "-f" - job: 'CheckDependency' diff --git a/eng/scripts/check-external-dependency.ps1 b/eng/scripts/check-external-dependency.ps1 index 06828ef59320..4a4cc9fcc8eb 100644 --- a/eng/scripts/check-external-dependency.ps1 +++ b/eng/scripts/check-external-dependency.ps1 @@ -59,10 +59,8 @@ function Set-GitHubIssue($Package) { else { write-Host "Creating issue for $pkgName" $newIssue = New-GitHubIssue -RepoOwner $RepoOwner -RepoName $RepoName -AuthToken $AuthToken -Title $issueTitle -Description $issueDesc - Write-Host "Created issue $($newIssue.number) with title '$issueTitle'" if ($newIssue) { $out = Add-GitHubIssueLabels -RepoOwner $RepoOwner -RepoName $RepoName -AuthToken $AuthToken -Labels $dependencyUpgradeLabel -IssueNumber $newIssue.number - Write-Host "Added label $dependencyUpgradeLabel to issue $($newIssue.number)" } } } @@ -82,11 +80,10 @@ else { Write-Error "Failed to find $($rushFile) and/or $($commonConfigFile). Verify repo root parameter." exit 1 } -Write-Host "Updated rush configuraion files" + # Run rush update --full Write-Host "Running rush update" $rushUpdateOutput = node common/scripts/install-run-rush.js update --full -Write-Host "Parsing rush update out" write-host $rushUpdateOutput foreach ($line in $rushUpdateOutput) { if ($line -match $dependencyRegex -and !$matches['pkg'].StartsWith("@azure")) {