Sync eng/common directory with azure-sdk-tools repository for Tools PR Azure/azure-sdk-tools#926

Author: azure-sdk

eng/common/pipelines/templates/steps/create-pull-request.yml

@@ -74,6 +74,7 @@ steps:
 - task: PowerShell@2
   displayName: Tag a Reviewer on PR
   condition: and(succeeded(), eq(variables['HasChanges'], 'true'))
+  continueOnError: true
   inputs:
     pwsh: true
     workingDirectory: ${{ parameters.WorkingDirectory }}

eng/common/pipelines/templates/steps/get-pr-owners.yml

@@ -18,12 +18,20 @@ steps:
         --kusto-database-var KUSTO_DB `
         --kusto-table-var KUSTO_TABLE `
         --identity "$(Build.QueuedBy)"
-      $resolvedIdentity = $result[-1] | ConvertFrom-Json
 
-      Write-Host $resolvedIdentity
+      $resolvedIdentity = ""
+      try { $resolvedIdentity = $result[-1] | ConvertFrom-Json } catch {}
 
-      Write-Output "##vso[task.setvariable variable=${{ parameters.TargetVariable }}]$($resolvedIdentity.GithubUserName)"
+      if($resolvedIdentity) {
+        Write-Host $resolvedIdentity
+
+        Write-Host "##vso[task.setvariable variable=${{ parameters.TargetVariable }}]$($resolvedIdentity.GithubUserName)"
+      }
+      else {
+        Write-Host "Unable to locate a github user for identity $(Build.QueuedBy)"
+      }
     displayName: 'Resolving Queuing User'
+    continueOnError: true
     workingDirectory: $(Build.SourcesDirectory)/tools_repo/tools/notification-configuration/identity-resolver
     env:
       APP_ID: $(notification-aad-app-id)
@@ -41,6 +49,6 @@ steps:
       $originalValue = "$(${{ parameters.TargetVariable }})"
       $result = $(Build.SourcesDirectory)/eng/common/scripts/get-codeowners.ps1 -TargetDirectory /sdk/${{ parameters.ServiceDirectory }}/ -RootDirectory $(Build.SourcesDirectory)
       if ($result) {
-        Write-Output "##vso[task.setvariable variable=${{ parameters.TargetVariable }}]$originalValue,$result"
+        Write-Host "##vso[task.setvariable variable=${{ parameters.TargetVariable }}]$originalValue,$result"
       }
     displayName: Add CodeOwners if Present

eng/common/pipelines/templates/steps/publish-blobs.yml

@@ -4,6 +4,8 @@ parameters:
   TargetLanguage: ''
   BlobName: ''
   ScriptPath: ''
+  RepoId: ''
+  ReleaseTag: ''
 
 steps:
 - pwsh: |
@@ -21,6 +23,8 @@ steps:
       -SASKey "${{ parameters.BlobSASKey }}"
       -Language "${{ parameters.TargetLanguage }}"
       -BlobName "${{ parameters.BlobName }}"
+      -RepoReplaceRegex "(https://github.com/${{ parameters.RepoId }}/(?:blob|tree)/)master(/.*)"
+      -Tag: "${{ parameters.ReleaseTag }}"
     pwsh: true
     workingDirectory: $(Pipeline.Workspace)
   displayName: Copy Docs to Blob

eng/common/scripts/add-pullrequest-reviewers.ps1

@@ -18,6 +18,32 @@ param(
     $AuthToken
 )
 
+function AddMembers($memberName, $additionSet) {
+  $headers = @{
+    Authorization = "bearer $AuthToken"
+  }
+  $uri = "https://api.github.com/repos/$RepoOwner/$RepoName/pulls/$PRNumber/requested_reviewers"
+  $errorOccurred = $false
+
+  foreach ($id in $additionSet) {
+    try {
+      $postResp = @{}
+      $postResp[$memberName] = @($id)
+      $postResp = $postResp | ConvertTo-Json
+
+      Write-Host $postResp
+      $resp = Invoke-RestMethod -Method Post -Headers $headers -Body $postResp -Uri $uri -MaximumRetryCount 3
+      $resp | Write-Verbose
+    }
+    catch {
+      Write-Host "Error attempting to add $user `n$_"
+      $errorOccurred = $true
+    }
+  }
+
+  return $errorOccurred
+}
+
 # at least one of these needs to be populated
 if (-not $GitHubUsers -and -not $GitHubTeams) {
   Write-Host "No user provided for addition, exiting."
@@ -27,54 +53,9 @@ if (-not $GitHubUsers -and -not $GitHubTeams) {
 $userAdditions = @($GitHubUsers.Split(",") | % { $_.Trim() } | ? { return $_ })
 $teamAdditions = @($GitHubTeams.Split(",") | % { $_.Trim() } | ? { return $_ })
 
-$headers = @{
-  Authorization = "bearer $AuthToken"
-}
-$uri = "https://api.github.com/repos/$RepoOwner/$RepoName/pulls/$PRNumber/requested_reviewers"
+$errorsOccurredAddingUsers = AddMembers -memberName "reviewers" -additionSet $userAdditions
+$errorsOccurredAddingTeams = AddMembers -memberName "team_reviewers" -additionSet $teamAdditions
 
-try {
-  $resp = Invoke-RestMethod -Headers $headers $uri -MaximumRetryCount 3
-}
-catch {
-  Write-Error "Invoke-RestMethod [$uri] failed with exception:`n$_"
+if ($errorsOccurredAddingUsers -or $errorsOccurredAddingTeams) {
   exit 1
 }
-
-# the response object takes this form: https://developer.github.com/v3/pulls/review_requests/#response-1
-# before we can push a new reviewer, we need to pull the simple Ids out of the complex objects that came back in the response
-$userReviewers = @($resp.users | % { return $_.login })
-$teamReviewers = @($resp.teams | % { return $_.slug })
-
-if (!$userReviewers) { $modifiedUserReviewers = @() } else { $modifiedUserReviewers = $userReviewers.Clone() }
-$modifiedUserReviewers += ($userAdditions | ? { !$modifiedUserReviewers.Contains($_) })
-
-if ($teamReviewers) { $modifiedTeamReviewers = @() } else { $modifiedTeamReviewers = $teamReviewers.Clone() }
-$modifiedTeamReviewers += ($teamAdditions | ? { !$modifiedTeamReviewers.Contains($_) })
-
-$detectedUserDiffs = Compare-Object -ReferenceObject $userReviewers -DifferenceObject $modifiedUserReviewers
-$detectedTeamDiffs = Compare-Object -ReferenceObject $teamReviewers -DifferenceObject $modifiedTeamReviewers
-
-# Compare-Object returns values when there is a difference between the comparied objects.
-# we only want to run the update if there IS a difference.
-if ($detectedUserDiffs -or $detectedTeamDiffs) {
-  $postResp = @{}
-
-  if ($modifiedUserReviewers) { $postResp["reviewers"] = $modifiedUserReviewers }
-  if ($modifiedTeamReviewers) { $postResp["team_reviewers"] = $modifiedTeamReviewers }
-
-  $postResp = $postResp | ConvertTo-Json
-
-  try {
-    Write-Host $postResp
-    $resp = Invoke-RestMethod -Method Post -Headers $headers -Body $postResp -Uri $uri -MaximumRetryCount 3
-    $resp | Write-Verbose
-  }
-  catch {
-    Write-Error "Unable to update PR reviewers. `n$_"
-  }
-}
-else {
-  $results = $GitHubUsers + $GitHubTeams
-  Write-Host "Reviewers $results already added. Exiting."
-  exit(0)
-}

eng/common/scripts/artifact-metadata-parsing.ps1

@@ -7,7 +7,7 @@ $SDIST_PACKAGE_REGEX = "^(?<package>.*)\-(?<versionstring>$([AzureEngSemanticVer
 function CreateReleases($pkgList, $releaseApiUrl, $releaseSha) {
   foreach ($pkgInfo in $pkgList) {
     Write-Host "Creating release $($pkgInfo.Tag)"
-
+    echo "##vso[task.setvariable variable=ReleaseTag;isOutput=true]$($pkgInfo.Tag)"
     $releaseNotes = ""
     if ($pkgInfo.ReleaseNotes -ne $null) {
       $releaseNotes = $pkgInfo.ReleaseNotes
@@ -502,7 +502,7 @@ function CheckArtifactShaAgainstTagsList($priorExistingTagList, $releaseSha, $ap
     else {
       Write-Host "The artifact SHA $releaseSha does not match that of the currently existing tag."
       Write-Host "Tag with issues is $tag with commit SHA $tagSha"
-
+      echo "##vso[task.setvariable variable=ReleaseTag;isOutput=true]$tag"
       $unmatchedTags += $tag
     }
   }

eng/common/scripts/copy-docs-to-blobstorage.ps1

@@ -7,7 +7,9 @@ param (
   $Language,
   $BlobName,
   $ExitOnError=1,
-  $UploadLatest=1
+  $UploadLatest=1,
+  $RepoReplaceRegex,
+  $Tag
 )
 
 $Language = $Language.ToLower()
@@ -197,6 +199,10 @@ function Upload-Blobs
     Write-Host "DocDir $($DocDir)"
     Write-Host "Final Dest $($DocDest)/$($PkgName)/$($DocVersion)"
 
+    # Use the step to replace master link to release tag link 
+    Write-Host "Replacing all readme master links with release tag $Tag"
+    ReplaceLink -scanFolder $DocDir -fileSuffix ".html" -replacement $Tag -customRegex $RepoReplaceRegex
+   
     Write-Host "Uploading $($PkgName)/$($DocVersion) to $($DocDest)..."
     & $($AzCopy) cp "$($DocDir)/**" "$($DocDest)/$($PkgName)/$($DocVersion)$($SASKey)" --recursive=true
 

eng/common/scripts/link-replacement.ps1

@@ -0,0 +1,54 @@
+param (
+  # url list to verify links. Can either be a http address or a local file request. Local file paths support md and html files.
+  [string] $sourceDir,
+  # file that contains a set of links to ignore when verifying
+  [string] $ignoreLinksFile = "$PSScriptRoot/ignore-links.txt",
+  # switch that will enable devops specific logging for warnings
+  [switch] $devOpsLogging = $false,
+  [string] $branchReplaceRegex = "(https://github.com/.*/(?:blob|tree)/)master(/.*)",
+  # the substitute branch name or SHA commit
+  [string] $releaseTag = ""
+)
+
+$regexOptions = [System.Text.RegularExpressions.RegexOptions]"Singleline, IgnoreCase";
+
+function ReplaceLink ($scanFolder, $fileSuffix, $replacement, $customRegex) {
+  $regex = new-object System.Text.RegularExpressions.Regex ($branchReplaceRegex, $regexOptions)
+  if ($customRegex) {
+    $regex = new-object System.Text.RegularExpressions.Regex ($customRegex, $regexOptions)
+  }
+  if (!$fileSuffix) {
+    Write-Error "Please provide at least one file extension to scan against."
+  }
+  $replacementPattern = "`${2}$replacement`$3"
+  if (!$replacement) {
+    Write-Error "Please provide the replacement string to replace with."
+  }
+  if ($scanFolder) {
+    $fileSuffixArray = $fileSuffix -split "," 
+    $url = @()
+
+    foreach ($fileExtension in $fileSuffixArray) {
+      $fileRegex = "*" + $fileSuffix.Trim()
+      $urls += Get-ChildItem -Path "$scanFolder/*" -Recurse -Include $fileRegex
+    }
+    
+    if ($urls.Count -eq 0) {
+      Write-Host "Usage $($MyInvocation.MyCommand.Name) <urls>";
+      return;
+    }  
+    $needTochange = @{}
+    foreach ($url in $urls) {
+      while ((Get-Content -Path $url) -match $regex) {
+        Write-Verbose "We have master link matches in page $url"
+        $needTochange[$url] = $true
+        (Get-Content -Path $url) -replace $regex, $replacementPattern | Set-Content -Path $url 
+      }
+    }
+    foreach ($page in $needTochange.Keys) {
+      Write-Host "There are replacements in page $page"
+    }
+  }
+}
+
+#ReplaceLink -scanFolder $sourceDir -replacement $releaseTag -fileSuffix ".html"

eng/common/scripts/update-docs-metadata.ps1

@@ -17,6 +17,8 @@ param (
 . (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1)
 . (Join-Path $PSScriptRoot SemVer.ps1)
 
+$releaseReplaceRegex = "(https://github.com/$RepoId/(?:blob|tree)/)master(/.*)"
+
 function GetMetaData($lang){
   switch ($lang) {
     "java" {
@@ -75,6 +77,12 @@ function GetAdjustedReadmeContent($pkgInfo, $lang){
     if ($headerContentMatches) {
       $foundTitle = $headerContentMatches.Matches[0]
       $fileContent = $pkgInfo.ReadmeContent -replace $foundTitle, "$foundTitle - Version $($pkgInfo.PackageVersion) `n"
+      # Replace github master link with release tag.
+      $regex = new-object System.Text.RegularExpressions.Regex ($releaseReplaceRegex,
+          [System.Text.RegularExpressions.RegexOptions]"Singleline, IgnoreCase")
+
+      $ReplacementPattern = "`${1}$($pkgInfo.Tag)`$2"
+      $fileContent = $fileContent -replace $regex, $ReplacementPattern
     }
 
     $header = "---`ntitle: $foundTitle`nkeywords: Azure, $lang, SDK, API, $($pkgInfo.PackageId), $service`nauthor: maggiepint`nms.author: magpint`nms.date: $date`nms.topic: article`nms.prod: azure`nms.technology: azure`nms.devlang: $lang`nms.service: $service`n---`n"