Automate docs.ms CI onboarding with docindex

eng/common/scripts/Update-DocsMsPackages.ps1

@@ -0,0 +1,31 @@
+# This script is intended to update docs.ms CI configuration (currently supports Java, Python, C#, JS) in nightly build
+# For details on calling, check `docindex.yml`. 
+
+# In this script, we will do the following business logic.
+# 1. Filter out the packages from release csv file by `New=true`, `Hide!=true`
+# 2. Compare current package list with the csv packages, and keep them in sync. Leave other packages as they are.
+# 3. Update the tarage packages back to CI config files. 
+param (
+  [Parameter(Mandatory = $true)]
+  $DocRepoLocation # the location of the cloned doc repo
+)
+
+. (Join-Path $PSScriptRoot common.ps1)
+
+if ($UpdateDocsMsPackagesFn -and (Test-Path "Function:$UpdateDocsMsPackagesFn")) {
+
+  try { 
+    &$UpdateDocsMsPackagesFn -DocsRepoLocation $DocRepoLocation
+  } catch { 
+    LogError "Exception while updating docs.ms packages"
+    LogError $_ 
+    LogError $_.ScriptStackTrace
+    exit 1
+  }
+
+} else {
+  LogError "The function for '$UpdateFn' was not found.`
+  Make sure it is present in eng/scripts/Language-Settings.ps1 and referenced in eng/common/scripts/common.ps1.`
+  See https://github.com/Azure/azure-sdk-tools/blob/master/doc/common/common_engsys.md#code-structure"
+  exit 1
+}

eng/common/scripts/common.ps1

@@ -39,6 +39,6 @@ if (!(Get-Variable -Name "LanguageDisplayName" -ValueOnly -ErrorAction "Ignore")
 $GetPackageInfoFromRepoFn = "Get-${Language}-PackageInfoFromRepo"
 $GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile"
 $PublishGithubIODocsFn = "Publish-${Language}-GithubIODocs"
-$UpdateDocCIFn = "Update-${Language}-CIConfig"
+$UpdateDocsMsPackagesFn = "Update-${Language}-DocsMsPackages"
 $GetGithubIoDocIndexFn = "Get-${Language}-GithubIoDocIndex"
-$FindArtifactForApiReviewFn = "Find-${Language}-Artifacts-For-Apireview"
+$FindArtifactForApiReviewFn = "Find-${Language}-Artifacts-For-Apireview"

eng/pipelines/docindex.yml

@@ -2,3 +2,44 @@ trigger: none
 
 jobs:
   - template: /eng/common/pipelines/templates/jobs/docindex.yml
+
+  - job: UpdateDocsMsBuildConfig
+    pool:
+      vmImage: ubuntu-18.04
+    variables:
+      DocRepoLocation: $(Pipeline.Workspace)/docs
+      DocRepoOwner: MicrosoftDocs
+      DocRepoName: azure-docs-sdk-node
+    steps:
+      # Sync docs repo (this can be sparse)
+      - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+        parameters:
+          SkipDefaultCheckout: true
+          Paths:
+            - ci-configs/packages-latest.json
+            - ci-configs/packages-preview.json
+          Repositories:
+            - Name: $(DocRepoOwner)/$(DocRepoName)
+              WorkingDirectory: $(DocRepoLocation)
+
+      # Call update docs ci script to onboard packages
+      - task: Powershell@2
+        inputs:
+          pwsh: true
+          filePath: eng/common/scripts/Update-DocsMsPackages.ps1
+          arguments: -DocRepoLocation $(DocRepoLocation)
+        displayName: Update Docs Onboarding
+
+      # Push changes to docs repo
+      - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
+        parameters:
+          WorkingDirectory: $(DocRepoLocation)
+
+      - template: /eng/common/pipelines/templates/steps/git-push-changes.yml
+        parameters:
+          BaseRepoBranch: $(DefaultBranch)
+          BaseRepoOwner: $(DocRepoOwner)
+          CommitMsg: "Update docs CI configuration"
+          TargetRepoName: $(DocRepoName)
+          TargetRepoOwner: $(DocRepoOwner)
+          WorkingDirectory: $(DocRepoLocation)

eng/scripts/Language-Settings.ps1

@@ -172,58 +172,110 @@ function Get-javascript-GithubIoDocIndex()
   GenerateDocfxTocContent -tocContent $tocContent -lang "JavaScript"
 }
 
-# Updates a js CI configuration json.
-# For "latest", we simply set a target package name
-# For "preview", we add @next to the target package name
-function Update-javascript-CIConfig($pkgs, $ciRepo, $locationInDocRepo, $monikerId = $null)
-{
-  $pkgJsonLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo)
-
-  if (-not (Test-Path $pkgJsonLoc))
-  {
-    Write-Error "Unable to locate package json at location $pkgJsonLoc, exiting."
-    exit(1)
+# "@azure/[email protected]" -> "@azure/package-name"
+function Get-PackageNameFromDocsMsConfig($DocsConfigName) { 
+  if ($DocsConfigName -match '^(?<pkgName>.+?)(?<pkgVersion>@.+)?$') { 
+    return $Matches['pkgName']
   }
 
-  $allJson = Get-Content $pkgJsonLoc | ConvertFrom-Json
+  throw "Cannot find package name in supplied parameter ($DocsConfigName)"
+}
 
-  $visibleInCI = @{}
+# Given the name of a package (possibly of the form "@azure/[email protected]")
+# return a package name with the version specified in $packageVersion
+# "@azure/[email protected]" "1.3.0" -> "@azure/[email protected]"
+function Get-DocsMsPackageName($packageName, $packageVersion) { 
+  return "$(Get-PackageNameFromDocsMsConfig $packageName)@$packageVersion"
+}
 
-  for ($i = 0; $i -lt $allJson.npm_package_sources.Length; $i++)
-  {
-    $pkgDef = $allJson.npm_package_sources[$i]
-    $accessor = ($pkgDef.name).Replace("`@next", "")
-    $visibleInCI[$accessor] = $i
-  }
+function Update-javascript-DocsMsPackages($DocsRepoLocation) {
+  UpdateDocsMsPackages `
+    (Join-Path $DocsRepoLocation 'ci-configs/packages-preview.json') `
+    'preview' 
 
-  foreach ($releasingPkg in $pkgs)
-  {
-    $name = $releasingPkg.PackageId
+  UpdateDocsMsPackages `
+    (Join-Path $DocsRepoLocation 'ci-configs/packages-latest.json') `
+    'latest'
+}
 
-    if ($releasingPkg.IsPrerelease)
-    {
-      $name += "`@next"
+function UpdateDocsMsPackages($DocConfigFile, $Mode) {
+  $publishedPackages = (Get-CSVMetadata).Where({ $_.New -eq 'true' -and $_.Hide -ne 'true' })
+
+  Write-Host "Updating configuration: $DocConfigFile with mode: $Mode"
+  $packageConfig = Get-Content $DocConfigFile -Raw | ConvertFrom-Json
+
+  $outputPackages = @()
+  foreach ($package in $packageConfig.npm_package_sources) {
+    # Do not filter by GA/Preview status because we want differentiate between
+    # tracked and non-tracked packages
+    $matchingPublishedPackageArray = $publishedPackages.Where({ $_.Package -eq (Get-PackageNameFromDocsMsConfig $package.name) })
+
+    if ($matchingPublishedPackageArray.Count -gt 1) { 
+      throw "Found more than one matching published package in metadata for $(package.name)"
     }
 
-    if ($visibleInCI.ContainsKey($releasingPkg.PackageId))
-    {
-      $packagesIndex = $visibleInCI[$releasingPkg.PackageId]
-      $existingPackageDef = $allJson.npm_package_sources[$packagesIndex]
-      $existingPackageDef.name = $name
+    # If this package does not match any published packages keep it in the list.
+    # This handles packages which are not tracked in metadata but still need to
+    # be built in Docs CI.
+    if ($matchingPublishedPackageArray.Count -eq 0) {
+      Write-Host "Keep non-tracked preview package: $($package.name)"
+      $outputPackages += $package
+      continue
+    }
+
+    $matchingPublishedPackage = $matchingPublishedPackageArray[0]
+
+    if ($Mode -eq 'preview' -and !$matchingPublishedPackage.VersionPreview.Trim()) { 
+      # If we are in preview mode and the package does not have a superseding
+      # preview version, remove the package from the list. 
+      Write-Host "Remove superseded preview package: $($package.name)"
+      continue
     }
-    else
-    {
-      $newItem = New-Object PSObject -Property @{
-        name = $name
-      }
 
-      if ($newItem) { $allJson.npm_package_sources += $newItem }
+    $packageVersion = $matchingPublishedPackage.VersionGA
+    if ($Mode -eq 'preview') {
+      $packageVersion = $matchingPublishedPackage.VersionPreview
     }
+
+    # Package name comes in the form "<package-name>@<version>". The version may 
+    # have changed. This parses the name of the package from the input and 
+    # appends the version specified in the metadata.
+    # Mutate the package name because there may be other properties of the
+    # package which are not accounted for in this code (e.g. "folder" in JS 
+    # packages)
+    $package.name = "$(Get-PackageNameFromDocsMsConfig $package.name)@$($packageVersion)" 
+    Write-Host "Keep tracked package: $($package.name)"
+    $outputPackages += $package
   }
 
-  $jsonContent = $allJson | ConvertTo-Json -Depth 10 | ForEach-Object { $_ -replace "(?m)  (?<=^(?:  )*)", "  " }
+  $outputPackagesHash = @{}
+  foreach ($package in $outputPackages) {
+    $outputPackagesHash[(Get-PackageNameFromDocsMsConfig $package.name)] = $true
+  }
+
+  $remainingPackages = @() 
+  if ($Mode -eq 'preview') { 
+    $remainingPackages = $publishedPackages.Where({ 
+      $_.VersionPreview.Trim() -and !$outputPackagesHash.ContainsKey($_.Package)
+    })
+  } else { 
+    $remainingPackages = $publishedPackages.Where({ 
+      $_.VersionGA.Trim() -and !$outputPackagesHash.ContainsKey($_.Package)
+    })
+  }
+
+  # Add packages that exist in the metadata but are not onboarded in docs config
+  foreach ($package in $remainingPackages) {
+    $packageVersion = $package.VersionGA
+    if ($Mode -eq 'preview') {
+      $packageVersion = $package.VersionPreview
+    }
+    Write-Host "Add new package from metadata: $($package.Package)@$($packageVersion)"
+    $outputPackages += @{ name = "$($package.Package)@$($packageVersion)" }
+  }
 
-  Set-Content -Path $pkgJsonLoc -Value $jsonContent
+  $packageConfig.npm_package_sources = $outputPackages
+  $packageConfig | ConvertTo-Json -Depth 100 | Set-Content $DocConfigFile
 }
 
 # function is used to auto generate API View