Automate docs.ms CI onboarding with docindex

eng/pipelines/docindex.yml

@@ -2,3 +2,44 @@ trigger: none
 
 jobs:
   - template: /eng/common/pipelines/templates/jobs/docindex.yml
+
+  - job: UpdateDocsMsBuildConfig
+    pool:
+      vmImage: ubuntu-18.04
+    variables:
+      DocRepoLocation: $(Pipeline.Workspace)/docs
+      DocRepoOwner: MicrosoftDocs
+      DocRepoName: azure-docs-sdk-node
+    steps:
+      # Sync docs repo (this can be sparse)
+      - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+        parameters:
+          SkipDefaultCheckout: true
+          Paths:
+            - ci-configs/packages-latest.json
+            - ci-configs/packages-preview.json
+          Repositories:
+            - Name: $(DocRepoOwner)/$(DocRepoName)
+              WorkingDirectory: $(DocRepoLocation)
+
+      # Call update docs ci script to onboard packages
+      - task: Powershell@2
+        inputs:
+          pwsh: true
+          filePath: eng/common/scripts/Update-DocsMsPackages.ps1
+          arguments: -DocRepoLocation $(DocRepoLocation)
+        displayName: Update Docs Onboarding
+
+      # Push changes to docs repo
+      - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
+        parameters:
+          WorkingDirectory: $(DocRepoLocation)
+
+      - template: /eng/common/pipelines/templates/steps/git-push-changes.yml
+        parameters:
+          BaseRepoBranch: $(DefaultBranch)
+          BaseRepoOwner: $(DocRepoOwner)
+          CommitMsg: "Update docs CI configuration"
+          TargetRepoName: $(DocRepoName)
+          TargetRepoOwner: $(DocRepoOwner)
+          WorkingDirectory: $(DocRepoLocation)

eng/scripts/Language-Settings.ps1

@@ -172,58 +172,126 @@ function Get-javascript-GithubIoDocIndex()
   GenerateDocfxTocContent -tocContent $tocContent -lang "JavaScript"
 }
 
-# Updates a js CI configuration json.
-# For "latest", we simply set a target package name
-# For "preview", we add @next to the target package name
-function Update-javascript-CIConfig($pkgs, $ciRepo, $locationInDocRepo, $monikerId = $null)
-{
-  $pkgJsonLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo)
-
-  if (-not (Test-Path $pkgJsonLoc))
-  {
-    Write-Error "Unable to locate package json at location $pkgJsonLoc, exiting."
-    exit(1)
+# "@azure/[email protected]" -> "@azure/package-name"
+function Get-PackageNameFromDocsMsConfig($DocsConfigName) { 
+  if ($DocsConfigName -match '^(?<pkgName>.+?)(?<pkgVersion>@.+)?$') { 
+    return $Matches['pkgName']
   }
+  LogWarning "Could not find package name in ($DocsConfigName)"
+  return ''
+}
 
-  $allJson = Get-Content $pkgJsonLoc | ConvertFrom-Json
+# Given the name of a package (possibly of the form "@azure/[email protected]")
+# return a package name with the version specified in $packageVersion
+# "@azure/[email protected]" "1.3.0" -> "@azure/[email protected]"
+function Get-DocsMsPackageName($packageName, $packageVersion) { 
+  return "$(Get-PackageNameFromDocsMsConfig $packageName)@$packageVersion"
+}
 
-  $visibleInCI = @{}
+function Update-javascript-DocsMsPackages($DocsRepoLocation, $DocsMetadata) {
+  UpdateDocsMsPackages `
+    (Join-Path $DocsRepoLocation 'ci-configs/packages-preview.json') `
+    'preview' `
+    $DocsMetadata 
 
-  for ($i = 0; $i -lt $allJson.npm_package_sources.Length; $i++)
-  {
-    $pkgDef = $allJson.npm_package_sources[$i]
-    $accessor = ($pkgDef.name).Replace("`@next", "")
-    $visibleInCI[$accessor] = $i
-  }
+  UpdateDocsMsPackages `
+    (Join-Path $DocsRepoLocation 'ci-configs/packages-latest.json') `
+    'latest' `
+    $DocsMetadata
+}
 
-  foreach ($releasingPkg in $pkgs)
-  {
-    $name = $releasingPkg.PackageId
+function UpdateDocsMsPackages($DocConfigFile, $Mode, $DocsMetadata) {
+  Write-Host "Updating configuration: $DocConfigFile with mode: $Mode"
+  $packageConfig = Get-Content $DocConfigFile -Raw | ConvertFrom-Json
+
+  $outputPackages = @()
+  foreach ($package in $packageConfig.npm_package_sources) {
+    # If Get-PackageNameFromDocsMsConfig cannot find the package name, keep the
+    # entry but do no additional processing on it.
+    if (!(Get-PackageNameFromDocsMsConfig $package.name)) {
+      LogWarning "Package name is not valid: ($($package.name)). Keeping entry in docs config but not updating."
+      $outputPackages += $package
+      continue
+    }
 
-    if ($releasingPkg.IsPrerelease)
-    {
-      $name += "`@next"
+    # Do not filter by GA/Preview status because we want differentiate between
+    # tracked and non-tracked packages
+    $matchingPublishedPackageArray = $DocsMetadata.Where({ $_.Package -eq (Get-PackageNameFromDocsMsConfig $package.name) })
+
+    # If this package does not match any published packages keep it in the list.
+    # This handles packages which are not tracked in metadata but still need to
+    # be built in Docs CI.
+    if ($matchingPublishedPackageArray.Count -eq 0) {
+      Write-Host "Keep non-tracked preview package: $($package.name)"
+      $outputPackages += $package
+      continue
     }
 
-    if ($visibleInCI.ContainsKey($releasingPkg.PackageId))
-    {
-      $packagesIndex = $visibleInCI[$releasingPkg.PackageId]
-      $existingPackageDef = $allJson.npm_package_sources[$packagesIndex]
-      $existingPackageDef.name = $name
+    if ($matchingPublishedPackageArray.Count -gt 1) { 
+      LogWarning "Found more than one matching published package in metadata for $(package.name); only updating first entry"
+    }
+    $matchingPublishedPackage = $matchingPublishedPackageArray[0]
+
+    if ($Mode -eq 'preview' -and !$matchingPublishedPackage.VersionPreview.Trim()) { 
+      # If we are in preview mode and the package does not have a superseding
+      # preview version, remove the package from the list. 
+      Write-Host "Remove superseded preview package: $($package.name)"
+      continue
     }
-    else
-    {
-      $newItem = New-Object PSObject -Property @{
-        name = $name
-      }
 
-      if ($newItem) { $allJson.npm_package_sources += $newItem }
+    $packageVersion = $matchingPublishedPackage.VersionGA
+    if ($Mode -eq 'preview') {
+      $packageVersion = $matchingPublishedPackage.VersionPreview
     }
+
+    # Package name comes in the form "<package-name>@<version>". The version may 
+    # have changed. This parses the name of the package from the input and 
+    # appends the version specified in the metadata.
+    # Mutate the package name because there may be other properties of the
+    # package which are not accounted for in this code (e.g. "folder" in JS 
+    # packages)
+    $package.name = Get-DocsMsPackageName $package.name $packageVersion
+    Write-Host "Keep tracked package: $($package.name)"
+    $outputPackages += $package
+  }
+
+  $outputPackagesHash = @{}
+  foreach ($package in $outputPackages) {
+    $outputPackagesHash[(Get-PackageNameFromDocsMsConfig $package.name)] = $true
   }
 
-  $jsonContent = $allJson | ConvertTo-Json -Depth 10 | ForEach-Object { $_ -replace "(?m)  (?<=^(?:  )*)", "  " }
+  $remainingPackages = @() 
+  if ($Mode -eq 'preview') { 
+    $remainingPackages = $DocsMetadata.Where({ 
+      $_.VersionPreview.Trim() -and !$outputPackagesHash.ContainsKey($_.Package)
+    })
+  } else { 
+    $remainingPackages = $DocsMetadata.Where({ 
+      $_.VersionGA.Trim() -and !$outputPackagesHash.ContainsKey($_.Package)
+    })
+  }
+
+  # Add packages that exist in the metadata but are not onboarded in docs config
+  foreach ($package in $remainingPackages) {
+    # If Get-PackageNameFromDocsMsConfig cannot find the package name, skip
+    # adding it to the packages
+    if (!(Get-PackageNameFromDocsMsConfig $package.Package)) {
+      LogWarning "Package name not valid: ($($package.Package)). Skipping adding from metadata to docs config"
+      continue
+    }
+
+
+    $packageVersion = $package.VersionGA
+    if ($Mode -eq 'preview') {
+      $packageVersion = $package.VersionPreview
+    }
+    $packageName = Get-DocsMsPackageName $package.Package $packageVersion
+    Write-Host "Add new package from metadata: $packageName"
+    $outputPackages += @{ name = $packageName }
+  }
 
-  Set-Content -Path $pkgJsonLoc -Value $jsonContent
+  $packageConfig.npm_package_sources = $outputPackages
+  $packageConfig | ConvertTo-Json -Depth 100 | Set-Content $DocConfigFile
 }
 
 # function is used to auto generate API View