Azure · sima-zhu · Oct 30, 2020 · Oct 30, 2020
diff --git a/eng/docgeneration/Generate-DocIndex.ps1 b/eng/docgeneration/Generate-DocIndex.ps1
@@ -1,164 +1,150 @@
+# Generates an index page for cataloging different versions of the Docs
 [CmdletBinding()]
 Param (
+    $DocFx,
     $RepoRoot,
     $DocGenDir
 )
-
-Write-Verbose "Name Reccuring paths with variable names"
-$DocFxTool = "${RepoRoot}/docfx/docfx.exe"
-$DocOutDir = "${RepoRoot}/docfx_project"
-
-Write-Verbose "Initializing Default DocFx Site..."
-& "${DocFxTool}" init -q -o "${DocOutDir}"
-
-Write-Verbose "Copying template and configuration..."
-New-Item -Path "${DocOutDir}" -Name "templates" -ItemType "directory"
-Copy-Item "${DocGenDir}/templates/*" -Destination "${DocOutDir}/templates" -Force -Recurse
-Copy-Item "${DocGenDir}/docfx.json" -Destination "${DocOutDir}/" -Force
-
-Write-Verbose "Creating Index using service directory and package names from repo..."
-# The service mapper is used to map the directory names to the service names to produce
-# a more friendly index. If something isn't in the mapper then the default will just be
-# the service name in all caps
-$serviceMapHash = Get-Content -Path "${DocGenDir}/service-mapper.json" | ConvertFrom-Json -AsHashtable
-
-# There are some artifact that show up, due to the way we do discovery, that are never shipped.
-# Keep a list of those here and anything we don't want to ship can be added to here which will
-# cause them to get skipped when generating the DocIndex
-$ArtifactsToSkip = (
-'azure-cosmos-benchmark',
-'azure-sdk-template'
-)
-
-# The list of services is being constructed from the directory list under the sdk folder
-# which, right now, only contains client/data directories. When management is moved to
-# the under sdk it'll automatically get picked up.
-$ServiceListData = Get-ChildItem "${RepoRoot}/sdk" -Directory
-$YmlPath = "${DocOutDir}/api"
-New-Item -Path $YmlPath -Name "toc.yml" -Force
-foreach ($Dir in $ServiceListData)
-{   
-    $mappedDir = ""
-    if ($serviceMapHash.ContainsKey($Dir.Name)) 
-    {
-        $mappedDir = $serviceMapHash[$Dir.Name]
-    }
-    else
-    {
-        $mappedDir = $Dir.Name.ToUpper()
+. "${PSScriptRoot}\..\common\scripts\common.ps1"
+$GetGithubIoDocIndexFn = "Get-${Language}-GithubIoDocIndex"
+
+# Given the metadata url under https://github.com/Azure/azure-sdk/tree/master/_data/releases/latest, 
+# the function will return the csv metadata back as part of response.
+function Get-CSVMetadata ([string]$MetadataUri) {
+    $metadataResponse = Invoke-RestMethod -Uri $MetadataUri -method "GET" -MaximumRetryCount 3 -RetryIntervalSec 10 | ConvertFrom-Csv
+    return $metadataResponse
+}
+
+# Given the github io blob storage url and language regex,
+# the helper function will return a list of artifact names.
+function Get-BlobStorage-Artifacts($blobStorageUrl, $blobDirectoryRegex, $blobArtifactsReplacement) {
+    LogDebug "Reading artifact from storage blob ..."
+    $returnedArtifacts = @()
+    $pageToken = ""
+    Do {
+      $resp = ""
+      if (!$pageToken) {
+        # First page call.
+        $resp = Invoke-RestMethod -Method Get -Uri $blobStorageUrl
+      }
+      else {
+        # Next page call
+        $blobStorageUrlPageToken = $blobStorageUrl + "&marker=$pageToken"
+        $resp = Invoke-RestMethod -Method Get -Uri $blobStorageUrlPageToken
+      }
+      # Convert to xml documents. 
+      $xmlDoc = [xml](removeBomFromString $resp)
+      foreach ($elem in $xmlDoc.EnumerationResults.Blobs.BlobPrefix) {
+        # What service return like "dotnet/Azure.AI.Anomalydetector/", needs to fetch out "Azure.AI.Anomalydetector"
+        $artifact = $elem.Name -replace $blobDirectoryRegex, $blobArtifactsReplacement
+        $returnedArtifacts += $artifact
+      }
+      # Fetch page token
+      $pageToken = $xmlDoc.EnumerationResults.NextMarker
+    } while ($pageToken)
+    return $returnedArtifacts
+  }
+
+# The sequence of Bom bytes differs by different encoding. 
+# The helper function here is only to strip the utf-8 encoding system as it is used by blob storage list api.
+# Return the original string if not in BOM utf-8 sequence.
+function RemoveBomFromString([string]$bomAwareString) {
+    if ($bomAwareString.length -le 3) {
+        return $bomAwareString
     }
-
-    # Store the list of artifacts into the arrays and write them into the .md file
-    # after processing the list of subdirectories. This will allow the correct 
-    # division of the artifacts under the Client or Management headings
-    $clientArr = @()
-    $mgmtArr = @()
-
-    $PkgList = Get-ChildItem $Dir.FullName -Directory -Exclude changelog, faq, .github, build
-    if (($PkgList | Measure-Object).count -eq 0)
-    {
-        continue
+    $bomPatternByteArray = [byte[]] (0xef, 0xbb, 0xbf)
+    # The default encoding for powershell is ISO-8859-1, so converting bytes with the encoding.
+    $bomAwareBytes = [Text.Encoding]::GetEncoding(28591).GetBytes($bomAwareString.Substring(0, 3))
+    if (@(Compare-Object $bomPatternByteArray $bomAwareBytes -SyncWindow 0).Length -eq 0) {
+        return $bomAwareString.Substring(3)
     }
-    foreach ($Pkg in $PkgList)
-    {
-        # Load the pom file to pull the artifact name and grab the
-        # parent's relative path to see which parent pom is being
-        # used to determine whether or not the artifact is client 
-        # or management.
-        $PomPath = Join-Path -Path $Pkg -ChildPath "pom.xml"
-
-        # no pom file = nothing to process
-        if (Test-Path -path $PomPath) 
-        {
-            $xml = New-Object xml
-            $xml.Load($PomPath)
-
-            # Get the artifactId from the POM
-            $artifactId = $xml.project.artifactId
-
-            $parent = $xml.project.parent.relativePath
-
-            # If this is an artifact that isn't shipping then just
-            # move on to the next one
-            if ($ArtifactsToSkip -contains $artifactId)
-            {
-                Write-Output "skipping $artifactId"
-                continue
-            }
-
-            # If the parent is null or empty then the pom isn't directly including 
-            # one of the pom.[client|data|management].xml and needs to be specially
-            # handled
-            if (("" -eq $parent) -or ($null -eq $parent))
-            {
-                # Cosmos has a root pom which includes pom.client.xml that won't
-                # be detected by this logic. It's easier to deal with specially
-                # than it is to try and climb the pom chain here.
-                if ($Dir.BaseName -eq 'cosmos') 
-                {
-                    $clientArr += $artifactId
-                } 
-                else 
-                {
-                    Write-Host "*snowflake* Pom $PomPath, has a null or empty relative path."
-                }
-            } 
-            else 
-            {
-                if (($parent.IndexOf('azure-client-sdk-parent') -ne -1) -or ($parent.IndexOf('azure-data-sdk-parent') -ne -1))
-                {
-                    $clientArr += $artifactId
-                } 
-                else 
-                {
-                    $mgmtArr += $artifactId
-                }
+    return $bomAwareString
+}
+
+function Get-TocMapping { 
+    Param (
+        [Parameter(Mandatory = $true)] [Object[]] $metadata,
+        [Parameter(Mandatory = $true)] [String[]] $artifacts
+    )
+    # Used for sorting the toc display order
+    $orderServiceMapping = @{}
+
+    foreach ($artifact in $artifacts) {
+        $packageInfo = $metadata | ? {$_.Package -eq $artifact}
+
+        if ($packageInfo -and $packageInfo[0].Hide -eq 'true') {
+            LogDebug "The artifact $artifact set 'Hide' to 'true'."
+            continue
+        }
+        $serviceName = ""
+        if (!$packageInfo -or !$packageInfo[0].ServiceName) {
+            LogWarning "There is no service name for artifact $artifact. Please check csv of Azure/azure-sdk/_data/release/latest repo if this is intended. "
+            # If no service name retrieved, print out warning message, and put it into Other page.
+            $serviceName = "Other"
+        }
+        else {
+            if ($packageInfo.Length -gt 1) {
+                LogWarning "There are more than 1 packages fetched out for artifact $artifact. Please check csv of Azure/azure-sdk/_data/release/latest repo if this is intended. "
             }
+            $serviceName = $packageInfo[0].ServiceName.Trim()
         }
+        $orderServiceMapping[$artifact] = $serviceName
     }
-    # Only create this if there's something to create
-    #if (($clientArr.Count -gt 0) -or ($mgmtArr.Count -gt 0))
-    if ($clientArr.Count -gt 0)
-    {
-    New-Item -Path $YmlPath -Name "$($Dir.Name).md" -Force
-    Add-Content -Path "$($YmlPath)/toc.yml" -Value "- name: $($mappedDir)`r`n  href: $($Dir.Name).md"
-    # loop through the arrays and add the appropriate artifacts under the appropriate headings
-    if ($clientArr.Count -gt 0)
-    {
-        Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "# Client Libraries"
-        foreach($lib in $clientArr) 
-        {
-            Write-Host "Write $($lib) to $($Dir.Name).md"
-            Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "#### $lib"
+    return $orderServiceMapping                   
+}
+
+function GenerateDocfxTocContent([Hashtable]$tocContent, [String]$lang) {
+    LogDebug "Start generating the docfx toc and build docfx site..."
+    $DocOutDir = "${RepoRoot}/docfx_project"
+
+    LogDebug "Initializing Default DocFx Site..."
+    & $($DocFx) init -q -o "${DocOutDir}"
+    # The line below is used for testing in local
+    # docfx init -q -o "${DocOutDir}"
+    LogDebug "Copying template and configuration..."
+    New-Item -Path "${DocOutDir}" -Name "templates" -ItemType "directory" -Force
+    Copy-Item "${DocGenDir}/templates/*" -Destination "${DocOutDir}/templates" -Force -Recurse
+    Copy-Item "${DocGenDir}/docfx.json" -Destination "${DocOutDir}/" -Force
+    $YmlPath = "${DocOutDir}/api"
+    New-Item -Path $YmlPath -Name "toc.yml" -Force
+    $visitedService = @{}
+    # Sort and display toc service name by alphabetical order.
+    foreach ($serviceMapping in $tocContent.getEnumerator() | Sort Value) {
+        $artifact = $serviceMapping.Key
+        $serviceName = $serviceMapping.Value
+        $fileName = ($serviceName -replace '\s', '').ToLower().Trim()
+        if ($visitedService.ContainsKey($serviceName)) {
+            Add-Content -Path "$($YmlPath)/${fileName}.md" -Value "#### $artifact"
+        }
+        else {
+            Add-Content -Path "$($YmlPath)/toc.yml" -Value "- name: ${serviceName}`r`n  href: ${fileName}.md"
+            New-Item -Path $YmlPath -Name "${fileName}.md" -Force
+            Add-Content -Path "$($YmlPath)/${fileName}.md" -Value "#### $artifact"
+            $visitedService[$serviceName] = $true
         }
     }
-    # For the moment there are no management docs and with the way some of the libraries
-    # in management are versioned is a bit wonky. They aren't versioned by releasing a new
-    # version with the same groupId/artifactId, they're versioned with the same artifactId
-    # and version with a different groupId and the groupId happens to include the date. For 
-    # example, the artifact/version of azure-mgmt-storage:1.0.0-beta has several different 
-    # groupIds. com.microsoft.azure.storage.v2016_01_01, com.microsoft.azure.storage.v2017_10_01,
-    # com.microsoft.azure.storage.v2018_11_01 etc.
-    #if ($mgmtArr.Count -gt 0) 
-    #{
-    #    Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "# Management Libraries"
-    #    foreach($lib in $mgmtArr) 
-    #    {
-    #        Write-Output "Write $($lib) to $($Dir.Name).md"
-    #        Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "#### $lib"
-    #    }
-    #}
-    }
-}
 
-Write-Verbose "Creating Site Title and Navigation..."
-New-Item -Path "${DocOutDir}" -Name "toc.yml" -Force
-Add-Content -Path "${DocOutDir}/toc.yml" -Value "- name: Azure SDK for Java APIs`r`n  href: api/`r`n  homepage: api/index.md"
+    # Generate toc homepage.
+    LogDebug "Creating Site Title and Navigation..."
+    New-Item -Path "${DocOutDir}" -Name "toc.yml" -Force
+    Add-Content -Path "${DocOutDir}/toc.yml" -Value "- name: Azure SDK for $lang APIs`r`n  href: api/`r`n  homepage: api/index.md"
 
-Write-Verbose "Copying root markdowns"
-Copy-Item "$($RepoRoot)/README.md" -Destination "${DocOutDir}/api/index.md" -Force
+    LogDebug "Copying root markdowns"
+    Copy-Item "$($RepoRoot)/README.md" -Destination "${DocOutDir}/api/index.md" -Force
+    Copy-Item "$($RepoRoot)/CONTRIBUTING.md" -Destination "${DocOutDir}/api/CONTRIBUTING.md" -Force
 
-Write-Verbose "Building site..."
-& "${DocFxTool}" build "${DocOutDir}/docfx.json"
+    LogDebug "Building site..."
+    & $($DocFx) build "${DocOutDir}/docfx.json"
+    # The line below is used for testing in local
+    # docfx build "${DocOutDir}/docfx.json"
+    Copy-Item "${DocGenDir}/assets/logo.svg" -Destination "${DocOutDir}/_site/" -Force    
+}
 
-Copy-Item "${DocGenDir}/assets/logo.svg" -Destination "${DocOutDir}/_site/" -Force
+if ((Get-ChildItem -Path Function: | ? { $_.Name -eq $GetGithubIoDocIndexFn  }).Count -gt 0)
+{
+    &$GetGithubIoDocIndexFn
+}
+else
+{
+    LogWarning "The function '$GetGithubIoDocIndexFn' was not found."
+}
diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml
@@ -10,21 +10,24 @@ jobs:
         inputs:
           versionSpec: '3.6'
 
-      - template: /eng/common/pipelines/templates/steps/replace-relative-links.yml
-        parameters:
-          TargetFolder: .
-          RootFolder: .
-          BuildSHA: $(Build.SourceVersion)
-
       - pwsh: |
           Invoke-WebRequest -Uri "https://github.com/dotnet/docfx/releases/download/v2.43.2/docfx.zip" `
-          -OutFile "docfx.zip" | Wait-Process; Expand-Archive -Path "docfx.zip" -DestinationPath "./docfx/"
+          -OutFile "docfx.zip" | Wait-Process; Expand-Archive -Path "docfx.zip" -DestinationPath ./docfx
+          echo "##vso[task.setvariable variable=docfxPath;isOutput=true]$(Build.SourcesDirectory)/docfx/docfx.exe"
         workingDirectory: $(Build.SourcesDirectory)
         displayName: Download and Extract DocFX
+        name: setupDocfxTool
 
-      - pwsh: |
-          $(Build.SourcesDirectory)/eng/docgeneration/Generate-DocIndex.ps1 -RepoRoot $(Build.SourcesDirectory) -DocGenDir "$(Build.SourcesDirectory)/eng/docgeneration" -verbose
+      - task: PowerShell@2
         displayName: 'Generate Doc Index'
+        inputs:
+          pwsh: true
+          filePath: $(Build.SourcesDirectory)/eng/docgeneration/Generate-DocIndex.ps1 
+          arguments: >
+            -Docfx $(setupDocfxTool.docfxPath)
+            -RepoRoot $(Build.SourcesDirectory)
+            -DocGenDir "$(Build.SourcesDirectory)/eng/docgeneration"
+            -verbose
 
       - task: UsePythonVersion@0
         displayName: 'Use Python 3.6'

diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1
@@ -2,6 +2,7 @@ $Language = "java"
 $PackageRepository = "Maven"
 $packagePattern = "*.pom"
 $MetadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/java-packages.csv"
+$BlobStorageUrl = "https://azuresdkdocs.blob.core.windows.net/%24web?restype=container&comp=list&prefix=java%2F&delimiter=%2F"
 
 function Get-java-PackageInfoFromRepo ($pkgPath, $serviceDirectory, $pkgName)
 {
@@ -152,4 +153,19 @@ function Publish-java-GithubIODocs ($DocLocation, $PublicArtifactLocation)
       }
     }
   }
-}
+}
+
+function Get-java-GithubIoDocIndex() {
+  # Fetch out all package metadata from csv file.
+  $metadata = Get-CSVMetadata -MetadataUri $MetadataUri
+  # Leave the track 2 packages if multiple packages fetched out.
+  $clientPackages = $metadata | Where-Object { $_.GroupId -eq 'com.azure' } 
+  $nonClientPackages = $metadata | Where-Object { $_.GroupId -ne 'com.azure' -and !$clientPackages.Package.Contains($_.Package) }
+  $uniquePackages = $clientPackages + $nonClientPackages
+  # Get the artifacts name from blob storage
+  $artifacts =  Get-BlobStorage-Artifacts -blobStorageUrl $BlobStorageUrl -blobDirectoryRegex "^java/(.*)/$" -blobArtifactsReplacement '$1'
+  # Build up the artifact to service name mapping for GithubIo toc.
+  $tocContent = Get-TocMapping -metadata $uniquePackages -artifacts $artifacts
+  # Generate yml/md toc files and build site.
+  GenerateDocfxTocContent -tocContent $tocContent -lang "Java"
+}