diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index 173eed02d58c..1095abb646d6 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -2,3 +2,45 @@ trigger: none jobs: - template: /eng/common/pipelines/templates/jobs/docindex.yml + + - job: UpdateDocsMsBuildConfig + pool: + vmImage: ubuntu-20.04 + variables: + DocRepoLocation: $(Pipeline.Workspace)/docs + DocRepoOwner: Azure + DocRepoName: azure-docs-sdk-dotnet + steps: + # Checkout the eng folder from the SDK repo + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + SkipDefaultCheckout: true + + # Sync docs repo + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - bundlepackages + Repositories: + - Name: $(DocRepoOwner)/$(DocRepoName) + WorkingDirectory: $(DocRepoLocation) + + # Call update docs ci script to onboard packages + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsPackages.ps1 + arguments: -DocRepoLocation $(DocRepoLocation) + displayName: Update Docs Onboarding + + - template: /eng/common/pipelines/templates/steps/set-default-branch.yml + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoBranch: $(DefaultBranch) + BaseRepoOwner: $(DocRepoOwner) + CommitMsg: "Update docs CI configuration" + TargetRepoName: $(DocRepoName) + TargetRepoOwner: $(DocRepoOwner) + WorkingDirectory: $(DocRepoLocation) + diff --git a/eng/pipelines/templates/stages/archetype-net-release.yml b/eng/pipelines/templates/stages/archetype-net-release.yml index 4b1929046510..d045cae9d94d 100644 --- a/eng/pipelines/templates/stages/archetype-net-release.yml +++ b/eng/pipelines/templates/stages/archetype-net-release.yml @@ -154,13 +154,7 @@ stages: deploy: steps: - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - paths: - - .github/CODEOWNERS - - template: /eng/common/pipelines/templates/steps/get-pr-owners.yml - parameters: - TargetVariable: "OwningGHUser" - ServiceDirectory: ${{parameters.ServiceDirectory}} + - template: /eng/common/pipelines/templates/steps/docs-metadata-release.yml parameters: ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}}-signed/${{artifact.name}} @@ -174,7 +168,6 @@ stages: ArtifactName: ${{artifact.name}} Language: '.net' DocRepoDestinationPath: 'api/overview/azure/' - GHReviewersVariable: 'OwningGHUser' CIConfigs: $(CIConfigs) OnboardingBranch: 'onboarding' CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 4eb0f2933807..aeaf5b84cb85 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -245,3 +245,201 @@ function GetExistingPackageVersions ($PackageName, $GroupId=$null) return $null } } + +function Get-dotnet-DocsMsMetadataForPackage($PackageInfo) { + $suffix = '' + $parsedVersion = [AzureEngSemanticVersion]::ParseVersionString($PackageInfo.Version) + if ($parsedVersion.IsPrerelease) { + $suffix = '-pre' + } + + New-Object PSObject -Property @{ + DocsMsReadMeName = $PackageInfo.Name.ToLower() -replace "." , "" + LatestReadMeLocation = 'api/overview/azure' + PreviewReadMeLocation = 'api/overview/azure' + Suffix = $suffix + } +} + + +# Details on CSV schema: +# https://review.docs.microsoft.com/en-us/help/onboard/admin/reference/dotnet/documenting-nuget?branch=master#set-up-the-ci-job +# +# PowerShell's included Import-Csv cmdlet is not sufficient for parsing this +# format because it does not easily handle rows whose number of columns is +# greater than the number of columns in the first row. We must manually parse +# this CSV file. +function Get-DocsCiConfig($configPath) { + Write-Host "Loading csv from $configPath" + $output = @() + foreach ($row in Get-Content $configPath) { + # CSV format: + # {package_moniker_base_string},{package_ID},{version_1},{version_2},...,{version_N} + # + # The {package_ID} field can contain optional properties denoted by square + # brackets of the format: [key=value;key=value;...] + + # Split the rows by the comma + $fields = $row.Split(',') + + # If the {package_ID} field contains optional properties inside square + # brackets, parse those properties into key value pairs. In the case of + # duplicate keys, the last one wins. + $rawProperties = '' + $packageProperties = [ordered]@{} + if ($fields[1] -match '\[(.*)\]') { + $rawProperties = $Matches[1] + foreach ($propertyExpression in $rawProperties.Split(';')) { + $propertyParts = $propertyExpression.Split('=') + $packageProperties[$propertyParts[0]] = $propertyParts[1] + } + } + + # Matches the "Package.Name" from the {package_ID} field. Possible + # formats: + # [key=value;key=value]Package.Name + # Package.Name + $packageName = '' + if ($fields[1] -match '(\[.*\])?(.*)') { + $packageName = $Matches[2] + } else { + Write-Error "Could not find package id in row: $row" + } + + # Remaining entries in the row are versions, add them to the package + # properties + $outputVersions = @() + if ($fields[2]) { + $outputVersions = $fields[2..($fields.Count - 1)] + } + + # Example row: + # packagemoniker,[key1=value1;key2=value2]Package.Name,1.0.0,1.2.3-beta.1 + $output += [PSCustomObject]@{ + Id = $fields[0]; # packagemoniker + Name = $packageName; # Package.Name + Properties = $packageProperties; # @{key1='value1'; key2='value2'} + Versions = $outputVersions # @('1.0.0', '1.2.3-beta.1') + } + } + + return $output +} + +function Get-DocsCiLine ($item) { + $line = '' + if ($item.Properties.Count) { + $propertyPairs = @() + foreach ($key in $item.Properties.Keys) { + $propertyPairs += "$key=$($item.Properties[$key])" + } + $packageProperties = $propertyPairs -join ';' + + $line = "$($item.Id),[$packageProperties]$($item.Name)" + } else { + $line = "$($item.Id),$($item.Name)" + } + + if ($item.Versions) { + $joinedVersions = $item.Versions -join ',' + $line += ",$joinedVersions" + } + + return $line +} + +function Update-dotnet-DocsMsPackages($DocsRepoLocation, $DocsMetadata) { + UpdateDocsMsPackages ` + (Join-Path $DocsRepoLocation 'bundlepackages/azure-dotnet-preview.csv') ` + 'preview' ` + $DocsMetadata + + UpdateDocsMsPackages ` + (Join-Path $DocsRepoLocation 'bundlepackages/azure-dotnet.csv') ` + 'latest' ` + $DocsMetadata +} + +function UpdateDocsMsPackages($DocConfigFile, $Mode, $DocsMetadata) { + Write-Host "Updating configuration: $DocConfigFile with mode: $Mode" + $packageConfig = Get-DocsCiConfig $DocConfigFile + + $outputPackages = @() + foreach ($package in $packageConfig) { + # Do not filter by GA/Preview status because we want differentiate between + # tracked and non-tracked packages + $matchingPublishedPackageArray = $DocsMetadata.Where({ $_.Package -eq $package.Name }) + + # If this package does not match any published packages keep it in the list. + # This handles packages which are not tracked in metadata but still need to + # be built in Docs CI. + if ($matchingPublishedPackageArray.Count -eq 0) { + Write-Host "Keep non-tracked preview package: $($package.Name)" + $outputPackages += $package + continue + } + + if ($matchingPublishedPackageArray.Count -gt 1) { + LogWarning "Found more than one matching published package in metadata for $($package.Name); only updating first entry" + } + $matchingPublishedPackage = $matchingPublishedPackageArray[0] + + if ($Mode -eq 'preview' -and !$matchingPublishedPackage.VersionPreview.Trim()) { + # If we are in preview mode and the package does not have a superseding + # preview version, remove the package from the list. + Write-Host "Remove superseded preview package: $($package.Name)" + continue + } + + Write-Host "Keep tracked package: $($package.Name)" + $package.Versions = @($matchingPublishedPackage.VersionGA.Trim()) + if ($Mode -eq 'preview') { + $package.Versions = @($matchingPublishedPackage.VersionPreview.Trim()) + } + $outputPackages += $package + } + + $outputPackagesHash = @{} + foreach ($package in $outputPackages) { + $outputPackagesHash[$package.Name] = $true + } + + $remainingPackages = @() + if ($Mode -eq 'preview') { + $remainingPackages = $DocsMetadata.Where({ + $_.VersionPreview.Trim() -and !$outputPackagesHash.ContainsKey($_.Package) + }) + } else { + $remainingPackages = $DocsMetadata.Where({ + $_.VersionGA.Trim() -and !$outputPackagesHash.ContainsKey($_.Package) + }) + } + + # Add packages that exist in the metadata but are not onboarded in docs config + # TODO: tfm='netstandard2.0' is a temporary workaround for + # https://github.com/Azure/azure-sdk-for-net/issues/22494 + $newPackageProperties = [ordered]@{ tfm = 'netstandard2.0' } + if ($Mode -eq 'preview') { + $newPackageProperties['isPrerelease'] = 'true' + } + + foreach ($package in $remainingPackages) { + Write-Host "Add new package from metadata: $($package.Package)" + $versions = @($package.VersionGA.Trim()) + if ($Mode -eq 'preview') { + $versions = @($package.VersionPreview.Trim()) + } + $outputPackages += [PSCustomObject]@{ + Id = $package.Package.Replace('.', '').ToLower(); + Name = $package.Package; + Properties = $newPackageProperties; + Versions = $versions + } + } + + $outputLines = @() + foreach ($package in $outputPackages) { + $outputLines += Get-DocsCiLine $package + } + Set-Content -Path $DocConfigFile -Value $outputLines +}