diff --git a/eng/common/scripts/Update-DocsMsMetadata.ps1 b/eng/common/scripts/Update-DocsMsMetadata.ps1 index 2a9d4f26a999..fccb663fa58c 100644 --- a/eng/common/scripts/Update-DocsMsMetadata.ps1 +++ b/eng/common/scripts/Update-DocsMsMetadata.ps1 @@ -50,10 +50,16 @@ $releaseReplaceRegex = "(https://github.com/$RepoId/(?:blob|tree)/)(?:master|mai $TITLE_REGEX = "(\#\s+(?Azure .+? (?:client|plugin|shared) library for (?:JavaScript|Java|Python|\.NET|C)))" function GetAdjustedReadmeContent($ReadmeContent, $PackageInfo, $PackageMetadata) { - # Normalize service name "Key Vault" -> "keyvault" - # TODO: Use taxonomy for service name -- https://github.com/Azure/azure-sdk-tools/issues/1442 - # probably from metadata - $service = $PackageMetadata.ServiceName.ToLower().Replace(" ", "") + # The $PackageMetadata could be $null if there is no associated metadata entry + # based on how the metadata CSV is filtered + $service = $PackageInfo.ServiceDirectory.ToLower() + if ($PackageMetadata -and $PackageMetadata.ServiceName) { + # Normalize service name "Key Vault" -> "keyvault" + # TODO: Use taxonomy for service name -- https://github.com/Azure/azure-sdk-tools/issues/1442 + # probably from metadata + $service = $PackageMetadata.ServiceName.ToLower().Replace(" ", "") + } + # Generate the release tag for use in link substitution $tag = "$($PackageInfo.Name)_$($PackageInfo.Version)" $date = Get-Date -Format "MM/dd/yyyy" @@ -110,11 +116,14 @@ function UpdateDocsMsMetadataForPackage($packageInfoJsonLocation) { $packageMetadataArray = (Get-CSVMetadata).Where({ $_.Package -eq $packageInfo.Name -and $_.GroupId -eq $packageInfo.Group -and $_.Hide -ne 'true' -and $_.New -eq 'true' }) if ($packageMetadataArray.Count -eq 0) { - LogError "Could not retrieve metadata for $($packageInfo.Name) from metadata CSV" + LogWarning "Could not retrieve metadata for $($packageInfo.Name) from metadata CSV. Using best effort defaults." + $packageMetadata = $null } elseif ($packageMetadataArray.Count -gt 1) { LogWarning "Multiple metadata entries for $($packageInfo.Name) in metadata CSV. Using first entry." + $packageMetadata = $packageMetadataArray[0] + } else { + $packageMetadata = $packageMetadataArray[0] } - $packageMetadata = $packageMetadataArray[0] $readmeContent = Get-Content $packageInfo.ReadMePath -Raw $outputReadmeContent = "" diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index 959893344c25..0ba7c06104b6 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -18,6 +18,7 @@ jobs: Paths: - ci-configs/packages-latest.json - ci-configs/packages-preview.json + - metadata/ Repositories: - Name: $(DocRepoOwner)/$(DocRepoName) WorkingDirectory: $(DocRepoLocation) @@ -43,3 +44,45 @@ jobs: TargetRepoName: $(DocRepoName) TargetRepoOwner: $(DocRepoOwner) WorkingDirectory: $(DocRepoLocation) + + # Prepare daily docs CI + - template: /eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml + parameters: + DailyBranchVariableName: DailyDocsBranchName + - pwsh: | + $ErrorActionPreference = "Continue" + git checkout "origin/$(DailyDocsBranchName)" 2>&1 | Out-Null + $LASTEXITCODE = 0 # This ignores any error from git checkout + git status + displayName: Checkout daily branch if it exists + workingDirectory: $(DocRepoLocation) + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsPackages.ps1 + arguments: -DocRepoLocation $(DocRepoLocation) + displayName: Update Docs Onboarding for Daily branch + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoBranch: $(DailyDocsBranchName) + BaseRepoOwner: $(DocRepoOwner) + CommitMsg: "Update targeting packages based on release metadata. (Daily docs)" + TargetRepoName: $(DocRepoName) + TargetRepoOwner: $(DocRepoOwner) + WorkingDirectory: $(DocRepoLocation) + ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts + + - task: PowerShell@2 + displayName: Queue Docs CI build + inputs: + pwsh: true + filePath: eng/common/scripts/Queue-Pipeline.ps1 + # SourceBranch must be "master" as this repo is managed by the docs + # team and will be updated according to their timeline. + arguments: > + -Organization "apidrop" + -Project "Content%20CI" + -SourceBranch "master" + -DefinitionId 3452 + -Base64EncodedAuthToken "$(azuresdk-apidrop-devops-queue-build-pat)" + -BuildParametersJson '{"params":"{ \"target_repo\": { \"url\": \"https://github.com/MicrosoftDocs/azure-docs-sdk-node\", \"branch\": \"$(DailyDocsBranchName)\", \"folder\": \"./\" }, \"source_repos\": [] }"}' diff --git a/eng/pipelines/templates/stages/archetype-js-release.yml b/eng/pipelines/templates/stages/archetype-js-release.yml index 187be7bc08fe..7ea8778fdab4 100644 --- a/eng/pipelines/templates/stages/archetype-js-release.yml +++ b/eng/pipelines/templates/stages/archetype-js-release.yml @@ -108,40 +108,31 @@ stages: dependsOn: PublishPackage pool: - name: azsdk-pool-mms-win-2019-general - vmImage: MMS2019 - variables: - CIConfigs: "{'targets':[{'path_to_config':'ci-configs/packages-preview.json','Mode':'Preview','content_folder':'docs-ref-services/preview'},{'path_to_config':'ci-configs/packages-latest.json','mode':'Latest','content_folder':'docs-ref-services/latest'}]}" + name: azsdk-pool-mms-ubuntu-2004-general + vmImage: MMSUbuntu20.04 strategy: runOnce: deploy: steps: - - checkout: self - - pwsh: | - Get-ChildItem -Recurse ${{parameters.ArtifactName}}/${{artifact.name}} - workingDirectory: $(Pipeline.Workspace) - displayName: Output Visible Artifacts - - template: /eng/common/pipelines/templates/steps/get-pr-owners.yml + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: - TargetVariable: "OwningGHUser" - ServiceDirectory: ${{parameters.ServiceDirectory}} - - template: /eng/common/pipelines/templates/steps/docs-metadata-release.yml + SkipDefaultCheckout: true + Paths: + - sdk/**/*.md + - download: current + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml parameters: - ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}} - PackageRepository: NPM - ReleaseSha: $(Build.SourceVersion) + PackageInfoLocations: + - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{artifact.name}}.json RepoId: Azure/azure-sdk-for-js WorkingDirectory: $(System.DefaultWorkingDirectory) TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} TargetDocRepoName: ${{parameters.TargetDocRepoName}} - PRBranchName: onboarding-${{artifact.name}}-$(Build.BuildId) - ArtifactName: ${{artifact.name}} Language: 'javascript' - DocRepoDestinationPath: 'docs-ref-services/' - GHReviewersVariable: 'OwningGHUser' - CIConfigs: $(CIConfigs) - CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' + SparseCheckoutPaths: + - docs-ref-services/ + - metadata/ - ${{if ne(artifact.skipPublishDocGithubIo, 'true')}}: - deployment: PublishDocsGitHubIO @@ -255,3 +246,34 @@ stages: targetType: filePath filePath: "eng/tools/publish-to-npm.ps1" arguments: '-pathToArtifacts $(Package.Archive) -accessLevel "public" -tag "dev" -registry "$(Registry)" -npmToken "$(NpmToken)"' + + - job: PublishDocsToNightlyBranch + condition: or(eq(variables['SetDevVersion'], 'true'), and(eq(variables['SetDevVersion'], ''), eq(variables['Build.Reason'],'Schedule'), eq(variables['System.TeamProject'], 'internal'))) + pool: + name: azsdk-pool-mms-ubuntu-2004-general + vmImage: MMSUbuntu20.04 + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + SkipDefaultCheckout: true + Paths: + - sdk/**/*.md + - download: current + - pwsh: | + Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/ + displayName: Show visible artifacts + + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml + parameters: + PackageInfoLocations: + - ${{ each artifact in parameters.Artifacts }}: + - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{artifact.name}}.json + RepoId: Azure/azure-sdk-for-js + WorkingDirectory: $(System.DefaultWorkingDirectory) + TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} + TargetDocRepoName: ${{parameters.TargetDocRepoName}} + Language: 'javascript' + DailyDocsBuild: true + SparseCheckoutPaths: + - docs-ref-services/ + - metadata/ diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml index d4230bcbef1e..0132c886cddc 100644 --- a/eng/pipelines/templates/steps/build.yml +++ b/eng/pipelines/templates/steps/build.yml @@ -28,6 +28,18 @@ steps: condition: and(succeeded(),eq(variables['SetDevVersion'],'true')) displayName: "Update package versions for dev build" + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 + arguments: > + -ServiceDirectory ${{parameters.ServiceDirectory}} + -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + -AddDevVersion + pwsh: true + workingDirectory: $(Pipeline.Workspace) + displayName: Update package properties with dev version + condition: and(succeeded(),eq(variables['SetDevVersion'],'true')) + - script: | node common/scripts/install-run-rush.js install condition: and(succeeded(),ne(variables['SetDevVersion'],'true')) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index bfe5fc689fe9..c1dba14557b3 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -122,6 +122,24 @@ function Get-javascript-PackageInfoFromPackageFile ($pkg, $workingDirectory) return $resultObj } +function Get-javascript-DocsMsMetadataForPackage($PackageInfo) { + New-Object PSObject -Property @{ + DocsMsReadMeName = $PackageInfo.Name -replace "^@azure/" , "" + LatestReadMeLocation = 'docs-ref-services/latest' + PreviewReadMeLocation = 'docs-ref-services/preview' + Suffix = '' + } +} + +# In the case of NPM packages, the "dev version" produced for the given build +# may not have been published if the code is identical to the code already +# published at the "dev" tag. To prevent using a version which does not exist in +# NPM, use the "dev" tag instead. +function Get-javascript-DocsMsDevLanguageSpecificPackageInfo($packageInfo) { + $packageInfo.Version = 'dev' + return $packageInfo +} + # Stage and Upload Docs to blob Storage function Publish-javascript-GithubIODocs ($DocLocation, $PublicArtifactLocation) {