diff --git a/eng/dailydocsconfig.json b/eng/dailydocsconfig.json new file mode 100644 index 000000000000..a9692f8ab707 --- /dev/null +++ b/eng/dailydocsconfig.json @@ -0,0 +1,7 @@ +{ + "output_path":"docs-ref-autogen", + "target_repo": { + "url": "https://github.com/Azure/azure-docs-sdk-java", + "branch": "%%DailyDocsBranchName%%" + } +} diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index 89ebeec2452c..3bea0b440edd 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -29,6 +29,7 @@ jobs: filePath: eng/common/scripts/Update-DocsMsPackages.ps1 arguments: -DocRepoLocation $(DocRepoLocation) displayName: Update Docs Onboarding + condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true'))) # Push changes to docs repo - template: /eng/common/pipelines/templates/steps/set-default-branch.yml @@ -43,3 +44,58 @@ jobs: TargetRepoName: $(DocRepoName) TargetRepoOwner: $(DocRepoOwner) WorkingDirectory: $(DocRepoLocation) + + # Prepare daily docs CI + - template: /eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml + parameters: + DailyBranchVariableName: DailyDocsBranchName + - pwsh: | + $ErrorActionPreference = "Continue" + git checkout "origin/$(DailyDocsBranchName)" 2>&1 | Out-Null + $LASTEXITCODE = 0 # This ignores any error from git checkout + git status + displayName: Checkout daily branch if it exists + workingDirectory: $(DocRepoLocation) + + # Docs daily updates is supposed to download packages from public feed repository, so we have to specify additional repositories in a POM or the profile. + # Here is maven documentation: https://maven.apache.org/guides/mini/guide-multiple-repositories.html + - powershell: | + # Linux mvn `setting.xml` is sitting under path `~/.m2/setting.xml` + Get-Command mvn + if (!(Test-Path '~/.m2/')) { + mkdir ~/.m2/ + } + if (Test-Path '~/.m2/setting.xml') { + Write-Host "'setting.xml' exists. Overwriting the file to support multiple repositories." + } + Copy-Item "./eng/repo-docs/docms/daily.update.setting.xml" -Destination "~/.m2/settings.xml" + displayName: 'Configure mvn' + workingDirectory: $(Build.SourcesDirectory) + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsPackages.ps1 + arguments: -DocRepoLocation $(DocRepoLocation) + displayName: Update Docs Onboarding for Daily branch + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoBranch: $(DailyDocsBranchName) + BaseRepoOwner: $(DocRepoOwner) + CommitMsg: "Update targeting packages based on release metadata. (Daily docs)" + TargetRepoName: $(DocRepoName) + TargetRepoOwner: $(DocRepoOwner) + WorkingDirectory: $(DocRepoLocation) + ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts + + - task: PowerShell@2 + displayName: Queue Docs CI build + inputs: + pwsh: true + filePath: eng/common/scripts/Queue-Pipeline.ps1 + arguments: > + -Organization "apidrop" + -Project "Content%20CI" + -DefinitionId 3188 + -Base64EncodedAuthToken "$(azuresdk-apidrop-devops-queue-build-pat)" + -BuildParametersJson (@{ params = (Get-Content ./eng/dailydocsconfig.json -Raw) -replace '%%DailyDocsBranchName%%', "$(DailyDocsBranchName)" } | ConvertTo-Json) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index d10ae8dab8b4..fbfb76936dcb 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -117,7 +117,20 @@ jobs: python3 eng/versioning/update_versions.py --update-type library --build-type ${{parameters.SDKType}} --sr condition: and(succeeded(), eq(variables['SetDevVersion'],'true')) displayName: Setup Dev versioning - + + - ${{if ne(parameters.ServiceDirectory, '')}}: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 + arguments: > + -ServiceDirectory ${{parameters.ServiceDirectory}} + -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + -AddDevVersion + pwsh: true + workingDirectory: $(Pipeline.Workspace) + displayName: Update package properties with dev version + condition: and(succeeded(), eq(variables['SetDevVersion'],'true')) + - script: | pip install markdown2 BeautifulSoup4 displayName: 'pip install markdown2 and BeautifulSoup4' diff --git a/eng/pipelines/templates/stages/archetype-java-release.yml b/eng/pipelines/templates/stages/archetype-java-release.yml index 3082abb4cf96..a4123aac082f 100644 --- a/eng/pipelines/templates/stages/archetype-java-release.yml +++ b/eng/pipelines/templates/stages/archetype-java-release.yml @@ -197,47 +197,37 @@ stages: ArtifactPath: $(Pipeline.Workspace)/${{parameters.ArtifactName}}-signed - ${{if ne(artifact.skipPublishDocMs, 'true')}}: - - deployment: PublicDocsMs - displayName: "Publish Updates for Docs.MS" + - deployment: PublishDocsMs + displayName: Docs.MS Release condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true')) - environment: github + environment: githubio dependsOn: PublishPackage - variables: - CIConfigs: "{'targets':[{'path_to_config':'package.json','mode':'Preview','monikerid':'1','content_folder':'docs-ref-services/preview'},{'path_to_config':'package.json','mode':'Latest','monikerid':'0','content_folder':'docs-ref-services/latest'}]}" - skipComponentGovernanceDetection: true + pool: - name: Azure Pipelines - vmImage: vs2017-win2016 + name: azsdk-pool-mms-ubuntu-2004-general + vmImage: MMSUbuntu20.04 + strategy: runOnce: deploy: steps: - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: - paths: - - .github/CODEOWNERS - - template: /eng/common/pipelines/templates/steps/get-pr-owners.yml - parameters: - TargetVariable: "OwningGHUser" - ServiceDirectory: ${{parameters.ServiceDirectory}} - - template: /eng/common/pipelines/templates/steps/docs-metadata-release.yml + SkipDefaultCheckout: true + Paths: + - sdk/**/*.md + - download: current + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml parameters: - ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}}-signed/${{artifact.groupId}}/${{artifact.name}} - PackageRepository: Maven - ReleaseSha: $(Build.SourceVersion) + PackageInfoLocations: + - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{artifact.name}}.json WorkingDirectory: $(System.DefaultWorkingDirectory) TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} TargetDocRepoName: ${{parameters.TargetDocRepoName}} - PRBranchName: onboarding-${{artifact.name}}-$(Build.BuildId) - ArtifactName: ${{artifact.name}} Language: 'java' - DocRepoDestinationPath: 'docs-ref-services/' - GHReviewersVariable: 'OwningGHUser' - CIConfigs: $(CIConfigs) - CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' - SkipPackageJson: ${{artifact.skipUpdatePackageJson}} SparseCheckoutPaths: - - docs-ref-services + - docs-ref-services/ + - metadata/ - ${{if ne(artifact.skipPublishDocGithubIo, 'true')}}: - deployment: PublishDocs @@ -372,3 +362,34 @@ stages: parameters: ArtifactName: integration-${{parameters.ArtifactName}}-${{artifact.name}}-javadevfeed-$(System.JobAttempt) ArtifactPath: $(Pipeline.Workspace)/${{parameters.ArtifactName}}-signed + - job: PublishDocsToNightlyBranch + dependsOn: PublishPackages + condition: or(eq(variables['SetDevVersion'], 'true'), and(eq(variables['Build.Reason'],'Schedule'), eq(variables['System.TeamProject'], 'internal'))) + pool: + name: azsdk-pool-mms-ubuntu-2004-general + vmImage: MMSUbuntu20.04 + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + SkipDefaultCheckout: true + Paths: + - sdk/**/*.md + - download: current + - pwsh: | + Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/ + displayName: Show visible artifacts + + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml + parameters: + PackageInfoLocations: + - ${{ each artifact in parameters.Artifacts }}: + - ${{if ne(artifact.skipPublishDocMs, 'true')}}: + - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{artifact.name}}.json + WorkingDirectory: $(System.DefaultWorkingDirectory) + TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} + TargetDocRepoName: ${{parameters.TargetDocRepoName}} + Language: 'java' + DailyDocsBuild: true + SparseCheckoutPaths: + - docs-ref-services/ + - metadata/ \ No newline at end of file diff --git a/eng/repo-docs/docms/daily.update.setting.xml b/eng/repo-docs/docms/daily.update.setting.xml new file mode 100644 index 000000000000..fc1b359a3a62 --- /dev/null +++ b/eng/repo-docs/docms/daily.update.setting.xml @@ -0,0 +1,24 @@ + + + + azure-sdk-for-java + + true + + + + azure-sdk-for-java + https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-java/maven/v1 + + true + + + true + + + + + + \ No newline at end of file diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 478b2ccd2a36..378a8ab70bb1 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -262,6 +262,7 @@ function Update-java-CIConfig($pkgs, $ciRepo, $locationInDocRepo, $monikerId=$nu $PackageExclusions = @{ "azure-core-experimental" = "Don't want to include an experimental package."; + "azure-sdk-bom" = "Don't want to include the sdk bom."; } # Validates if the package will succeed in the CI build by validating the @@ -579,3 +580,27 @@ function GetExistingPackageVersions ($PackageName, $GroupId=$null) return $null } } + +function Get-java-DocsMsMetadataForPackage($PackageInfo) { + $readmeName = $PackageInfo.Name.ToLower() + Write-Host "Docs.ms Readme name: $($readmeName)" + + # Readme names (which are used in the URL) should not include redundant terms + # when viewed in URL form. For example: + # https://review.docs.microsoft.com/en-us/java/api/overview/azure/storage-blob-readme + # Note how the end of the URL doesn't look like: + # ".../azure/azure-storage-blobs-readme" + + # This logic eliminates a preceeding "azure-" in the readme filename. + # "azure-storage-blobs" -> "storage-blobs" + if ($readmeName.StartsWith('azure-')) { + $readmeName = $readmeName.Substring(6) + } + + New-Object PSObject -Property @{ + DocsMsReadMeName = $readmeName + LatestReadMeLocation = 'docs-ref-services/latest' + PreviewReadMeLocation = 'docs-ref-services/preview' + Suffix = '' + } +} \ No newline at end of file