From 91dd8ab840e776e0fb1c955e0462264eeccafd5a Mon Sep 17 00:00:00 2001 From: Sima Zhu Date: Wed, 13 Oct 2021 15:45:50 -0700 Subject: [PATCH 01/15] Daily update docs for java on daily branch --- eng/dailydocsconfig.json | 6 + eng/pipelines/docindex.yml | 81 +++++++++++++- .../stages/archetype-java-release.yml | 105 +++++++++++------- eng/scripts/Language-Settings.ps1 | 24 ++++ 4 files changed, 173 insertions(+), 43 deletions(-) create mode 100644 eng/dailydocsconfig.json diff --git a/eng/dailydocsconfig.json b/eng/dailydocsconfig.json new file mode 100644 index 000000000000..21753a0d032f --- /dev/null +++ b/eng/dailydocsconfig.json @@ -0,0 +1,6 @@ +{ + "target_repo": { + "url": "https://github.com/Azure/azure-docs-sdk-java", + "branch": "%%DailyDocsBranchName%%" + } +} diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index 89ebeec2452c..9c14a0e27350 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -5,7 +5,7 @@ jobs: - job: UpdateDocsMsBuildConfig pool: - vmImage: ubuntu-20.04 + vmImage: windows-2019 variables: DocRepoLocation: $(Pipeline.Workspace)/docs DocRepoOwner: Azure @@ -29,6 +29,7 @@ jobs: filePath: eng/common/scripts/Update-DocsMsPackages.ps1 arguments: -DocRepoLocation $(DocRepoLocation) displayName: Update Docs Onboarding + condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true'))) # Push changes to docs repo - template: /eng/common/pipelines/templates/steps/set-default-branch.yml @@ -43,3 +44,81 @@ jobs: TargetRepoName: $(DocRepoName) TargetRepoOwner: $(DocRepoOwner) WorkingDirectory: $(DocRepoLocation) + + # Prepare daily docs CI + - template: /eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml + parameters: + DailyBranchVariableName: DailyDocsBranchName + - pwsh: | + $ErrorActionPreference = "Continue" + git checkout "origin/$(DailyDocsBranchName)" 2>&1 | Out-Null + $LASTEXITCODE = 0 # This ignores any error from git checkout + git status + displayName: Checkout daily branch if it exists + workingDirectory: $(DocRepoLocation) + - powershell: | + dir $env:USERPROFILE + Get-Command mvn + if (!(Test-Path '$env:USERPROFILE/.m2')) { + mkdir $env:USERPROFILE/.m2 + } + + @" + + + + azure-sdk-for-java + + true + + + + azure-sdk-for-java + https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-java/maven/v1 + + true + + + true + + + + + + + "@ | Set-Content $env:USERPROFILE/.m2/settings.xml + dir $env:USERPROFILE/.m2 + + displayName: 'Configure mvn' + workingDirectory: $(Build.SourcesDirectory) + #condition: and(succeeded(), startsWith(variables['Build.SourceBranchName'], 'daily/')) + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsPackages.ps1 + arguments: -DocRepoLocation $(DocRepoLocation) + displayName: Update Docs Onboarding for Daily branch + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoBranch: $(DailyDocsBranchName) + BaseRepoOwner: $(DocRepoOwner) + CommitMsg: "Update targeting packages based on release metadata. (Daily docs)" + TargetRepoName: $(DocRepoName) + TargetRepoOwner: $(DocRepoOwner) + WorkingDirectory: $(DocRepoLocation) + ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts + + - task: PowerShell@2 + displayName: Queue Docs CI build + inputs: + pwsh: true + filePath: eng/common/scripts/Queue-Pipeline.ps1 + arguments: > + -Organization "apidrop" + -Project "Content%20CI" + -DefinitionId 3188 + -Base64EncodedAuthToken "$(azuresdk-apidrop-devops-queue-build-pat)" + -BuildParametersJson (@{ params = (Get-Content ./eng/dailydocsconfig.json -Raw) -replace '%%DailyDocsBranchName%%', "$(DailyDocsBranchName)" } | ConvertTo-Json) diff --git a/eng/pipelines/templates/stages/archetype-java-release.yml b/eng/pipelines/templates/stages/archetype-java-release.yml index 3082abb4cf96..477441cdcc67 100644 --- a/eng/pipelines/templates/stages/archetype-java-release.yml +++ b/eng/pipelines/templates/stages/archetype-java-release.yml @@ -196,48 +196,38 @@ stages: ArtifactName: ${{parameters.ArtifactName}}-${{artifact.name}}-javadevfeed-$(System.JobAttempt) ArtifactPath: $(Pipeline.Workspace)/${{parameters.ArtifactName}}-signed - - ${{if ne(artifact.skipPublishDocMs, 'true')}}: - - deployment: PublicDocsMs - displayName: "Publish Updates for Docs.MS" - condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true')) - environment: github - dependsOn: PublishPackage - variables: - CIConfigs: "{'targets':[{'path_to_config':'package.json','mode':'Preview','monikerid':'1','content_folder':'docs-ref-services/preview'},{'path_to_config':'package.json','mode':'Latest','monikerid':'0','content_folder':'docs-ref-services/latest'}]}" - skipComponentGovernanceDetection: true - pool: - name: Azure Pipelines - vmImage: vs2017-win2016 - strategy: - runOnce: - deploy: - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - paths: - - .github/CODEOWNERS - - template: /eng/common/pipelines/templates/steps/get-pr-owners.yml - parameters: - TargetVariable: "OwningGHUser" - ServiceDirectory: ${{parameters.ServiceDirectory}} - - template: /eng/common/pipelines/templates/steps/docs-metadata-release.yml - parameters: - ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}}-signed/${{artifact.groupId}}/${{artifact.name}} - PackageRepository: Maven - ReleaseSha: $(Build.SourceVersion) - WorkingDirectory: $(System.DefaultWorkingDirectory) - TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} - TargetDocRepoName: ${{parameters.TargetDocRepoName}} - PRBranchName: onboarding-${{artifact.name}}-$(Build.BuildId) - ArtifactName: ${{artifact.name}} - Language: 'java' - DocRepoDestinationPath: 'docs-ref-services/' - GHReviewersVariable: 'OwningGHUser' - CIConfigs: $(CIConfigs) - CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' - SkipPackageJson: ${{artifact.skipUpdatePackageJson}} - SparseCheckoutPaths: - - docs-ref-services + - ${{if ne(artifact.skipPublishDocMs, 'true')}}: + - deployment: PublishDocs + displayName: Docs.MS Release + condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true')) + environment: githubio + dependsOn: PublishPackage + + pool: + name: azsdk-pool-mms-ubuntu-2004-general + vmImage: MMSUbuntu20.04 + + strategy: + runOnce: + deploy: + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + SkipDefaultCheckout: true + Paths: + - sdk/**/*.md + - download: current + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml + parameters: + PackageInfoLocations: + - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{artifact.name}}.json + WorkingDirectory: $(System.DefaultWorkingDirectory) + TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} + TargetDocRepoName: ${{parameters.TargetDocRepoName}} + Language: 'python' + SparseCheckoutPaths: + - docs-ref-services/ + - metadata/ - ${{if ne(artifact.skipPublishDocGithubIo, 'true')}}: - deployment: PublishDocs @@ -372,3 +362,34 @@ stages: parameters: ArtifactName: integration-${{parameters.ArtifactName}}-${{artifact.name}}-javadevfeed-$(System.JobAttempt) ArtifactPath: $(Pipeline.Workspace)/${{parameters.ArtifactName}}-signed + - job: PublishDocsToNightlyBranch + dependsOn: PublishPackages + condition: or(eq(variables['SetDevVersion'], 'true'), and(eq(variables['Build.Reason'],'Schedule'), eq(variables['System.TeamProject'], 'internal'))) + pool: + name: azsdk-pool-mms-ubuntu-2004-general + vmImage: MMSUbuntu20.04 + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + SkipDefaultCheckout: true + Paths: + - sdk/**/*.md + - download: current + - pwsh: | + Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/ + displayName: Show visible artifacts + + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml + parameters: + PackageInfoLocations: + - ${{ each artifact in parameters.Artifacts }}: + - ${{if ne(artifact.skipPublishDocMs, 'true')}}: + - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{artifact.name}}.json + WorkingDirectory: $(System.DefaultWorkingDirectory) + TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} + TargetDocRepoName: ${{parameters.TargetDocRepoName}} + Language: 'java' + DailyDocsBuild: true + SparseCheckoutPaths: + - docs-ref-services/ + - metadata/ \ No newline at end of file diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 478b2ccd2a36..87dec38a8219 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -579,3 +579,27 @@ function GetExistingPackageVersions ($PackageName, $GroupId=$null) return $null } } + +function Get-java-DocsMsMetadataForPackage($PackageInfo) { + $readmeName = $PackageInfo.Name.ToLower() + Write-Host "Docs.ms Readme name: $($readmeName)" + + # Readme names (which are used in the URL) should not include redundant terms + # when viewed in URL form. For example: + # https://review.docs.microsoft.com/en-us/java/api/overview/azure/storage-blob-readme + # Note how the end of the URL doesn't look like: + # ".../azure/azure-storage-blobs-readme" + + # This logic eliminates a preceeding "azure-" in the readme filename. + # "azure-storage-blobs" -> "storage-blobs" + if ($readmeName.StartsWith('azure-')) { + $readmeName = $readmeName.Substring(6) + } + + New-Object PSObject -Property @{ + DocsMsReadMeName = $readmeName + LatestReadMeLocation = 'docs-ref-services/latest' + PreviewReadMeLocation = 'docs-ref-services/preview' + Suffix = '' + } +} \ No newline at end of file From ef635d105c364c4296ee6cdb72404964625487e4 Mon Sep 17 00:00:00 2001 From: Sima Zhu Date: Wed, 13 Oct 2021 16:44:03 -0700 Subject: [PATCH 02/15] Add dev version in build --- eng/pipelines/templates/jobs/ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index d10ae8dab8b4..f442cf0c459c 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -100,6 +100,19 @@ jobs: - template: /eng/common/pipelines/templates/steps/daily-dev-build-variable.yml parameters: ServiceDirectory: ${{parameters.ServiceDirectory}} + + - ${{if ne(parameters.ServiceDirectory, '')}}: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 + arguments: > + -ServiceDirectory ${{parameters.ServiceDirectory}} + -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + -AddDevVersion + pwsh: true + workingDirectory: $(Pipeline.Workspace) + displayName: Dump Package properties + condition: and(succeeded(), eq(variables['SetDevVersion'],'true') - pwsh: | $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json From a81e3dcc8164929b6e48435332ff4ece873656a0 Mon Sep 17 00:00:00 2001 From: Sima Zhu Date: Wed, 13 Oct 2021 16:52:54 -0700 Subject: [PATCH 03/15] Make the pipeline yaml style correct. --- eng/pipelines/templates/jobs/ci.yml | 2 +- .../stages/archetype-java-release.yml | 60 +++++++++---------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index f442cf0c459c..9759a66e06fb 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -112,7 +112,7 @@ jobs: pwsh: true workingDirectory: $(Pipeline.Workspace) displayName: Dump Package properties - condition: and(succeeded(), eq(variables['SetDevVersion'],'true') + condition: and(succeeded(), eq(variables['SetDevVersion'],'true')) - pwsh: | $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json diff --git a/eng/pipelines/templates/stages/archetype-java-release.yml b/eng/pipelines/templates/stages/archetype-java-release.yml index 477441cdcc67..a4123aac082f 100644 --- a/eng/pipelines/templates/stages/archetype-java-release.yml +++ b/eng/pipelines/templates/stages/archetype-java-release.yml @@ -196,38 +196,38 @@ stages: ArtifactName: ${{parameters.ArtifactName}}-${{artifact.name}}-javadevfeed-$(System.JobAttempt) ArtifactPath: $(Pipeline.Workspace)/${{parameters.ArtifactName}}-signed - - ${{if ne(artifact.skipPublishDocMs, 'true')}}: - - deployment: PublishDocs - displayName: Docs.MS Release - condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true')) - environment: githubio - dependsOn: PublishPackage + - ${{if ne(artifact.skipPublishDocMs, 'true')}}: + - deployment: PublishDocsMs + displayName: Docs.MS Release + condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true')) + environment: githubio + dependsOn: PublishPackage - pool: - name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 + pool: + name: azsdk-pool-mms-ubuntu-2004-general + vmImage: MMSUbuntu20.04 - strategy: - runOnce: - deploy: - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - SkipDefaultCheckout: true - Paths: - - sdk/**/*.md - - download: current - - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml - parameters: - PackageInfoLocations: - - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{artifact.name}}.json - WorkingDirectory: $(System.DefaultWorkingDirectory) - TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} - TargetDocRepoName: ${{parameters.TargetDocRepoName}} - Language: 'python' - SparseCheckoutPaths: - - docs-ref-services/ - - metadata/ + strategy: + runOnce: + deploy: + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + SkipDefaultCheckout: true + Paths: + - sdk/**/*.md + - download: current + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml + parameters: + PackageInfoLocations: + - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{artifact.name}}.json + WorkingDirectory: $(System.DefaultWorkingDirectory) + TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} + TargetDocRepoName: ${{parameters.TargetDocRepoName}} + Language: 'java' + SparseCheckoutPaths: + - docs-ref-services/ + - metadata/ - ${{if ne(artifact.skipPublishDocGithubIo, 'true')}}: - deployment: PublishDocs From d26b7d80f7d03cb8663fb122eaf8b95cbd4309e7 Mon Sep 17 00:00:00 2001 From: Sima Zhu Date: Wed, 13 Oct 2021 23:58:30 -0700 Subject: [PATCH 04/15] Dump the right dev version --- eng/common/scripts/Update-DocsMsPackages.ps1 | 1 + eng/pipelines/templates/jobs/ci.yml | 28 ++++++++++---------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/eng/common/scripts/Update-DocsMsPackages.ps1 b/eng/common/scripts/Update-DocsMsPackages.ps1 index 562cc307bef5..9c79b62a2a2d 100644 --- a/eng/common/scripts/Update-DocsMsPackages.ps1 +++ b/eng/common/scripts/Update-DocsMsPackages.ps1 @@ -61,6 +61,7 @@ function GetDocsMetadataForMoniker($moniker) { RepoPath = $fileObject.ServiceDirectory; Type = $fileObject.SdkType; New = $fileObject.IsNewSdk; + GroupId = $fileObject.Group; } } diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 9759a66e06fb..1d8a4a525852 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -100,19 +100,6 @@ jobs: - template: /eng/common/pipelines/templates/steps/daily-dev-build-variable.yml parameters: ServiceDirectory: ${{parameters.ServiceDirectory}} - - - ${{if ne(parameters.ServiceDirectory, '')}}: - - task: Powershell@2 - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 - arguments: > - -ServiceDirectory ${{parameters.ServiceDirectory}} - -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo - -AddDevVersion - pwsh: true - workingDirectory: $(Pipeline.Workspace) - displayName: Dump Package properties - condition: and(succeeded(), eq(variables['SetDevVersion'],'true')) - pwsh: | $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json @@ -130,7 +117,20 @@ jobs: python3 eng/versioning/update_versions.py --update-type library --build-type ${{parameters.SDKType}} --sr condition: and(succeeded(), eq(variables['SetDevVersion'],'true')) displayName: Setup Dev versioning - + + - ${{if ne(parameters.ServiceDirectory, '')}}: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 + arguments: > + -ServiceDirectory ${{parameters.ServiceDirectory}} + -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + -AddDevVersion + pwsh: true + workingDirectory: $(Pipeline.Workspace) + displayName: Dump Package properties + condition: and(succeeded(), eq(variables['SetDevVersion'],'true')) + - script: | pip install markdown2 BeautifulSoup4 displayName: 'pip install markdown2 and BeautifulSoup4' From d70de3d25db6262a1238f1f0a340752d0b46a705 Mon Sep 17 00:00:00 2001 From: Sima Zhu <48036328+sima-zhu@users.noreply.github.com> Date: Mon, 18 Oct 2021 16:21:55 -0700 Subject: [PATCH 05/15] Update Update-DocsMsPackages.ps1 --- eng/common/scripts/Update-DocsMsPackages.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/common/scripts/Update-DocsMsPackages.ps1 b/eng/common/scripts/Update-DocsMsPackages.ps1 index dbfd38f91df6..7b957c884765 100644 --- a/eng/common/scripts/Update-DocsMsPackages.ps1 +++ b/eng/common/scripts/Update-DocsMsPackages.ps1 @@ -61,7 +61,6 @@ function GetDocsMetadataForMoniker($moniker) { RepoPath = $fileObject.ServiceDirectory; Type = $fileObject.SdkType; New = $fileObject.IsNewSdk; - GroupId = $fileObject.Group; } if ($fileObject.PSObject.Members.Name -contains "Group") { From d07ca1ef341e1fe15013056f33eba38c9e304ff1 Mon Sep 17 00:00:00 2001 From: Sima Zhu Date: Tue, 19 Oct 2021 09:45:34 -0700 Subject: [PATCH 06/15] Switch back to linux --- eng/pipelines/docindex.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index 9c14a0e27350..4ea6c07dcb00 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -5,7 +5,7 @@ jobs: - job: UpdateDocsMsBuildConfig pool: - vmImage: windows-2019 + vmImage: ubuntu-20.04 variables: DocRepoLocation: $(Pipeline.Workspace)/docs DocRepoOwner: Azure @@ -57,10 +57,9 @@ jobs: displayName: Checkout daily branch if it exists workingDirectory: $(DocRepoLocation) - powershell: | - dir $env:USERPROFILE Get-Command mvn - if (!(Test-Path '$env:USERPROFILE/.m2')) { - mkdir $env:USERPROFILE/.m2 + if (!(Test-Path '~/.m2/')) { + mkdir ~/.m2/ } @" @@ -88,8 +87,7 @@ jobs: - "@ | Set-Content $env:USERPROFILE/.m2/settings.xml - dir $env:USERPROFILE/.m2 + "@ | Set-Content ~/.m2/settings.xml displayName: 'Configure mvn' workingDirectory: $(Build.SourcesDirectory) From 578db42b952a22b198b3801350f60206073e1615 Mon Sep 17 00:00:00 2001 From: Sima Zhu <48036328+sima-zhu@users.noreply.github.com> Date: Tue, 19 Oct 2021 21:05:27 -0700 Subject: [PATCH 07/15] Update dailydocsconfig.json --- eng/dailydocsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/dailydocsconfig.json b/eng/dailydocsconfig.json index 21753a0d032f..a9692f8ab707 100644 --- a/eng/dailydocsconfig.json +++ b/eng/dailydocsconfig.json @@ -1,4 +1,5 @@ { + "output_path":"docs-ref-autogen", "target_repo": { "url": "https://github.com/Azure/azure-docs-sdk-java", "branch": "%%DailyDocsBranchName%%" From 5a9ef5290ed92e2e54efe016b62e24ad1dd6bfe4 Mon Sep 17 00:00:00 2001 From: Sima Zhu <48036328+sima-zhu@users.noreply.github.com> Date: Tue, 19 Oct 2021 21:13:18 -0700 Subject: [PATCH 08/15] Update ci.yml --- eng/pipelines/templates/jobs/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 1d8a4a525852..fbfb76936dcb 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -128,7 +128,7 @@ jobs: -AddDevVersion pwsh: true workingDirectory: $(Pipeline.Workspace) - displayName: Dump Package properties + displayName: Update package properties with dev version condition: and(succeeded(), eq(variables['SetDevVersion'],'true')) - script: | From 229b42a21e75be0a63c9336f54c199d24acde842 Mon Sep 17 00:00:00 2001 From: Sima Zhu <48036328+sima-zhu@users.noreply.github.com> Date: Tue, 19 Oct 2021 21:18:15 -0700 Subject: [PATCH 09/15] Update docindex.yml --- eng/pipelines/docindex.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index 4ea6c07dcb00..2e08fa875b88 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -56,7 +56,11 @@ jobs: git status displayName: Checkout daily branch if it exists workingDirectory: $(DocRepoLocation) + + # Docs daily updates is supposed to download packages from public feed repository, so we have to specify additional repositories in a POM or the profile. + # Here is maven documentation: https://maven.apache.org/guides/mini/guide-multiple-repositories.html - powershell: | + # Linux mvn `setting.xml` is sitting under path `~/.m2/setting.xml` Get-Command mvn if (!(Test-Path '~/.m2/')) { mkdir ~/.m2/ From e17d9d8c75f6cc0f565b162bf948f48e451792d6 Mon Sep 17 00:00:00 2001 From: Sima Zhu <48036328+sima-zhu@users.noreply.github.com> Date: Thu, 21 Oct 2021 10:15:46 -0700 Subject: [PATCH 10/15] Update docindex.yml --- eng/pipelines/docindex.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index 2e08fa875b88..48f35624a536 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -95,7 +95,7 @@ jobs: displayName: 'Configure mvn' workingDirectory: $(Build.SourcesDirectory) - #condition: and(succeeded(), startsWith(variables['Build.SourceBranchName'], 'daily/')) + condition: and(succeeded(), startsWith(variables['Build.SourceBranchName'], 'daily/')) - task: Powershell@2 inputs: From ba61f4a8a8f9f1f40f140b04a4b9e36b5779b2f0 Mon Sep 17 00:00:00 2001 From: Sima Zhu Date: Thu, 21 Oct 2021 12:39:24 -0700 Subject: [PATCH 11/15] Pull xml logic in seperate file --- eng/dailyupdateSetting.xml | 24 ++++++++++++++++++++++++ eng/pipelines/docindex.yml | 27 +-------------------------- 2 files changed, 25 insertions(+), 26 deletions(-) create mode 100644 eng/dailyupdateSetting.xml diff --git a/eng/dailyupdateSetting.xml b/eng/dailyupdateSetting.xml new file mode 100644 index 000000000000..fc1b359a3a62 --- /dev/null +++ b/eng/dailyupdateSetting.xml @@ -0,0 +1,24 @@ + + + + azure-sdk-for-java + + true + + + + azure-sdk-for-java + https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-java/maven/v1 + + true + + + true + + + + + + \ No newline at end of file diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index 2e08fa875b88..12c9bb75e0db 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -66,32 +66,7 @@ jobs: mkdir ~/.m2/ } - @" - - - - azure-sdk-for-java - - true - - - - azure-sdk-for-java - https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-java/maven/v1 - - true - - - true - - - - - - - "@ | Set-Content ~/.m2/settings.xml + Get-Content "./eng/dailyupdateSetting.xml" | Set-Content ~/.m2/settings.xml displayName: 'Configure mvn' workingDirectory: $(Build.SourcesDirectory) From 5431c9054541a4ed2419931376e136a193859450 Mon Sep 17 00:00:00 2001 From: Sima Zhu Date: Thu, 21 Oct 2021 13:21:52 -0700 Subject: [PATCH 12/15] Rename the file and change the logic --- eng/pipelines/docindex.yml | 3 +-- .../docms/daily.update.setting.xml} | 0 2 files changed, 1 insertion(+), 2 deletions(-) rename eng/{dailyupdateSetting.xml => repo-docs/docms/daily.update.setting.xml} (100%) diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index ad654925875b..c03d1d2bfa83 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -66,8 +66,7 @@ jobs: mkdir ~/.m2/ } - Get-Content "./eng/dailyupdateSetting.xml" | Set-Content ~/.m2/settings.xml - + Copy-Item "./eng/repo-docs/docms/daily.update.setting.xml" -Destination "~/.m2/settings.xml" displayName: 'Configure mvn' workingDirectory: $(Build.SourcesDirectory) condition: and(succeeded(), startsWith(variables['Build.SourceBranchName'], 'daily/')) diff --git a/eng/dailyupdateSetting.xml b/eng/repo-docs/docms/daily.update.setting.xml similarity index 100% rename from eng/dailyupdateSetting.xml rename to eng/repo-docs/docms/daily.update.setting.xml From e2eceefd079461f0a5df58d11b08b3ab67afeacb Mon Sep 17 00:00:00 2001 From: Sima Zhu Date: Thu, 21 Oct 2021 15:26:24 -0700 Subject: [PATCH 13/15] Added more comments --- eng/pipelines/docindex.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index c03d1d2bfa83..41c5e71cbb1b 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -65,7 +65,9 @@ jobs: if (!(Test-Path '~/.m2/')) { mkdir ~/.m2/ } - + if (Test-Path '~/.m2/setting.xml') { + Write-Host "'setting.xml' exists. Overwriting the file to support multiple repositories." + } Copy-Item "./eng/repo-docs/docms/daily.update.setting.xml" -Destination "~/.m2/settings.xml" displayName: 'Configure mvn' workingDirectory: $(Build.SourcesDirectory) From 819c4d1885e33552ed2bc3e3a41dfa290b0fd9a9 Mon Sep 17 00:00:00 2001 From: Sima Zhu Date: Tue, 26 Oct 2021 11:48:48 -0700 Subject: [PATCH 14/15] Add variable to update mvn setting --- eng/pipelines/docindex.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index 41c5e71cbb1b..dcd9e4f364ab 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -71,7 +71,7 @@ jobs: Copy-Item "./eng/repo-docs/docms/daily.update.setting.xml" -Destination "~/.m2/settings.xml" displayName: 'Configure mvn' workingDirectory: $(Build.SourcesDirectory) - condition: and(succeeded(), startsWith(variables['Build.SourceBranchName'], 'daily/')) + condition: and(succeeded(), or(startsWith(variables['Build.SourceBranchName'], 'daily/'), eq(variables['Force.SettingUpdate'], 'true'))) - task: Powershell@2 inputs: From f4f8d4cd7c071da20f2fecd1df97acb6c68b1ea7 Mon Sep 17 00:00:00 2001 From: Sima Zhu Date: Tue, 26 Oct 2021 12:20:28 -0700 Subject: [PATCH 15/15] Address feedback --- eng/pipelines/docindex.yml | 1 - eng/scripts/Language-Settings.ps1 | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index dcd9e4f364ab..3bea0b440edd 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -71,7 +71,6 @@ jobs: Copy-Item "./eng/repo-docs/docms/daily.update.setting.xml" -Destination "~/.m2/settings.xml" displayName: 'Configure mvn' workingDirectory: $(Build.SourcesDirectory) - condition: and(succeeded(), or(startsWith(variables['Build.SourceBranchName'], 'daily/'), eq(variables['Force.SettingUpdate'], 'true'))) - task: Powershell@2 inputs: diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 87dec38a8219..378a8ab70bb1 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -262,6 +262,7 @@ function Update-java-CIConfig($pkgs, $ciRepo, $locationInDocRepo, $monikerId=$nu $PackageExclusions = @{ "azure-core-experimental" = "Don't want to include an experimental package."; + "azure-sdk-bom" = "Don't want to include the sdk bom."; } # Validates if the package will succeed in the CI build by validating the