From 005a6c2256aafd3c2736bc939302d6d0935dc9f0 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 25 Aug 2023 04:51:49 +0000 Subject: [PATCH 1/8] eng/common changes --- .../Service-Level-Readme-Automation.ps1 | 2 +- .../scripts/Update-DocsMsPackageMonikers.ps1 | 85 +++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 eng/common/scripts/Update-DocsMsPackageMonikers.ps1 diff --git a/eng/common/scripts/Service-Level-Readme-Automation.ps1 b/eng/common/scripts/Service-Level-Readme-Automation.ps1 index 2d0e5c67920a..02d429b52ffe 100644 --- a/eng/common/scripts/Service-Level-Readme-Automation.ps1 +++ b/eng/common/scripts/Service-Level-Readme-Automation.ps1 @@ -50,7 +50,7 @@ param( Set-StrictMode -Version 3 $fullMetadata = Get-CSVMetadata -$monikers = @("latest", "preview") +$monikers = @("latest", "preview", "legacy") foreach($moniker in $monikers) { # The onboarded packages return is key-value pair, which key is the package index, and value is the package info from {metadata}.json # E.g. diff --git a/eng/common/scripts/Update-DocsMsPackageMonikers.ps1 b/eng/common/scripts/Update-DocsMsPackageMonikers.ps1 new file mode 100644 index 000000000000..20fb6443cf41 --- /dev/null +++ b/eng/common/scripts/Update-DocsMsPackageMonikers.ps1 @@ -0,0 +1,85 @@ +param( + [Parameter(Mandatory = $true)] + [string] $DocRepoLocation +) + +. (Join-Path $PSScriptRoot common.ps1) + +Set-StrictMode -Version 3 + +function getPackageMetadata($moniker) { + $jsonFiles = Get-ChildItem -Path (Join-Path $DocRepoLocation "metadata/$moniker") -Filter *.json + $metadata = @{} + + foreach ($jsonFile in $jsonFiles) { + $packageMetadata = Get-Content $jsonFile -Raw | ConvertFrom-Json -AsHashtable + $packageIdentity = $packageMetadata.Name + if (Test-Path "Function:$GetPackageIdentity") { + $packageIdentity = &$GetPackageIdentity $packageMetadata + } + + $metadata[$packageIdentity] = @{ File = $jsonFile; Metadata = $packageMetadata } + } + + return $metadata +} + +function getPackageMetadataFileLocation($packageIdentity, $lookupTable) { + if ($lookupTable.ContainsKey($packageIdentity)) { + return $lookupTable[$packageIdentity]['File'] + } + + return $null +} + +$metadataLookup = @{ + 'latest' = getPackageMetadata 'latest' + 'preview' = getPackageMetadata 'preview' + 'legacy' = getPackageMetadata 'legacy' +} +$deprecatedPackages = (Get-CSVMetadata).Where({ $_.Support -eq 'deprecated' }) + +foreach ($package in $deprecatedPackages) { + $packageIdentity = $package.Package + # TODO: Ensure this works + if (Test-Path "Function:$GetPackageIdentity") { + $packageIdentity = &$GetPackageIdentity $package + } + + $previewMetadataPath = getPackageMetadataFileLocation ` + -packageIdentity $packageIdentity ` + -lookupTable $metadataLookup['preview'] + + if ($previewMetadataPath) { + $metadata = &$GetDocsMsMetadataForPackageFn -PackageInfo $metadataLookup['preview'][$packageIdentity]['Metadata'] + + Write-Host "Package $packageIdentity is deprecated but has file in preview metadata folder. Moving to legacy." + Move-Item $previewMetadataPath "$DocRepoLocation/metadata/legacy/" -Force + + + if (Test-Path "$DocRepoLocation/$($metadata.PreviewReadMeLocation)/$($metadata.DocsMsReadMeName)-readme.md") { + Move-Item ` + "$DocRepoLocation/$($metadata.PreviewReadMeLocation)/$($metadata.DocsMsReadMeName)-readme.md" ` + "$DocRepoLocation/$($metadata.LegacyReadMeLocation)/" ` + -Force + } + + } + + $latestMetadataPath = getPackageMetadataFileLocation ` + -packageIdentity $packageIdentity ` + -lookupTable $metadataLookup['latest'] + + if ($latestMetadataPath) { + $metadata = &$GetDocsMsMetadataForPackageFn -PackageInfo $metadataLookup['latest'][$packageIdentity]['Metadata'] + + Write-Host "Package $packageIdentity is deprecated but has file in latest metadata folder. Moving to legacy. (might overwrite preview version if it exists in metadata/legacy)" + Move-Item $latestMetadataPath "$DocRepoLocation/metadata/legacy/" -Force + if (Test-Path "$DocRepoLocation/$($metadata.LatestReadMeLocation)/$($metadata.DocsMsReadMeName)-readme.md") { + Move-Item ` + "$DocRepoLocation/$($metadata.LatestReadMeLocation)/$($metadata.DocsMsReadMeName)-readme.md" ` + "$DocRepoLocation/$($metadata.LegacyReadMeLocation)/" ` + -Force + } + } +} From 3d6afb0114a880e8a54b4580eccbc25d883db02c Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 25 Aug 2023 04:54:43 +0000 Subject: [PATCH 2/8] Test changes --- eng/pipelines/docindex.yml | 137 ++++++++++++++++-------------- eng/scripts/Language-Settings.ps1 | 1 + 2 files changed, 75 insertions(+), 63 deletions(-) diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index fb38d9da14ec..222e34a33c22 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -26,6 +26,8 @@ jobs: Repositories: - Name: $(DocRepoOwner)/$(DocRepoName) WorkingDirectory: $(DocRepoLocation) + # TODO: Remove this + Commitish: djurek/deprecated-packages-testing # Pull and build the docker image. - template: /eng/common/pipelines/templates/steps/docker-pull-image.yml parameters: @@ -33,6 +35,14 @@ jobs: ContainerRegistryClientSecret: $(azuresdkimages-cr-clientsecret) ImageId: "$(DocValidationImageId)" + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsPackageMonikers.ps1 + arguments: -DocRepoLocation $(DocRepoLocation) + displayName: Move deprecated packages to legacy moniker + condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true'))) + # Call update docs ci script to onboard packages - task: Powershell@2 inputs: @@ -79,7 +89,8 @@ jobs: - template: /eng/common/pipelines/templates/steps/git-push-changes.yml parameters: - BaseRepoBranch: $(DefaultBranch) + # BaseRepoBranch: $(DefaultBranch) + BaseRepoBranch: djurek/deprecated-packages-testing BaseRepoOwner: $(DocRepoOwner) CommitMsg: "Update docs CI configuration" TargetRepoName: $(DocRepoName) @@ -87,70 +98,70 @@ jobs: WorkingDirectory: $(DocRepoLocation) # Prepare daily docs CI - - template: /eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml - parameters: - DailyBranchVariableName: DailyDocsBranchName - - pwsh: | - $ErrorActionPreference = "Continue" - git checkout "origin/$(DailyDocsBranchName)" 2>&1 | Out-Null - $LASTEXITCODE = 0 # This ignores any error from git checkout - git status - displayName: Checkout daily branch if it exists - workingDirectory: $(DocRepoLocation) - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Update-DocsMsPackages.ps1 - arguments: -DocRepoLocation $(DocRepoLocation) - displayName: Update Docs Onboarding for Daily docs + # - template: /eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml + # parameters: + # DailyBranchVariableName: DailyDocsBranchName + # - pwsh: | + # $ErrorActionPreference = "Continue" + # git checkout "origin/$(DailyDocsBranchName)" 2>&1 | Out-Null + # $LASTEXITCODE = 0 # This ignores any error from git checkout + # git status + # displayName: Checkout daily branch if it exists + # workingDirectory: $(DocRepoLocation) + # - task: Powershell@2 + # inputs: + # pwsh: true + # filePath: eng/common/scripts/Update-DocsMsPackages.ps1 + # arguments: -DocRepoLocation $(DocRepoLocation) + # displayName: Update Docs Onboarding for Daily docs - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Service-Level-Readme-Automation.ps1 - arguments: >- - -DocRepoLocation $(DocRepoLocation) - -TenantId '$(opensource-aad-tenant-id)' - -ClientId '$(opensource-aad-app-id)' - -ClientSecret '$(opensource-aad-secret)' - displayName: Generate Service Level Readme for Daily docs + # - task: Powershell@2 + # inputs: + # pwsh: true + # filePath: eng/common/scripts/Service-Level-Readme-Automation.ps1 + # arguments: >- + # -DocRepoLocation $(DocRepoLocation) + # -TenantId '$(opensource-aad-tenant-id)' + # -ClientId '$(opensource-aad-app-id)' + # -ClientSecret '$(opensource-aad-secret)' + # displayName: Generate Service Level Readme for Daily docs - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Update-DocsMsToc.ps1 - arguments: >- - -DocRepoLocation $(DocRepoLocation) - -OutputLocation $(DocRepoLocation)/docs-ref-mapping/reference-unified.yml - displayName: Generate ToC for Daily docs + # - task: Powershell@2 + # inputs: + # pwsh: true + # filePath: eng/common/scripts/Update-DocsMsToc.ps1 + # arguments: >- + # -DocRepoLocation $(DocRepoLocation) + # -OutputLocation $(DocRepoLocation)/docs-ref-mapping/reference-unified.yml + # displayName: Generate ToC for Daily docs - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Verify-RequiredDocsJsonMembers.ps1 - arguments: >- - -DocRepoLocation $(DocRepoLocation) - displayName: Verify Required Docs Json Members + # - task: Powershell@2 + # inputs: + # pwsh: true + # filePath: eng/common/scripts/Verify-RequiredDocsJsonMembers.ps1 + # arguments: >- + # -DocRepoLocation $(DocRepoLocation) + # displayName: Verify Required Docs Json Members - - template: /eng/common/pipelines/templates/steps/git-push-changes.yml - parameters: - BaseRepoBranch: $(DailyDocsBranchName) - BaseRepoOwner: $(DocRepoOwner) - CommitMsg: "Update targeting packages based on release metadata. (Daily docs)" - TargetRepoName: $(DocRepoName) - TargetRepoOwner: $(DocRepoOwner) - WorkingDirectory: $(DocRepoLocation) - ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts - PushArgs: -f + # - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + # parameters: + # BaseRepoBranch: $(DailyDocsBranchName) + # BaseRepoOwner: $(DocRepoOwner) + # CommitMsg: "Update targeting packages based on release metadata. (Daily docs)" + # TargetRepoName: $(DocRepoName) + # TargetRepoOwner: $(DocRepoOwner) + # WorkingDirectory: $(DocRepoLocation) + # ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts + # PushArgs: -f - - task: PowerShell@2 - displayName: Queue Docs CI build - inputs: - pwsh: true - filePath: eng/common/scripts/Queue-Pipeline.ps1 - arguments: > - -Organization "apidrop" - -Project "Content%20CI" - -DefinitionId 3452 - -AuthToken "$(azuresdk-apidrop-devops-queue-build-pat)" - -BuildParametersJson '{"params":"{ \"target_repo\": { \"url\": \"https://github.com/MicrosoftDocs/azure-docs-sdk-node\", \"branch\": \"$(DailyDocsBranchName)\", \"folder\": \"./\" }, \"source_repos\": [] }"}' + # - task: PowerShell@2 + # displayName: Queue Docs CI build + # inputs: + # pwsh: true + # filePath: eng/common/scripts/Queue-Pipeline.ps1 + # arguments: > + # -Organization "apidrop" + # -Project "Content%20CI" + # -DefinitionId 3452 + # -AuthToken "$(azuresdk-apidrop-devops-queue-build-pat)" + # -BuildParametersJson '{"params":"{ \"target_repo\": { \"url\": \"https://github.com/MicrosoftDocs/azure-docs-sdk-node\", \"branch\": \"$(DailyDocsBranchName)\", \"folder\": \"./\" }, \"source_repos\": [] }"}' diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 4742cb0374c8..11ad9fd0f702 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -128,6 +128,7 @@ function Get-javascript-DocsMsMetadataForPackage($PackageInfo) { DocsMsReadMeName = $docsReadmeName LatestReadMeLocation = 'docs-ref-services/latest' PreviewReadMeLocation = 'docs-ref-services/preview' + LegacyReadMeLocation = 'docs-ref-services/legacy' Suffix = '' } } From 4250d4a0506c40b7f4536b0d0ae1256cf7157aad Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 29 Aug 2023 17:02:06 +0000 Subject: [PATCH 3/8] Revert Service-Level-Readme-Automation.ps1 --- eng/common/scripts/Service-Level-Readme-Automation.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/Service-Level-Readme-Automation.ps1 b/eng/common/scripts/Service-Level-Readme-Automation.ps1 index 02d429b52ffe..2d0e5c67920a 100644 --- a/eng/common/scripts/Service-Level-Readme-Automation.ps1 +++ b/eng/common/scripts/Service-Level-Readme-Automation.ps1 @@ -50,7 +50,7 @@ param( Set-StrictMode -Version 3 $fullMetadata = Get-CSVMetadata -$monikers = @("latest", "preview", "legacy") +$monikers = @("latest", "preview") foreach($moniker in $monikers) { # The onboarded packages return is key-value pair, which key is the package index, and value is the package info from {metadata}.json # E.g. From f7968f08e39de5a00f39cf85349a50629bf3740b Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 29 Aug 2023 20:50:03 +0000 Subject: [PATCH 4/8] Requrie version match to deprecate --- .../scripts/Update-DocsMsPackageMonikers.ps1 | 114 +++++++++--------- 1 file changed, 56 insertions(+), 58 deletions(-) diff --git a/eng/common/scripts/Update-DocsMsPackageMonikers.ps1 b/eng/common/scripts/Update-DocsMsPackageMonikers.ps1 index 20fb6443cf41..43f10625416e 100644 --- a/eng/common/scripts/Update-DocsMsPackageMonikers.ps1 +++ b/eng/common/scripts/Update-DocsMsPackageMonikers.ps1 @@ -1,6 +1,6 @@ param( - [Parameter(Mandatory = $true)] - [string] $DocRepoLocation + [Parameter(Mandatory = $true)] + [string] $DocRepoLocation ) . (Join-Path $PSScriptRoot common.ps1) @@ -8,78 +8,76 @@ param( Set-StrictMode -Version 3 function getPackageMetadata($moniker) { - $jsonFiles = Get-ChildItem -Path (Join-Path $DocRepoLocation "metadata/$moniker") -Filter *.json - $metadata = @{} + $jsonFiles = Get-ChildItem -Path (Join-Path $DocRepoLocation "metadata/$moniker") -Filter *.json + $metadata = @{} - foreach ($jsonFile in $jsonFiles) { - $packageMetadata = Get-Content $jsonFile -Raw | ConvertFrom-Json -AsHashtable - $packageIdentity = $packageMetadata.Name - if (Test-Path "Function:$GetPackageIdentity") { - $packageIdentity = &$GetPackageIdentity $packageMetadata - } - - $metadata[$packageIdentity] = @{ File = $jsonFile; Metadata = $packageMetadata } + foreach ($jsonFile in $jsonFiles) { + $packageMetadata = Get-Content $jsonFile -Raw | ConvertFrom-Json -AsHashtable + $packageIdentity = $packageMetadata.Name + if (Test-Path "Function:$GetPackageIdentity") { + $packageIdentity = &$GetPackageIdentity $packageMetadata } + + $metadata[$packageIdentity] = @{ File = $jsonFile; Metadata = $packageMetadata } + } - return $metadata + return $metadata } -function getPackageMetadataFileLocation($packageIdentity, $lookupTable) { - if ($lookupTable.ContainsKey($packageIdentity)) { - return $lookupTable[$packageIdentity]['File'] +function getPackageMetadataFileLocation($packageIdentity, $packageVersion, $lookupTable) { + if ($lookupTable.ContainsKey($packageIdentity)) { + # Only return a metadata file if the version matches a deprecated version + if ($lookupTable[$packageIdentity]['Metadata'].Version -eq $packageVersion) { + return $lookupTable[$packageIdentity]['File'] } + } - return $null + return $null } $metadataLookup = @{ - 'latest' = getPackageMetadata 'latest' - 'preview' = getPackageMetadata 'preview' - 'legacy' = getPackageMetadata 'legacy' + 'latest' = getPackageMetadata 'latest' + 'preview' = getPackageMetadata 'preview' } $deprecatedPackages = (Get-CSVMetadata).Where({ $_.Support -eq 'deprecated' }) foreach ($package in $deprecatedPackages) { - $packageIdentity = $package.Package - # TODO: Ensure this works - if (Test-Path "Function:$GetPackageIdentity") { - $packageIdentity = &$GetPackageIdentity $package - } - + $packageIdentity = $package.Package + # TODO: Ensure this works + if (Test-Path "Function:$GetPackageIdentity") { + $packageIdentity = &$GetPackageIdentity $package + } + + # In cases where $targetPackages contains both a preview and a latest moniker + # it's possible that both metadata files will be moved. In this case, the + # sequence is important and the GA version will win over the Preview version + $targetPackages = @() + if ($package.VersionPreview) { + $targetPackages += @{ Moniker = 'preview'; Version = $package.VersionPreview } + } + if ($package.VersionGA) { + $targetPackages += @{ Moniker = 'latest'; Version = $package.VersionGA } + } + + foreach ($targetPackage in $targetPackages) { $previewMetadataPath = getPackageMetadataFileLocation ` - -packageIdentity $packageIdentity ` - -lookupTable $metadataLookup['preview'] + -packageIdentity $packageIdentity ` + -packageVersion $targetPackage.Version ` + -lookupTable $metadataLookup['preview'] if ($previewMetadataPath) { - $metadata = &$GetDocsMsMetadataForPackageFn -PackageInfo $metadataLookup['preview'][$packageIdentity]['Metadata'] - - Write-Host "Package $packageIdentity is deprecated but has file in preview metadata folder. Moving to legacy." - Move-Item $previewMetadataPath "$DocRepoLocation/metadata/legacy/" -Force - - - if (Test-Path "$DocRepoLocation/$($metadata.PreviewReadMeLocation)/$($metadata.DocsMsReadMeName)-readme.md") { - Move-Item ` - "$DocRepoLocation/$($metadata.PreviewReadMeLocation)/$($metadata.DocsMsReadMeName)-readme.md" ` - "$DocRepoLocation/$($metadata.LegacyReadMeLocation)/" ` - -Force - } - - } - - $latestMetadataPath = getPackageMetadataFileLocation ` - -packageIdentity $packageIdentity ` - -lookupTable $metadataLookup['latest'] - - if ($latestMetadataPath) { - $metadata = &$GetDocsMsMetadataForPackageFn -PackageInfo $metadataLookup['latest'][$packageIdentity]['Metadata'] - - Write-Host "Package $packageIdentity is deprecated but has file in latest metadata folder. Moving to legacy. (might overwrite preview version if it exists in metadata/legacy)" - Move-Item $latestMetadataPath "$DocRepoLocation/metadata/legacy/" -Force - if (Test-Path "$DocRepoLocation/$($metadata.LatestReadMeLocation)/$($metadata.DocsMsReadMeName)-readme.md") { - Move-Item ` - "$DocRepoLocation/$($metadata.LatestReadMeLocation)/$($metadata.DocsMsReadMeName)-readme.md" ` - "$DocRepoLocation/$($metadata.LegacyReadMeLocation)/" ` - -Force - } + $metadata = &$GetDocsMsMetadataForPackageFn ` + -PackageInfo $metadataLookup[$targetPackage.Moniker][$packageIdentity]['Metadata'] + + Write-Host "Package $packageIdentity is deprecated but has file in preview metadata folder. Moving to legacy." + Move-Item $previewMetadataPath "$DocRepoLocation/metadata/legacy/" -Force + + if (Test-Path "$DocRepoLocation/$($metadata.PreviewReadMeLocation)/$($metadata.DocsMsReadMeName)-readme.md") { + Move-Item ` + "$DocRepoLocation/$($metadata.PreviewReadMeLocation)/$($metadata.DocsMsReadMeName)-readme.md" ` + "$DocRepoLocation/$($metadata.LegacyReadMeLocation)/" ` + -Force + } } + } } From a65bcc7a9d8648f429eca44ba412e49fa54a581f Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 30 Aug 2023 22:53:54 +0000 Subject: [PATCH 5/8] Update --- .../scripts/Update-DocsMsPackageMonikers.ps1 | 103 ++++++++++++------ 1 file changed, 72 insertions(+), 31 deletions(-) diff --git a/eng/common/scripts/Update-DocsMsPackageMonikers.ps1 b/eng/common/scripts/Update-DocsMsPackageMonikers.ps1 index 43f10625416e..f10baa44d118 100644 --- a/eng/common/scripts/Update-DocsMsPackageMonikers.ps1 +++ b/eng/common/scripts/Update-DocsMsPackageMonikers.ps1 @@ -1,3 +1,16 @@ +<# +.SYNOPSIS +Move metadata JSON and package-level overview markdown files for deprecated packages to the legacy folder. + +.DESCRIPTION +Move onboarding information to the "legacy" moniker for whose support is "deprecated" in the Metadata CSV. +Only one version of a package can be documented in the "legacy" moniker. If multiple versions are available, +the "latest" version will be used and the "preview" version will be deleted. + +.PARAMETER DocRepoLocation +The location of the target docs repository. +#> + param( [Parameter(Mandatory = $true)] [string] $DocRepoLocation @@ -24,17 +37,46 @@ function getPackageMetadata($moniker) { return $metadata } -function getPackageMetadataFileLocation($packageIdentity, $packageVersion, $lookupTable) { +function getPackageInfoFromLookup($packageIdentity, $version, $lookupTable) { if ($lookupTable.ContainsKey($packageIdentity)) { - # Only return a metadata file if the version matches a deprecated version - if ($lookupTable[$packageIdentity]['Metadata'].Version -eq $packageVersion) { - return $lookupTable[$packageIdentity]['File'] + if ($lookupTable[$packageIdentity]['Metadata'].Version -eq $version) { + # Only return if the version matches + return $lookupTable[$packageIdentity] } } return $null } +function moveToLegacy($packageInfo) { + $docsMsMetadata = &$GetDocsMsMetadataForPackageFn -PackageInfo $packageInfo['Metadata'] + + Write-Host "Move to legacy: $($packageInfo['Metadata'].Name)" + $packageInfoPath = $packageInfo['File'] + Move-Item "$($packageInfoPath.Directory)/$($packageInfoPath.BaseName).*" "$DocRepoLocation/metadata/legacy/" -Force + + $readmePath = "$DocRepoLocation/$($docsMsMetadata.PreviewReadMeLocation)/$($docsMsMetadata.DocsMsReadMeName)-readme.md" + if (Test-Path $readmePath) { + Move-Item ` + $readmePath ` + "$DocRepoLocation/$($docsMsMetadata.LegacyReadMeLocation)/" ` + -Force + } +} + +function deletePackageInfo($packageInfo) { + $docsMsMetadata = &$GetDocsMsMetadataForPackageFn -PackageInfo $packageInfo['Metadata'] + + Write-Host "Delete superseded package: $($packageInfo['Metadata'].Name)" + $packageInfoPath = $packageInfo['File'] + Remove-Item "$($packageInfoPath.Directory)/$($packageInfoPath.BaseName).*" -Force + + $readmePath = "$DocRepoLocation/$($docsMsMetadata.PreviewReadMeLocation)/$($docsMsMetadata.DocsMsReadMeName)-readme.md" + if (Test-Path $readmePath) { + Remove-Item $readmePath -Force + } +} + $metadataLookup = @{ 'latest' = getPackageMetadata 'latest' 'preview' = getPackageMetadata 'preview' @@ -43,41 +85,40 @@ $deprecatedPackages = (Get-CSVMetadata).Where({ $_.Support -eq 'deprecated' }) foreach ($package in $deprecatedPackages) { $packageIdentity = $package.Package - # TODO: Ensure this works + # TODO: Ensure this works in Java if (Test-Path "Function:$GetPackageIdentity") { $packageIdentity = &$GetPackageIdentity $package } - # In cases where $targetPackages contains both a preview and a latest moniker - # it's possible that both metadata files will be moved. In this case, the - # sequence is important and the GA version will win over the Preview version - $targetPackages = @() - if ($package.VersionPreview) { - $targetPackages += @{ Moniker = 'preview'; Version = $package.VersionPreview } - } - if ($package.VersionGA) { - $targetPackages += @{ Moniker = 'latest'; Version = $package.VersionGA } - } - - foreach ($targetPackage in $targetPackages) { - $previewMetadataPath = getPackageMetadataFileLocation ` + $packageInfoPreview = $packageInfoLatest = $null + if ($package.VersionPreview) { + $packageInfoPreview = getPackageInfoFromLookup ` -packageIdentity $packageIdentity ` - -packageVersion $targetPackage.Version ` + -version $package.VersionPreview ` -lookupTable $metadataLookup['preview'] + } - if ($previewMetadataPath) { - $metadata = &$GetDocsMsMetadataForPackageFn ` - -PackageInfo $metadataLookup[$targetPackage.Moniker][$packageIdentity]['Metadata'] + if ($package.VersionGA) { + $packageInfoLatest = getPackageInfoFromLookup ` + -packageIdentity $packageIdentity ` + -version $package.VersionGA ` + -lookupTable $metadataLookup['latest'] + } - Write-Host "Package $packageIdentity is deprecated but has file in preview metadata folder. Moving to legacy." - Move-Item $previewMetadataPath "$DocRepoLocation/metadata/legacy/" -Force + if (!$packageInfoPreview -and !$packageInfoLatest) { + # Nothing to move or delete + continue + } - if (Test-Path "$DocRepoLocation/$($metadata.PreviewReadMeLocation)/$($metadata.DocsMsReadMeName)-readme.md") { - Move-Item ` - "$DocRepoLocation/$($metadata.PreviewReadMeLocation)/$($metadata.DocsMsReadMeName)-readme.md" ` - "$DocRepoLocation/$($metadata.LegacyReadMeLocation)/" ` - -Force - } - } + if ($packageInfoPreview -and $packageInfoLatest) { + # Delete metadata JSON and package-level overview markdown files for + # the preview version instead of moving both. This mitigates situations + # where the "latest" verison doesn't have a package-level overview + # markdown file and the "preview" version does. + deletePackageInfo $packageInfoPreview + moveToLegacy $packageInfoLatest + } + else { + moveToLegacy ($packageInfoPreview ?? $packageInfoLatest) } } From d0b76ad0c4983b71fef075fe96378800a3d9dd9a Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 31 Aug 2023 04:46:19 +0000 Subject: [PATCH 6/8] Add legacy moniker to Service-Level-Readme-Automation.ps1 --- eng/common/scripts/Service-Level-Readme-Automation.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/common/scripts/Service-Level-Readme-Automation.ps1 b/eng/common/scripts/Service-Level-Readme-Automation.ps1 index 2d0e5c67920a..66646bc0b7c9 100644 --- a/eng/common/scripts/Service-Level-Readme-Automation.ps1 +++ b/eng/common/scripts/Service-Level-Readme-Automation.ps1 @@ -50,8 +50,8 @@ param( Set-StrictMode -Version 3 $fullMetadata = Get-CSVMetadata -$monikers = @("latest", "preview") -foreach($moniker in $monikers) { +$monikers = @("latest", "preview", "legacy") +foreach ($moniker in $monikers) { # The onboarded packages return is key-value pair, which key is the package index, and value is the package info from {metadata}.json # E.g. # Key as: @azure/storage-blob @@ -73,10 +73,10 @@ foreach($moniker in $monikers) { $onboardedPackages = &$GetOnboardedDocsMsPackagesForMonikerFn ` -DocRepoLocation $DocRepoLocation -moniker $moniker $csvMetadata = @() - foreach($metadataEntry in $fullMetadata) { + foreach ($metadataEntry in $fullMetadata) { if ($metadataEntry.Package -and $metadataEntry.Hide -ne 'true') { $pkgKey = GetPackageKey $metadataEntry - if($onboardedPackages.ContainsKey($pkgKey)) { + if ($onboardedPackages.ContainsKey($pkgKey)) { if ($onboardedPackages[$pkgKey] -and $onboardedPackages[$pkgKey].DirectoryPath) { if (!($metadataEntry.PSObject.Members.Name -contains "DirectoryPath")) { Add-Member -InputObject $metadataEntry ` From 5af69a905849616e3ec7b9203e261afb5272c1cd Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 6 Sep 2023 15:17:17 +0000 Subject: [PATCH 7/8] Revert eng/common changes --- .../Service-Level-Readme-Automation.ps1 | 8 +- .../scripts/Update-DocsMsPackageMonikers.ps1 | 124 ------------------ 2 files changed, 4 insertions(+), 128 deletions(-) delete mode 100644 eng/common/scripts/Update-DocsMsPackageMonikers.ps1 diff --git a/eng/common/scripts/Service-Level-Readme-Automation.ps1 b/eng/common/scripts/Service-Level-Readme-Automation.ps1 index 66646bc0b7c9..2d0e5c67920a 100644 --- a/eng/common/scripts/Service-Level-Readme-Automation.ps1 +++ b/eng/common/scripts/Service-Level-Readme-Automation.ps1 @@ -50,8 +50,8 @@ param( Set-StrictMode -Version 3 $fullMetadata = Get-CSVMetadata -$monikers = @("latest", "preview", "legacy") -foreach ($moniker in $monikers) { +$monikers = @("latest", "preview") +foreach($moniker in $monikers) { # The onboarded packages return is key-value pair, which key is the package index, and value is the package info from {metadata}.json # E.g. # Key as: @azure/storage-blob @@ -73,10 +73,10 @@ foreach ($moniker in $monikers) { $onboardedPackages = &$GetOnboardedDocsMsPackagesForMonikerFn ` -DocRepoLocation $DocRepoLocation -moniker $moniker $csvMetadata = @() - foreach ($metadataEntry in $fullMetadata) { + foreach($metadataEntry in $fullMetadata) { if ($metadataEntry.Package -and $metadataEntry.Hide -ne 'true') { $pkgKey = GetPackageKey $metadataEntry - if ($onboardedPackages.ContainsKey($pkgKey)) { + if($onboardedPackages.ContainsKey($pkgKey)) { if ($onboardedPackages[$pkgKey] -and $onboardedPackages[$pkgKey].DirectoryPath) { if (!($metadataEntry.PSObject.Members.Name -contains "DirectoryPath")) { Add-Member -InputObject $metadataEntry ` diff --git a/eng/common/scripts/Update-DocsMsPackageMonikers.ps1 b/eng/common/scripts/Update-DocsMsPackageMonikers.ps1 deleted file mode 100644 index f10baa44d118..000000000000 --- a/eng/common/scripts/Update-DocsMsPackageMonikers.ps1 +++ /dev/null @@ -1,124 +0,0 @@ -<# -.SYNOPSIS -Move metadata JSON and package-level overview markdown files for deprecated packages to the legacy folder. - -.DESCRIPTION -Move onboarding information to the "legacy" moniker for whose support is "deprecated" in the Metadata CSV. -Only one version of a package can be documented in the "legacy" moniker. If multiple versions are available, -the "latest" version will be used and the "preview" version will be deleted. - -.PARAMETER DocRepoLocation -The location of the target docs repository. -#> - -param( - [Parameter(Mandatory = $true)] - [string] $DocRepoLocation -) - -. (Join-Path $PSScriptRoot common.ps1) - -Set-StrictMode -Version 3 - -function getPackageMetadata($moniker) { - $jsonFiles = Get-ChildItem -Path (Join-Path $DocRepoLocation "metadata/$moniker") -Filter *.json - $metadata = @{} - - foreach ($jsonFile in $jsonFiles) { - $packageMetadata = Get-Content $jsonFile -Raw | ConvertFrom-Json -AsHashtable - $packageIdentity = $packageMetadata.Name - if (Test-Path "Function:$GetPackageIdentity") { - $packageIdentity = &$GetPackageIdentity $packageMetadata - } - - $metadata[$packageIdentity] = @{ File = $jsonFile; Metadata = $packageMetadata } - } - - return $metadata -} - -function getPackageInfoFromLookup($packageIdentity, $version, $lookupTable) { - if ($lookupTable.ContainsKey($packageIdentity)) { - if ($lookupTable[$packageIdentity]['Metadata'].Version -eq $version) { - # Only return if the version matches - return $lookupTable[$packageIdentity] - } - } - - return $null -} - -function moveToLegacy($packageInfo) { - $docsMsMetadata = &$GetDocsMsMetadataForPackageFn -PackageInfo $packageInfo['Metadata'] - - Write-Host "Move to legacy: $($packageInfo['Metadata'].Name)" - $packageInfoPath = $packageInfo['File'] - Move-Item "$($packageInfoPath.Directory)/$($packageInfoPath.BaseName).*" "$DocRepoLocation/metadata/legacy/" -Force - - $readmePath = "$DocRepoLocation/$($docsMsMetadata.PreviewReadMeLocation)/$($docsMsMetadata.DocsMsReadMeName)-readme.md" - if (Test-Path $readmePath) { - Move-Item ` - $readmePath ` - "$DocRepoLocation/$($docsMsMetadata.LegacyReadMeLocation)/" ` - -Force - } -} - -function deletePackageInfo($packageInfo) { - $docsMsMetadata = &$GetDocsMsMetadataForPackageFn -PackageInfo $packageInfo['Metadata'] - - Write-Host "Delete superseded package: $($packageInfo['Metadata'].Name)" - $packageInfoPath = $packageInfo['File'] - Remove-Item "$($packageInfoPath.Directory)/$($packageInfoPath.BaseName).*" -Force - - $readmePath = "$DocRepoLocation/$($docsMsMetadata.PreviewReadMeLocation)/$($docsMsMetadata.DocsMsReadMeName)-readme.md" - if (Test-Path $readmePath) { - Remove-Item $readmePath -Force - } -} - -$metadataLookup = @{ - 'latest' = getPackageMetadata 'latest' - 'preview' = getPackageMetadata 'preview' -} -$deprecatedPackages = (Get-CSVMetadata).Where({ $_.Support -eq 'deprecated' }) - -foreach ($package in $deprecatedPackages) { - $packageIdentity = $package.Package - # TODO: Ensure this works in Java - if (Test-Path "Function:$GetPackageIdentity") { - $packageIdentity = &$GetPackageIdentity $package - } - - $packageInfoPreview = $packageInfoLatest = $null - if ($package.VersionPreview) { - $packageInfoPreview = getPackageInfoFromLookup ` - -packageIdentity $packageIdentity ` - -version $package.VersionPreview ` - -lookupTable $metadataLookup['preview'] - } - - if ($package.VersionGA) { - $packageInfoLatest = getPackageInfoFromLookup ` - -packageIdentity $packageIdentity ` - -version $package.VersionGA ` - -lookupTable $metadataLookup['latest'] - } - - if (!$packageInfoPreview -and !$packageInfoLatest) { - # Nothing to move or delete - continue - } - - if ($packageInfoPreview -and $packageInfoLatest) { - # Delete metadata JSON and package-level overview markdown files for - # the preview version instead of moving both. This mitigates situations - # where the "latest" verison doesn't have a package-level overview - # markdown file and the "preview" version does. - deletePackageInfo $packageInfoPreview - moveToLegacy $packageInfoLatest - } - else { - moveToLegacy ($packageInfoPreview ?? $packageInfoLatest) - } -} From 225521d1102f65a307c14533c0212a1b77480c6b Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 6 Sep 2023 15:17:52 +0000 Subject: [PATCH 8/8] Changes into docindex.yml --- eng/pipelines/docindex.yml | 130 ++++++++++++++++++------------------- 1 file changed, 63 insertions(+), 67 deletions(-) diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index 222e34a33c22..b59062a9124a 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -26,8 +26,6 @@ jobs: Repositories: - Name: $(DocRepoOwner)/$(DocRepoName) WorkingDirectory: $(DocRepoLocation) - # TODO: Remove this - Commitish: djurek/deprecated-packages-testing # Pull and build the docker image. - template: /eng/common/pipelines/templates/steps/docker-pull-image.yml parameters: @@ -43,7 +41,6 @@ jobs: displayName: Move deprecated packages to legacy moniker condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true'))) - # Call update docs ci script to onboard packages - task: Powershell@2 inputs: pwsh: true @@ -89,8 +86,7 @@ jobs: - template: /eng/common/pipelines/templates/steps/git-push-changes.yml parameters: - # BaseRepoBranch: $(DefaultBranch) - BaseRepoBranch: djurek/deprecated-packages-testing + BaseRepoBranch: $(DefaultBranch) BaseRepoOwner: $(DocRepoOwner) CommitMsg: "Update docs CI configuration" TargetRepoName: $(DocRepoName) @@ -98,70 +94,70 @@ jobs: WorkingDirectory: $(DocRepoLocation) # Prepare daily docs CI - # - template: /eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml - # parameters: - # DailyBranchVariableName: DailyDocsBranchName - # - pwsh: | - # $ErrorActionPreference = "Continue" - # git checkout "origin/$(DailyDocsBranchName)" 2>&1 | Out-Null - # $LASTEXITCODE = 0 # This ignores any error from git checkout - # git status - # displayName: Checkout daily branch if it exists - # workingDirectory: $(DocRepoLocation) - # - task: Powershell@2 - # inputs: - # pwsh: true - # filePath: eng/common/scripts/Update-DocsMsPackages.ps1 - # arguments: -DocRepoLocation $(DocRepoLocation) - # displayName: Update Docs Onboarding for Daily docs + - template: /eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml + parameters: + DailyBranchVariableName: DailyDocsBranchName + - pwsh: | + $ErrorActionPreference = "Continue" + git checkout "origin/$(DailyDocsBranchName)" 2>&1 | Out-Null + $LASTEXITCODE = 0 # This ignores any error from git checkout + git status + displayName: Checkout daily branch if it exists + workingDirectory: $(DocRepoLocation) + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsPackages.ps1 + arguments: -DocRepoLocation $(DocRepoLocation) + displayName: Update Docs Onboarding for Daily docs - # - task: Powershell@2 - # inputs: - # pwsh: true - # filePath: eng/common/scripts/Service-Level-Readme-Automation.ps1 - # arguments: >- - # -DocRepoLocation $(DocRepoLocation) - # -TenantId '$(opensource-aad-tenant-id)' - # -ClientId '$(opensource-aad-app-id)' - # -ClientSecret '$(opensource-aad-secret)' - # displayName: Generate Service Level Readme for Daily docs + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Service-Level-Readme-Automation.ps1 + arguments: >- + -DocRepoLocation $(DocRepoLocation) + -TenantId '$(opensource-aad-tenant-id)' + -ClientId '$(opensource-aad-app-id)' + -ClientSecret '$(opensource-aad-secret)' + displayName: Generate Service Level Readme for Daily docs - # - task: Powershell@2 - # inputs: - # pwsh: true - # filePath: eng/common/scripts/Update-DocsMsToc.ps1 - # arguments: >- - # -DocRepoLocation $(DocRepoLocation) - # -OutputLocation $(DocRepoLocation)/docs-ref-mapping/reference-unified.yml - # displayName: Generate ToC for Daily docs + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsToc.ps1 + arguments: >- + -DocRepoLocation $(DocRepoLocation) + -OutputLocation $(DocRepoLocation)/docs-ref-mapping/reference-unified.yml + displayName: Generate ToC for Daily docs - # - task: Powershell@2 - # inputs: - # pwsh: true - # filePath: eng/common/scripts/Verify-RequiredDocsJsonMembers.ps1 - # arguments: >- - # -DocRepoLocation $(DocRepoLocation) - # displayName: Verify Required Docs Json Members + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Verify-RequiredDocsJsonMembers.ps1 + arguments: >- + -DocRepoLocation $(DocRepoLocation) + displayName: Verify Required Docs Json Members - # - template: /eng/common/pipelines/templates/steps/git-push-changes.yml - # parameters: - # BaseRepoBranch: $(DailyDocsBranchName) - # BaseRepoOwner: $(DocRepoOwner) - # CommitMsg: "Update targeting packages based on release metadata. (Daily docs)" - # TargetRepoName: $(DocRepoName) - # TargetRepoOwner: $(DocRepoOwner) - # WorkingDirectory: $(DocRepoLocation) - # ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts - # PushArgs: -f + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoBranch: $(DailyDocsBranchName) + BaseRepoOwner: $(DocRepoOwner) + CommitMsg: "Update targeting packages based on release metadata. (Daily docs)" + TargetRepoName: $(DocRepoName) + TargetRepoOwner: $(DocRepoOwner) + WorkingDirectory: $(DocRepoLocation) + ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts + PushArgs: -f - # - task: PowerShell@2 - # displayName: Queue Docs CI build - # inputs: - # pwsh: true - # filePath: eng/common/scripts/Queue-Pipeline.ps1 - # arguments: > - # -Organization "apidrop" - # -Project "Content%20CI" - # -DefinitionId 3452 - # -AuthToken "$(azuresdk-apidrop-devops-queue-build-pat)" - # -BuildParametersJson '{"params":"{ \"target_repo\": { \"url\": \"https://github.com/MicrosoftDocs/azure-docs-sdk-node\", \"branch\": \"$(DailyDocsBranchName)\", \"folder\": \"./\" }, \"source_repos\": [] }"}' + - task: PowerShell@2 + displayName: Queue Docs CI build + inputs: + pwsh: true + filePath: eng/common/scripts/Queue-Pipeline.ps1 + arguments: > + -Organization "apidrop" + -Project "Content%20CI" + -DefinitionId 3452 + -AuthToken "$(azuresdk-apidrop-devops-queue-build-pat)" + -BuildParametersJson '{"params":"{ \"target_repo\": { \"url\": \"https://github.com/MicrosoftDocs/azure-docs-sdk-node\", \"branch\": \"$(DailyDocsBranchName)\", \"folder\": \"./\" }, \"source_repos\": [] }"}'