From 99de99017f93393e7c60087f2bb6ae894582686a Mon Sep 17 00:00:00 2001 From: tadelesh Date: Fri, 20 Aug 2021 20:34:58 +0800 Subject: [PATCH 1/6] add manual tag and release pipeline for track2 --- .../templates/jobs/archetype-go-release.yml | 33 +++++++ .../templates/jobs/archetype-sdk-client.yml | 17 +--- eng/scripts/Language-Settings.ps1 | 98 +++++++++++++++++++ 3 files changed, 136 insertions(+), 12 deletions(-) create mode 100644 eng/pipelines/templates/jobs/archetype-go-release.yml create mode 100644 eng/scripts/Language-Settings.ps1 diff --git a/eng/pipelines/templates/jobs/archetype-go-release.yml b/eng/pipelines/templates/jobs/archetype-go-release.yml new file mode 100644 index 000000000000..b83c1665a223 --- /dev/null +++ b/eng/pipelines/templates/jobs/archetype-go-release.yml @@ -0,0 +1,33 @@ +parameters: + DependsOn: Build + ServiceDirectory: '' + +stages: + - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}: + - stage: Release + displayName: 'Release: ${{parameters.ServiceDirectory}}' + dependsOn: ${{parameters.DependsOn}} + condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-go-pr')) + jobs: + - deployment: TagRepository + displayName: "Create release tag" + condition: ne(variables['Skip.TagRepository'], 'true') + environment: github + + pool: + name: azsdk-pool-mms-ubuntu-2004-general + vmImage: MMSUbuntu20.04 + + strategy: + runOnce: + deploy: + steps: + - checkout: self + - template: /eng/common/pipelines/templates/steps/retain-run.yml + - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml + parameters: + ArtifactLocation: $(Build.SourcesDirectory)/sdk/${{parameters.ServiceDirectory}} + ReleaseSha: $(Build.SourceVersion) + RepoId: Azure/azure-sdk-for-go + WorkingDirectory: $(System.DefaultWorkingDirectory) + \ No newline at end of file diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 6b385ef5d9e2..7a5d0eb8fc3b 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -87,16 +87,9 @@ stages: Scope: $(SCOPE) LintVersion: $(GoLintCLIVersion) -# Below stage won't work until the release stage is added/necessary. -# "Releasing" is just the package in the repository on github, but there may be some other metadata related -# tasks that become necessary later on. - # The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch. - # - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}: - # - template: archetype-go-release.yml - # parameters: - # DependsOn: Build - # ServiceDirectory: ${{parameters.ServiceDirectory}} - # Artifacts: ${{parameters.Artifacts}} - # ArtifactName: packages - # DocArtifact: documentation + - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}: + - template: archetype-go-release.yml + parameters: + DependsOn: Build + ServiceDirectory: ${{parameters.ServiceDirectory}} \ No newline at end of file diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 new file mode 100644 index 000000000000..bf275e1e764d --- /dev/null +++ b/eng/scripts/Language-Settings.ps1 @@ -0,0 +1,98 @@ +$Language = "go" +$packagePattern = "go.mod" +$RELEASE_TITLE_REGEX = "(?^\#+\s+(?v$([AzureEngSemanticVersion]::SEMVER_REGEX))(\s+(?\(.+\))))" +$AUTOREST_VERSION_REGEX = "^module-version:\s+(?$([AzureEngSemanticVersion]::SEMVER_REGEX))" + +function Get-go-PackageInfoFromPackageFile ($pkg, $workingDirectory) +{ + $workFolder = $($pkg.Directory) + $releaseNotes = "" + $namespaceName = $workFolder.Name + $rpName = $workFolder.Parent.Name + + $pkgId = "sdk/$rpName/$namespaceName" + $pkgVersion = Get-Version($workFolder) + + $changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0] + if ($changeLogLoc) + { + $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString v$pkgVersion + } + + $resultObj = New-Object PSObject -Property @{ + PackageId = $pkgId + PackageVersion = $pkgVersion + ReleaseTag = "$($pkgId)/v$($pkgVersion)" + Deployable = $true + ReleaseNotes = $releaseNotes + } + + return $resultObj +} + +# get version from go config auterest.md +function Get-Version ($pkgPath) +{ + $content = Get-Content(Join-Path $pkgPath "autorest.md") + $content = $content.Split("`n") + + try + { + # walk the document, finding where the version specifiers are + foreach ($line in $content) + { + if ($line -match $AUTOREST_VERSION_REGEX) + { + return $matches["version"] + } + } + } + catch + { + Write-Error "Error parsing version." + Write-Error $_ + } + Write-Host "Cannot find release version." + exit(1) +} + +# rewrite for go as all 0.x.x version should be considered prerelease +function CreateReleases($pkgList, $releaseApiUrl, $releaseSha) +{ + foreach ($pkgInfo in $pkgList) + { + Write-Host "Creating release $($pkgInfo.Tag)" + + $releaseNotes = "" + if ($pkgInfo.ReleaseNotes -ne $null) + { + $releaseNotes = $pkgInfo.ReleaseNotes + } + + $isPrerelease = $False + + $parsedSemver = [AzureEngSemanticVersion]::ParseVersionString($pkgInfo.PackageVersion) + + if ($parsedSemver -and ($parsedSemver.IsPrerelease -or ($parsedSemver.Major -eq 0))) + { + $isPrerelease = $true + } + + $url = $releaseApiUrl + $body = ConvertTo-Json @{ + tag_name = $pkgInfo.Tag + target_commitish = $releaseSha + name = $pkgInfo.Tag + draft = $False + prerelease = $isPrerelease + body = $releaseNotes + } + + $headers = @{ + "Content-Type" = "application/json" + "Authorization" = "token $env:GH_TOKEN" + } + + Invoke-RestMethod -Uri $url -Body $body -Headers $headers -Method "Post" -MaximumRetryCount 3 -RetryIntervalSec 10 + } +} \ No newline at end of file From f947a75086303fc944b6aca300ead00622dda703 Mon Sep 17 00:00:00 2001 From: tadelesh Date: Tue, 24 Aug 2021 11:36:04 +0800 Subject: [PATCH 2/6] fix: get version support data plane --- eng/scripts/Language-Settings.ps1 | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index bf275e1e764d..d870763a7def 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -2,7 +2,9 @@ $Language = "go" $packagePattern = "go.mod" $RELEASE_TITLE_REGEX = "(?^\#+\s+(?v$([AzureEngSemanticVersion]::SEMVER_REGEX))(\s+(?\(.+\))))" $AUTOREST_VERSION_REGEX = "^module-version:\s+(?$([AzureEngSemanticVersion]::SEMVER_REGEX))" - +$GO_VERSION_REGEX = "Version\s=\s`"v(?$([AzureEngSemanticVersion]::SEMVER_REGEX))`"" +$MGMT_PLANE_VERSION_FILE = "autorest.md" +$DATA_PLANE_VERSION_FILE = "version.go" function Get-go-PackageInfoFromPackageFile ($pkg, $workingDirectory) { $workFolder = $($pkg.Directory) @@ -33,7 +35,18 @@ function Get-go-PackageInfoFromPackageFile ($pkg, $workingDirectory) # get version from go config auterest.md function Get-Version ($pkgPath) { - $content = Get-Content(Join-Path $pkgPath "autorest.md") + if (Test-Path (Join-Path $pkgPath $MGMT_PLANE_VERSION_FILE)) + { + $versionPath = Join-Path $pkgPath $MGMT_PLANE_VERSION_FILE + $versionRegex = $AUTOREST_VERSION_REGEX + } + elseif (Test-Path (Join-Path $pkgPath $DATA_PLANE_VERSION_FILE)) + { + $versionPath = Join-Path $pkgPath $DATA_PLANE_VERSION_FILE + $versionRegex = $GO_VERSION_REGEX + } + + $content = Get-Content($versionPath) $content = $content.Split("`n") try @@ -41,7 +54,7 @@ function Get-Version ($pkgPath) # walk the document, finding where the version specifiers are foreach ($line in $content) { - if ($line -match $AUTOREST_VERSION_REGEX) + if ($line -match $versionRegex) { return $matches["version"] } @@ -52,6 +65,7 @@ function Get-Version ($pkgPath) Write-Error "Error parsing version." Write-Error $_ } + Write-Host "Cannot find release version." exit(1) } From 5b15e1ac062017bea5afbd655d81b58f4f361c97 Mon Sep 17 00:00:00 2001 From: tadelesh Date: Thu, 26 Aug 2021 11:02:34 +0800 Subject: [PATCH 3/6] chore: refine powershell syntax --- eng/scripts/Language-Settings.ps1 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index d870763a7def..f1bb548ce2a7 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -7,13 +7,13 @@ $MGMT_PLANE_VERSION_FILE = "autorest.md" $DATA_PLANE_VERSION_FILE = "version.go" function Get-go-PackageInfoFromPackageFile ($pkg, $workingDirectory) { - $workFolder = $($pkg.Directory) + $workFolder = $pkg.Directory $releaseNotes = "" $namespaceName = $workFolder.Name $rpName = $workFolder.Parent.Name $pkgId = "sdk/$rpName/$namespaceName" - $pkgVersion = Get-Version($workFolder) + $pkgVersion = Get-Version $workFolder $changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0] if ($changeLogLoc) @@ -24,7 +24,7 @@ function Get-go-PackageInfoFromPackageFile ($pkg, $workingDirectory) $resultObj = New-Object PSObject -Property @{ PackageId = $pkgId PackageVersion = $pkgVersion - ReleaseTag = "$($pkgId)/v$($pkgVersion)" + ReleaseTag = "$pkgId/v$pkgVersion" Deployable = $true ReleaseNotes = $releaseNotes } @@ -46,7 +46,7 @@ function Get-Version ($pkgPath) $versionRegex = $GO_VERSION_REGEX } - $content = Get-Content($versionPath) + $content = Get-Content $versionPath $content = $content.Split("`n") try @@ -67,7 +67,7 @@ function Get-Version ($pkgPath) } Write-Host "Cannot find release version." - exit(1) + exit 1 } # rewrite for go as all 0.x.x version should be considered prerelease @@ -78,7 +78,7 @@ function CreateReleases($pkgList, $releaseApiUrl, $releaseSha) Write-Host "Creating release $($pkgInfo.Tag)" $releaseNotes = "" - if ($pkgInfo.ReleaseNotes -ne $null) + if ($null -ne $pkgInfo.ReleaseNotes) { $releaseNotes = $pkgInfo.ReleaseNotes } From 5d556723a654456063ec8234ed43a4229e513385 Mon Sep 17 00:00:00 2001 From: tadelesh Date: Fri, 27 Aug 2021 11:42:49 +0800 Subject: [PATCH 4/6] feat: use regrex to fetch version from all possible version file under module folders --- eng/scripts/Language-Settings.ps1 | 60 ++++++++++++++++--------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index f1bb548ce2a7..dec5f81e7227 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -1,10 +1,12 @@ $Language = "go" $packagePattern = "go.mod" +# rewrite from ChangeLog-Operations.ps1 used in Get-ChangeLogEntriesFromContent for go uses vx.x.x as version number $RELEASE_TITLE_REGEX = "(?^\#+\s+(?v$([AzureEngSemanticVersion]::SEMVER_REGEX))(\s+(?\(.+\))))" -$AUTOREST_VERSION_REGEX = "^module-version:\s+(?$([AzureEngSemanticVersion]::SEMVER_REGEX))" -$GO_VERSION_REGEX = "Version\s=\s`"v(?$([AzureEngSemanticVersion]::SEMVER_REGEX))`"" -$MGMT_PLANE_VERSION_FILE = "autorest.md" -$DATA_PLANE_VERSION_FILE = "version.go" +# constants for go version fetch +$GO_VERSION_REGEX = ".+\s*=\s*`".*v?(?$([AzureEngSemanticVersion]::SEMVER_REGEX))`"" +$VERSION_FILE_SUFFIXS = "*constants.go", "*version.go" + +# rewrite from artifact-metadata-parsing.ps1 used in RetrievePackages for fetch go single module info function Get-go-PackageInfoFromPackageFile ($pkg, $workingDirectory) { $workFolder = $pkg.Directory @@ -32,45 +34,47 @@ function Get-go-PackageInfoFromPackageFile ($pkg, $workingDirectory) return $resultObj } -# get version from go config auterest.md +# get version from specific files (*constants.go, *version.go) function Get-Version ($pkgPath) { - if (Test-Path (Join-Path $pkgPath $MGMT_PLANE_VERSION_FILE)) - { - $versionPath = Join-Path $pkgPath $MGMT_PLANE_VERSION_FILE - $versionRegex = $AUTOREST_VERSION_REGEX - } - elseif (Test-Path (Join-Path $pkgPath $DATA_PLANE_VERSION_FILE)) + # find any file with surfix + $versionFiles = [Collections.Generic.List[String]]@() + foreach ($versionFileSuffix in $VERSION_FILE_SUFFIXS) { - $versionPath = Join-Path $pkgPath $DATA_PLANE_VERSION_FILE - $versionRegex = $GO_VERSION_REGEX + Get-ChildItem -Recurse -Path $pkgPath -Filter $versionFileSuffix | ForEach-Object { + Write-Host "Adding $_ to list of version files" + $versionFiles.Add($_) + } } - - $content = Get-Content $versionPath - $content = $content.Split("`n") - - try + + # for each version file, use regex to search go version num + foreach ($versionFile in $versionFiles) { - # walk the document, finding where the version specifiers are - foreach ($line in $content) + $content = Get-Content $versionFile + $content = $content.Split("`n") + try { - if ($line -match $versionRegex) + # walk the document, finding where the version number are + foreach ($line in $content) { - return $matches["version"] + if ($line -match $GO_VERSION_REGEX) + { + return $matches["version"] + } } } - } - catch - { - Write-Error "Error parsing version." - Write-Error $_ + catch + { + Write-Error "Error parsing version." + Write-Error $_ + } } Write-Host "Cannot find release version." exit 1 } -# rewrite for go as all 0.x.x version should be considered prerelease +# rewrite from artifact-metadata-parsing.ps1 as all 0.x.x version should be considered prerelease in go sdk function CreateReleases($pkgList, $releaseApiUrl, $releaseSha) { foreach ($pkgInfo in $pkgList) From f2f5e75e7a9627e8ff81c7067ee9798f5d023740 Mon Sep 17 00:00:00 2001 From: tadelesh Date: Thu, 2 Sep 2021 14:03:36 +0800 Subject: [PATCH 5/6] fix: polish according to pr review --- eng/scripts/Language-Settings.ps1 | 43 ++++++++++++++++++------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index dec5f81e7227..f2c23c9c757b 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -2,19 +2,28 @@ $Language = "go" $packagePattern = "go.mod" # rewrite from ChangeLog-Operations.ps1 used in Get-ChangeLogEntriesFromContent for go uses vx.x.x as version number $RELEASE_TITLE_REGEX = "(?^\#+\s+(?v$([AzureEngSemanticVersion]::SEMVER_REGEX))(\s+(?\(.+\))))" -# constants for go version fetch -$GO_VERSION_REGEX = ".+\s*=\s*`".*v?(?$([AzureEngSemanticVersion]::SEMVER_REGEX))`"" -$VERSION_FILE_SUFFIXS = "*constants.go", "*version.go" # rewrite from artifact-metadata-parsing.ps1 used in RetrievePackages for fetch go single module info function Get-go-PackageInfoFromPackageFile ($pkg, $workingDirectory) { $workFolder = $pkg.Directory $releaseNotes = "" - $namespaceName = $workFolder.Name - $rpName = $workFolder.Parent.Name - $pkgId = "sdk/$rpName/$namespaceName" + if ($workFolder -match "sdk[\/|\\]((?(?.*?)[\/|\\])?(?arm)?(?.*))") + { + if ($matches["arm"]) + { + $packageName = "arm" + $matches["pkgname"] + $rpName = $matches["service"] + $pkgId = "sdk/$rpName/$packageName" + } + else + { + $packageName = $matches["pkgname"] + $pkgId = "sdk/$packageName" + } + } + $pkgVersion = Get-Version $workFolder $changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0] @@ -37,30 +46,28 @@ function Get-go-PackageInfoFromPackageFile ($pkg, $workingDirectory) # get version from specific files (*constants.go, *version.go) function Get-Version ($pkgPath) { - # find any file with surfix - $versionFiles = [Collections.Generic.List[String]]@() - foreach ($versionFileSuffix in $VERSION_FILE_SUFFIXS) + # find any file with suffix + $versionFiles = @() + $version_file_suffixs = "*constants.go", "*version.go" + foreach ($versionFileSuffix in $version_file_suffixs) { Get-ChildItem -Recurse -Path $pkgPath -Filter $versionFileSuffix | ForEach-Object { Write-Host "Adding $_ to list of version files" - $versionFiles.Add($_) + $versionFiles += $_ } } # for each version file, use regex to search go version num + $go_version_regex = ".+\s*=\s*`".*v?(?$([AzureEngSemanticVersion]::SEMVER_REGEX))`"" foreach ($versionFile in $versionFiles) { - $content = Get-Content $versionFile - $content = $content.Split("`n") try { - # walk the document, finding where the version number are - foreach ($line in $content) + $content = Get-Content $versionFile -Raw + # finding where the version number are + if ($content -match $go_version_regex) { - if ($line -match $GO_VERSION_REGEX) - { - return $matches["version"] - } + return $matches["version"] } } catch From 67373560d29a09a1f8f3d5680e9be1809c45d905 Mon Sep 17 00:00:00 2001 From: tadelesh Date: Fri, 3 Sep 2021 11:08:08 +0800 Subject: [PATCH 6/6] fix: remove rewrite CreateReleases function as eng sys has already done it --- eng/scripts/Language-Settings.ps1 | 41 ------------------------------- 1 file changed, 41 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index f2c23c9c757b..7f020182c021 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -79,45 +79,4 @@ function Get-Version ($pkgPath) Write-Host "Cannot find release version." exit 1 -} - -# rewrite from artifact-metadata-parsing.ps1 as all 0.x.x version should be considered prerelease in go sdk -function CreateReleases($pkgList, $releaseApiUrl, $releaseSha) -{ - foreach ($pkgInfo in $pkgList) - { - Write-Host "Creating release $($pkgInfo.Tag)" - - $releaseNotes = "" - if ($null -ne $pkgInfo.ReleaseNotes) - { - $releaseNotes = $pkgInfo.ReleaseNotes - } - - $isPrerelease = $False - - $parsedSemver = [AzureEngSemanticVersion]::ParseVersionString($pkgInfo.PackageVersion) - - if ($parsedSemver -and ($parsedSemver.IsPrerelease -or ($parsedSemver.Major -eq 0))) - { - $isPrerelease = $true - } - - $url = $releaseApiUrl - $body = ConvertTo-Json @{ - tag_name = $pkgInfo.Tag - target_commitish = $releaseSha - name = $pkgInfo.Tag - draft = $False - prerelease = $isPrerelease - body = $releaseNotes - } - - $headers = @{ - "Content-Type" = "application/json" - "Authorization" = "token $env:GH_TOKEN" - } - - Invoke-RestMethod -Uri $url -Body $body -Headers $headers -Method "Post" -MaximumRetryCount 3 -RetryIntervalSec 10 - } } \ No newline at end of file