Merge branch 'master' of https://github.com/Azure/azure-sdk-for-python into analyze_redesign

* 'master' of https://github.com/Azure/azure-sdk-for-python: (35 commits)
  Sync eng/common directory with azure-sdk-tools for PR 1353 (Azure#16465)
  Normalize the package name for Doc.Ms readme (Azure#16401)
  fix changelog and version (Azure#16445)
  we should always run integration to publish from our artifacts. if the build pipeline crashed too early in the pipeline, this will fail regardless, due to inability to pull the artifacts down (Azure#15058)
  Fix DateTime bug (Azure#16456)
  Resolve Regression Failures (Azure#16455)
  [text analytics] Expose 'string_index_type' parameter in all service client methods where applicable (Azure#16412)
  adding devtools to the appconfig dev_reqs to solve python core issue (Azure#16381)
  Copy job matrix functionality (Azure#16450)
  Add APIView KV variable group to prepare pipelines bot (Azure#16451)
  [Datalake] Added support for PurePosixPath (Azure#16400)
  Regenerate baseline because last one break. (Azure#16415)
  adding step to test for crlf line endings (Azure#16398)
  [Datalake] Removed list_paths manual paging and deserialization (Azure#16309)
  Sync eng/common directory with azure-sdk-tools for PR 1351 (Azure#16448)
  Update auto_codegen.py (Azure#16443)
  First release purview (Azure#16440)
  1ES pools update for release pipeline. (Azure#16419)
  Add Cloud environment for Teams user (Azure#16359)
  Sync eng/common directory with azure-sdk-tools for PR 1345 (Azure#16404)
  ...

Author: iscai-msft

.gitignore

@@ -33,6 +33,7 @@ build/
 
 # Test results
 TestResults/
+ENV_DIR/
 
 # tox generated artifacts
 test-junit-*.xml

build_package.py

@@ -9,6 +9,7 @@
 import argparse
 import os
 import glob
+import sys
 from subprocess import check_call
 
 
@@ -19,8 +20,8 @@ def create_package(name, dest_folder=DEFAULT_DEST_FOLDER):
     absdirs = [os.path.dirname(package) for package in (glob.glob('{}/setup.py'.format(name)) + glob.glob('sdk/*/{}/setup.py'.format(name)))]
 
     absdirpath = os.path.abspath(absdirs[0])
-    check_call(['python', 'setup.py', 'bdist_wheel', '-d', dest_folder], cwd=absdirpath)
-    check_call(['python', 'setup.py', "sdist", "--format", "zip", '-d', dest_folder], cwd=absdirpath)
+    check_call([sys.executable, 'setup.py', 'bdist_wheel', '-d', dest_folder], cwd=absdirpath)
+    check_call([sys.executable, 'setup.py', "sdist", "--format", "zip", '-d', dest_folder], cwd=absdirpath)
 
 
 if __name__ == '__main__':

ci_template.yml

@@ -0,0 +1,33 @@
+# DO NOT EDIT THIS FILE
+# This file is generated automatically and any changes will be lost.
+
+trigger:
+  branches:
+    include:
+    - master
+    - hotfix/*
+    - release/*
+    - restapi*
+  paths:
+    include:
+    - sdk/MyService/
+
+pr:
+  branches:
+    include:
+    - master
+    - feature/*
+    - hotfix/*
+    - release/*
+    - restapi*
+  paths:
+    include:
+    - sdk/MyService/
+
+extends:
+  template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml
+  parameters:
+    ServiceDirectory: MyService
+    Artifacts:
+    - name: azure_mgmt_MyService
+      safeName: azuremgmtMyService

eng/common/TestResources/README.md

@@ -30,7 +30,7 @@ Connect-AzAccount -Subscription 'YOUR SUBSCRIPTION ID'
 eng\common\TestResources\New-TestResources.ps1 -ServiceDirectory 'search'
 ```
 
-The `OutFile` switch would be set if you are running this for a .NET project on Windows. This will save test environment settings
+The `OutFile` switch will be set by default if you are running this for a .NET project on Windows. This will save test environment settings
 into a test-resources.json.env file next to test-resources.json. The file is protected via DPAPI.
 The environment file would be scoped to the current repository directory and avoids the need to
 set environment variables or restart your IDE to recognize them.
@@ -89,6 +89,12 @@ Remove-AzADServicePrincipal -ApplicationId $sp.ApplicationId -Force
 
 If you persisted environment variables, you should also remove those as well.
 
+Some test-resources.json templates utilize the `AdditionalParameters` parameter to control additional resource configuration options. For example:
+
+```powershell
+New-TestResources.ps1 keyvault -AdditionalParameters @{enableHsm = $true}
+```
+
 ## In CI
 
 Test pipelines should include deploy-test-resources.yml and

eng/common/pipelines/templates/steps/daily-dev-build-variable.yml

@@ -1,7 +1,19 @@
 # This script fragment is used across our repos to set a variable "SetDevVersion" which
 # is used when this pipeline is going to be generating and publishing daily dev builds.
-
+parameters:
+  ServiceDirectory: ''
 steps:
+- ${{if ne(parameters.ServiceDirectory, '')}}:
+  - task: Powershell@2
+    inputs:
+      filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1
+      arguments: >
+        -ServiceDirectory ${{parameters.ServiceDirectory}}
+        -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo
+      pwsh: true
+      workingDirectory: $(Pipeline.Workspace)
+    displayName: Dump Package properties
+    condition: succeeded()
 - pwsh: |
     $setDailyDevBuild = "false"
     if (('$(Build.Reason)' -eq 'Schedule') -and ('$(System.TeamProject)' -eq 'internal')) {

eng/common/scripts/Create-APIReview.ps1

@@ -40,6 +40,7 @@ function Submit-APIReview($packagename, $filePath, $uri, $apiKey, $apiLabel)
     try
     {
         $Response = Invoke-WebRequest -Method 'POST' -Uri $uri -Body $multipartContent -Headers $headers
+        Write-Host "API Review: $($Response)"
         $StatusCode = $Response.StatusCode
     }
     catch
@@ -81,23 +82,47 @@ else
 }
 
 $FoundFailure = $False
+$pkgInfoPath = Join-Path -Path $ArtifactPath "PackageInfo"
 foreach ($pkgName in $responses.Keys)
 {    
     $respCode = $responses[$pkgName]
     if ($respCode -ne '200')
     {
-        $FoundFailure = $True
-        if ($respCode -eq '201')
+        $pkgPropPath = Join-Path -Path $pkgInfoPath ($PackageName + ".json")
+        if (-Not (Test-Path $pkgPropPath))
         {
-            Write-Host "API Review is pending for package $pkgName"
+            Write-Host " Package property file path $($pkgPropPath) is invalid."
+            $FoundFailure = $True
         }
         else
         {
-            Write-Host "Failed to create API Review for package $pkgName"
+            $pkgInfo = Get-Content $pkgPropPath | ConvertFrom-Json
+            $version = [AzureEngSemanticVersion]::ParseVersionString($pkgInfo.Version)
+            if ($version.IsPrerelease)
+            {
+                Write-Host "Package version is not GA. Ignoring API view approval status"
+            }
+            elseif ($pkgInfo.SdkType -eq "client" -and $pkgInfo.IsNewSdk)
+            {
+                $FoundFailure = $True
+                if ($respCode -eq '201')
+                {
+                    Write-Error "Automatic API Review approval is pending for package $($PackageName)"
+                }
+                else
+                {
+                    Write-Error "Failed to create API Review for package $($PackageName)"
+                }                
+            }
+            else
+            {
+                Write-Host "API review is not approved for package $($PackageName). Management and track1 package can be released without API review approval."
+            }      
         }
     }
 }
 if ($FoundFailure)
 {
-    Write-Host "Atleast one API review is not yet approved"
+    Write-Error "Automatic API review is not yet approved for package $($PackageName)"
+    exit 1
 }

eng/common/scripts/Package-Properties.ps1

@@ -10,6 +10,8 @@ class PackageProps
     [string]$ReadMePath
     [string]$ChangeLogPath
     [string]$Group
+    [string]$SdkType
+    [boolean]$IsNewSdk
 
     PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory)
     {

eng/common/scripts/Prepare-Release.ps1

@@ -55,7 +55,7 @@ if (!$ReleaseDate)
 }
 else
 {
-    $ParsedReleaseDate = ([datetime]$ReleaseDate, 'MM/dd/yyyy', [Globalization.CultureInfo]::InvariantCulture)
+    $ParsedReleaseDate = [datetime]$ReleaseDate
 }
 
 $releaseDateString = $ParsedReleaseDate.ToString("MM/dd/yyyy")

eng/common/scripts/Save-Package-Properties.ps1

@@ -0,0 +1,34 @@
+[CmdletBinding()]
+Param (
+  [Parameter(Mandatory=$True)]
+  [string] $serviceDirectory,
+  [Parameter(Mandatory=$True)]
+  [string] $outDirectory
+)
+
+. (Join-Path $PSScriptRoot common.ps1)
+$allPackageProperties = Get-AllPkgProperties $serviceDirectory
+if ($allPackageProperties)
+{
+    New-Item -ItemType Directory -Force -Path $outDirectory
+    foreach($pkg in $allPackageProperties)
+    {
+        if ($pkg.IsNewSDK)
+        {
+            Write-Host "Package Name: $($pkg.Name)"
+            Write-Host "Package Version: $($pkg.Version)"
+            Write-Host "Package SDK Type: $($pkg.SdkType)"
+            $outputPath = Join-Path -Path $outDirectory ($pkg.Name + ".json")
+            $outputObject = $pkg | ConvertTo-Json
+            Set-Content -Path $outputPath -Value $outputObject
+        }        
+    }
+
+    Get-ChildItem -Path $outDirectory
+}
+else
+{
+    Write-Error "Package properties are not available for service directory $($serviceDirectory)"
+    exit 1
+}
+

eng/common/scripts/artifact-metadata-parsing.ps1

@@ -123,14 +123,18 @@ function VerifyPackages($artifactLocation, $workingDirectory, $apiUrl, $releaseS
         Write-Host "Maybe a pkg version wasn't updated properly?"
         exit(1)
       }
-
+      $docsReadMeName = $parsedPackage.PackageId
+      if ($parsedPackage.DocsReadMeName) {
+        $docsReadMeName = $parsedPackage.DocsReadMeName
+      }
       $pkgList += New-Object PSObject -Property @{
         PackageId      = $parsedPackage.PackageId
         PackageVersion = $parsedPackage.PackageVersion
         GroupId        = $parsedPackage.GroupId
         Tag            = $parsedPackage.ReleaseTag
         ReleaseNotes   = $parsedPackage.ReleaseNotes
         ReadmeContent  = $parsedPackage.ReadmeContent
+        DocsReadMeName = $docsReadMeName
         IsPrerelease   = [AzureEngSemanticVersion]::ParseVersionString($parsedPackage.PackageVersion).IsPrerelease
       }
     }