Merge branch 'master' of https://github.com/Azure/azure-sdk-for-python into analyze_redesign

* 'master' of https://github.com/Azure/azure-sdk-for-python: (32 commits)
  Adopt new MSAL auth code flow API (Azure#16449)
  [formrecognizer] use ARM template for tests (Azure#16432)
  T2 kusto 2021 02 04 (Azure#16527)
  T2 applicationinsights 2021 02 04 (Azure#16525)
  Sync eng/common directory with azure-sdk-tools for PR 1366 (Azure#16506)
  [Python] python track2 new pipeline fix (Azure#16494)
  Added package properties SDKType and NewSDK (Azure#16476)
  bump six dependencies in some libraries (Azure#16496)
  call on_error if timeout in flush (Azure#16485)
  Sync eng/common directory with azure-sdk-tools for PR 1365 (Azure#16505)
  Fix min dependency tests - update azure core (Azure#16504)
  Sync eng/common directory with azure-sdk-tools for PR 1364 (Azure#16503)
  Ma arch feedback (Azure#16502)
  Adding a new limitation to the README file. (Azure#16475)
  [Blob][Datalake] STG76 Preview (Azure#16349)
  append code coverage over each other (Azure#16202)
  Arch preview feedback (Azure#16441)
  Support CAE in azure-identity (Azure#16323)
  [EventHubs] Support for Custom endpoint adddress and custom certificate  (Azure#16295)
  [Communication] - Phone Number Management - Added support for AAD auth (Azure#16075)
  ...

Author: iscai-msft

.coveragerc

@@ -6,6 +6,11 @@ omit =
 	*/test*
 	env*
 
+[paths]
+source =
+	sdk/
+	**/sdk
+
 [report]
 exclude_lines =
 	pragma: no cover

eng/ci_tools.txt

@@ -3,7 +3,7 @@ cryptography==3.1
 setuptools==44.1.0; python_version == '2.7'
 setuptools==46.4.0; python_version >= '3.5'
 virtualenv==20.0.23
-wheel==0.34.2 
+wheel==0.34.2
 Jinja2==2.11.2
 packaging==20.4
 tox==3.15.0
@@ -18,6 +18,7 @@ coverage==4.5.4
 codecov==2.1.0
 beautifulsoup4==4.9.1
 pkginfo==1.5.0.1
+pip==20.2
 
 # locking packages defined as deps from azure-sdk-tools or azure-devtools
 pytoml==0.1.21

eng/common/docgeneration/Generate-DocIndex.ps1

@@ -65,12 +65,7 @@ function Get-TocMapping {
     $orderServiceMapping = @{}
 
     foreach ($artifact in $artifacts) {
-        $packageInfo = $metadata | ? {$_.Package -eq $artifact}
-        
-        if ($packageInfo -and $packageInfo[0].Hide -eq 'true') {
-            LogDebug "The artifact $artifact set 'Hide' to 'true'."
-            continue
-        }
+        $packageInfo = $metadata | ? { $_.Package -eq $artifact -and $_.Hide -ne "true" }
         $serviceName = ""
         $displayName = ""
         if (!$packageInfo) {

eng/common/pipelines/templates/steps/cosmos-emulator.yml

@@ -9,8 +9,87 @@ steps:
       Write-Host "Target Dir: $targetDir"
       msiexec /a ${{ parameters.EmulatorMsiUrl }} TARGETDIR=$targetDir /qn | wait-process
     displayName: Download and Extract Public Cosmos DB Emulator
+  - powershell: |
+      Write-Host "Deleting Cosmos DB Emulator data"
+      if (Test-Path $Env:LOCALAPPDATA\CosmosDbEmulator) { Remove-Item -Recurse -Force $Env:LOCALAPPDATA\CosmosDbEmulator }
+    displayName: Delete Cosmos DB Emulator data
+  - powershell: |
+      Write-Host "Getting Cosmos DB Emulator Version"
+      $ProductName = "Azure Cosmos DB Emulator"
+      $Emulator = (Join-Path $env:temp (Join-Path $ProductName "Microsoft.Azure.Cosmos.Emulator.exe"))
+      $fileVersion = Get-ChildItem $Emulator
+      Write-Host $Emulator $fileVersion.VersionInfo
+    displayName: Get Cosmos DB Emulator Version
   - powershell: |
       Write-Host "Launching Cosmos DB Emulator"
-      Import-Module "$env:temp\Azure Cosmos DB Emulator\PSModules\Microsoft.Azure.CosmosDB.Emulator"
-      Start-CosmosDbEmulator -NoUI ${{ parameters.StartParameters }}
+      $ProductName = "Azure Cosmos DB Emulator"
+      $Emulator = (Join-Path $env:temp (Join-Path $ProductName "Microsoft.Azure.Cosmos.Emulator.exe"))
+      if (!(Test-Path $Emulator)) {
+        Write-Error "The emulator is not installed where expected at '$Emulator'"
+        return
+      }
+      $process = Start-Process $Emulator -ArgumentList "/getstatus" -PassThru -Wait
+      switch ($process.ExitCode) {
+        1 {
+          Write-Host "The emulator is already starting"
+          return
+        }
+        2 {
+          Write-Host "The emulator is already running"
+          return
+        }
+        3 {
+          Write-Host "The emulator is stopped"
+        }
+        default {
+          Write-Host "Unrecognized exit code $process.ExitCode"
+          return
+        }
+      }
+      $argumentList = ""
+      if (-not [string]::IsNullOrEmpty("${{ parameters.StartParameters }}")) {
+          $argumentList += , "${{ parameters.StartParameters }}"
+      } else {
+        # Use the default params if none provided
+        $argumentList = "/noexplorer /noui /enablepreview /disableratelimiting /enableaadauthentication"
+      }
+      Write-Host "Starting emulator process: $Emulator $argumentList"
+      $process=Start-Process $Emulator -ArgumentList $argumentList -ErrorAction Stop -PassThru
+      Write-Host "Emulator process started: $($process.Name), $($process.FileVersion)"
+      $Timeout = 600
+      $result="NotYetStarted"
+      $complete = if ($Timeout -gt 0) {
+        $start = [DateTimeOffset]::Now
+        $stop = $start.AddSeconds($Timeout)
+        {
+          $result -eq "Running" -or [DateTimeOffset]::Now -ge $stop
+        }
+      }
+      else {
+        {
+          $result -eq "Running"
+        }
+      }
+      do {
+        $process = Start-Process $Emulator -ArgumentList "/getstatus" -PassThru -Wait
+        switch ($process.ExitCode) {
+          1 {
+            Write-Host "The emulator is starting"
+          }
+          2 {
+            Write-Host "The emulator is running"
+            $result="Running"
+            return
+          }
+          3 {
+            Write-Host "The emulator is stopped"
+          }
+          default {
+            Write-Host "Unrecognized exit code $process.ExitCode"
+          }
+        }
+        Start-Sleep -Seconds 5
+      }
+      until ($complete.Invoke())
+      Write-Error "The emulator failed to reach Running status within ${Timeout} seconds"
     displayName: Start Cosmos DB Emulator

eng/common/scripts/Create-APIReview.ps1

@@ -8,7 +8,8 @@ Param (
   [string] $APIKey,
   [Parameter(Mandatory=$True)]
   [string] $APILabel,
-  [string] $PackageName = ""
+  [string] $PackageName,
+  [string] $ConfigFileDir = ""
 )
 
 
@@ -82,17 +83,19 @@ else
 }
 
 $FoundFailure = $False
-$pkgInfoPath = Join-Path -Path $ArtifactPath "PackageInfo"
+if (-not $ConfigFileDir)
+{
+    $ConfigFileDir = Join-Path -Path $ArtifactPath "PackageInfo"
+}
 foreach ($pkgName in $responses.Keys)
 {    
     $respCode = $responses[$pkgName]
     if ($respCode -ne '200')
     {
-        $pkgPropPath = Join-Path -Path $pkgInfoPath ($PackageName + ".json")
+        $pkgPropPath = Join-Path -Path $ConfigFileDir "$PackageName.json"
         if (-Not (Test-Path $pkgPropPath))
         {
             Write-Host " Package property file path $($pkgPropPath) is invalid."
-            $FoundFailure = $True
         }
         else
         {

eng/common/scripts/Save-Package-Properties.ps1

@@ -13,7 +13,7 @@ if ($allPackageProperties)
     New-Item -ItemType Directory -Force -Path $outDirectory
     foreach($pkg in $allPackageProperties)
     {
-        if ($pkg.IsNewSDK)
+        if ($pkg.IsNewSdk)
         {
             Write-Host "Package Name: $($pkg.Name)"
             Write-Host "Package Version: $($pkg.Version)"

eng/pipelines/templates/jobs/archetype-sdk-client.yml

@@ -39,31 +39,31 @@ parameters:
         Pool: $(LinuxPool)
         OSVmImage:
         PythonVersion: '2.7'
-        CoverageArg: ''
+        CoverageArg: '--disablecov'
         RunForPR: true
       Linux_Python35:
         Pool: $(LinuxPool)
         OSVmImage:
         PythonVersion: '3.5'
-        CoverageArg: ''
+        CoverageArg: '--disablecov'
         RunForPR: false
       Linux_Python38:
         Pool: $(LinuxPool)
         OSVmImage:
         PythonVersion: '3.8'
-        CoverageArg: ''
+        CoverageArg: '--disablecov'
         RunForPR: true
       Windows_Python35:
         Pool: $(WindowsPool)
         OSVmImage:
         PythonVersion: '3.5'
-        CoverageArg: ''
+        CoverageArg: '--disablecov'
         RunForPR: true
       MacOS_Python27:
         Pool:
         OSVmImage: 'macOS-10.15'
         PythonVersion: '2.7'
-        CoverageArg: ''
+        CoverageArg: '--disablecov'
         RunForPR: false
       Linux_pypy3:
         Pool: $(LinuxPool)
@@ -100,7 +100,7 @@ jobs:
 
     steps:
     - template: ../steps/build-artifacts.yml
-      parameters: 
+      parameters:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
         BuildTargetingString: ${{ parameters.BuildTargetingString }}
         BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
@@ -133,7 +133,7 @@ jobs:
         CheckLinkGuidance: $true
 
     - template: ../steps/analyze.yml
-      parameters: 
+      parameters:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
         BuildTargetingString: ${{ parameters.BuildTargetingString }}
         TestMarkArgument: ${{ parameters.TestMarkArgument }}
@@ -168,7 +168,7 @@ jobs:
           ${{ if or(eq(matrixEntry.value.RunForPR, 'true'), ne(variables['Build.Reason'], 'PullRequest')) }}:
             ${{ matrixEntry.key }}:
               ${{ insert }}: ${{ matrixEntry.value }}
-        
+
     pool:
       name: $[coalesce(variables['Pool'], '')]
       vmImage: $[coalesce(variables['OSVmImage'], '')]
@@ -195,7 +195,7 @@ jobs:
         {
           $toxenvvar = '$(Run.ToxCustomEnvs)'
         }
-        
+
         echo "##vso[task.setvariable variable=toxenv]$toxenvvar"
       displayName: "Set Tox Environment"
 
@@ -211,10 +211,10 @@ jobs:
         ToxTestEnv: $(toxenv)
         ToxEnvParallel: ${{ parameters.ToxEnvParallel }}
         InjectedPackages: $(InjectedPackages)
-        BeforeTestSteps: 
+        BeforeTestSteps:
           - task: DownloadPipelineArtifact@0
             inputs:
-              artifactName: 'artifacts' 
+              artifactName: 'artifacts'
               targetPath: $(Build.ArtifactStagingDirectory)
 
           - template: ../steps/set-dev-build.yml
@@ -237,6 +237,6 @@ jobs:
 
     steps:
     - template: ../steps/test_regression.yml
-      parameters: 
+      parameters:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
         BuildTargetingString: ${{ parameters.BuildTargetingString }}