Merge branch 'master' of https://github.com/Azure/azure-sdk-for-python into fix_paging_types

* 'master' of https://github.com/Azure/azure-sdk-for-python: (37 commits)
  [translation] poller design (Azure#19041)
  [AutoRelease] t2-resourcemover-2021-05-21-54304(wave4) (Azure#18849)
  [AutoRelease] t2-logz-2021-06-02-49354 (Azure#19035)
  prepare pipeline (Azure#19062)
  [AutoRelease] t2-recoveryservicesbackup-2021-05-26-33193 (Azure#18953)
  Get rid of DataFeedOptions (Azure#19054)
  [AutoRelease] t2-batchai-2021-06-02-38609 (Azure#19036)
  [appconfig] new gen code (Azure#18859)
  [Tables] mypy (Azure#19001)
  fix misleading url (Azure#19044)
  Create py.typed (Azure#19052)
  add override of opentelemetry-api to 1.3.0 (Azure#19048)
  Prepare for June Release (Azure#19043)
  [Key Vault] Add API version 7.2 for keys (Azure#18586)
  some misc fixes. (Azure#19024)
  [AppConfig] pre release (Azure#19027)
  [ACR] prerelease (Azure#19028)
  Add sdk/core to triggers and pr (Azure#19023)
  Event grid 4.3.0 regen code  (Azure#19025)
  Handle 6 decimal places cloud event (Azure#19019)
  ...

Author: iscai-msft

.gitignore

@@ -92,10 +92,17 @@ venv/
 code_reports
 
 # Azure Storage test credentials
+sdk/storage/azure-storage-blob/tests/_shared/settings_real.py
+sdk/storage/azure-storage-queue/tests/_shared/settings_real.py
+sdk/storage/azure-storage-file-share/tests/_shared/settings_real.py
+sdk/storage/azure-storage-file-datalake/tests/settings_real.py
+
+# The locations below are deprecated - keep to prevent any accidental secrets leakage ==========
 sdk/storage/azure-storage-blob/tests/settings_real.py
 sdk/storage/azure-storage-queue/tests/settings_real.py
 sdk/storage/azure-storage-file-share/tests/settings_real.py
-sdk/storage/azure-storage-file-datalake/tests/settings_real.py
+# ==============================================================================================
+
 *.code-workspace
 sdk/cosmos/azure-cosmos/test/test_config.py
 

doc/eng_sys_checks.md

@@ -15,13 +15,23 @@ Example PR build:
  - `Analyze` tox envs run during the `Analyze job.
  - `Test <platform>_<pyversion>` runs PR/Nightly tox envs, depending on context.
 
+
+## Targeting a specific package at build time
+
+In both `public` and `internal` projects, all builds allow a filter to be introduced at build time to narrow the set of packages build/tested.
+
+1. Click `Run New` on your target build.
+2. Before clicking `run` against `master` or your target commit, click `Variables` and add a variable. Add variable `BuildTargetingString` with value of a valid glob string.
+   1. For example, setting filter string `azure-mgmt-*` will filter a build to only management packages. A value of `azure-keyvault-secrets` will result in only building THAT specific package.
+3. Once it's set, run the build!
+
 ## Skipping a tox test environment at queue time
 
 All build definitions allow choice at queue time as to which `tox` environments actually run during the test phase.
 
 1. Find your target service `internal` build.
-2. Click `Run New`
-3. Before clicking `run` against `master` or your target commit, click `Variables` and add a variable. The value should be a comma separated list of tox environments that you want to run in the test phase.
+2. Click `Run New`.
+3. Before clicking `run` against `master` or your target commit, click `Variables` and add a variable of name `Run.ToxCustomEnvs`. The value should be a comma separated list of tox environments that you want to run in the test phase.
 4. Once it's set, run the build!
 
 This is an example setting of that narrows the default set from `whl, sdist, depends, latestdependency, minimumdependency`.

eng/.docsettings.yml

@@ -84,6 +84,7 @@ known_content_issues:
   - ['sdk/monitor/azure-monitor-opentelemetry-exporter/README.md', '#4554']
   - ['sdk/digitaltwins/azure-digitaltwins-core/swagger/README.md', '#4554']
   - ['sdk/textanalytics/azure-ai-textanalytics/swagger/README.md', '#4554']
+  - ['sdk/translation/azure-ai-translation-document/swagger/README.md', '#4554']
   - ['sdk/purview/azure-purview-catalog/swagger/README.md',  '#4554']
   - ['sdk/purview/azure-purview-scanning/swagger/README.md',  '#4554']
   - ['sdk/agrifood/azure-agrifood-farming/swagger/README.md',  '#4554']

eng/common/TestResources/AzurePowerShellV4/Utility.ps1

@@ -33,10 +33,12 @@ parameters:
 
 
 steps:
-  - template: /eng/common/TestResources/setup-az-modules.yml
+  - template: /eng/common/pipelines/templates/steps/cache-ps-modules.yml
+
+  - template: /eng/common/TestResources/setup-environments.yml
 
   - pwsh: |
-      eng/common/TestResources/Import-AzModules.ps1
+      eng/common/scripts/Import-AzModules.ps1
 
       $subscriptionConfiguration = @'
         ${{ parameters.SubscriptionConfiguration }}

eng/common/TestResources/Import-AzModules.ps1

@@ -21,7 +21,7 @@ parameters:
 
 steps:
   - pwsh: |
-      eng/common/TestResources/Import-AzModules.ps1
+      eng/common/scripts/Import-AzModules.ps1
 
       $subscriptionConfiguration = @"
         ${{ parameters.SubscriptionConfiguration }}

eng/common/TestResources/deploy-test-resources.yml

@@ -21,12 +21,12 @@ steps:
     condition: contains(variables['OSVmImage'], 'mac')
 
   - task: Powershell@2
+    displayName: Register Dogfood environment
     inputs:
-      displayName: Register Dogfood environment
       targetType: inline
       pwsh: true
       script: |
-        eng/common/TestResources/Import-AzModules.ps1
+        eng/common/scripts/Import-AzModules.ps1
 
         $environmentSpec = @"
           $(env-config-dogfood)

eng/common/TestResources/remove-test-resources.yml

@@ -0,0 +1,10 @@
+steps:
+- pwsh: |
+    . ./eng/common/scripts/Helpers/PSModule-Helpers.ps1
+    Write-Host "##vso[task.setvariable variable=CachedPSModulePath]$global:CurrentUserModulePath"
+  displayName: Set PS Modules Cache Directory
+- task: Cache@2
+  inputs:
+    key: 'PSModulePath | $(CacheSalt) | $(Agent.OS) | $(Build.SourcesDirectory)/eng/common/scripts/Import-AzModules.ps1'
+    path: $(CachedPSModulePath)
+  displayName: Cache PS Modules

eng/common/TestResources/setup-az-modules.yml renamed to eng/common/TestResources/setup-environments.yml

@@ -0,0 +1,55 @@
+function MapLanguageName($language)
+{
+    $lang = $language
+    # Update language name to match those in API cosmos DB. Cosmos SQL is case sensitive and handling this within the query makes it slow.
+    if($lang -eq 'javascript'){
+        $lang = "JavaScript"
+    }
+    elseif ($lang -eq "dotnet"){
+        $lang = "C#"
+    }
+    elseif ($lang -eq "java"){
+        $lang = "Java"
+    }
+    elseif ($lang -eq "python"){
+        $lang = "Python"
+    }
+    else{
+      $lang = $null
+    }
+    return $lang
+}
+
+function Check-ApiReviewStatus($packageName, $packageVersion, $language, $url, $apiKey)
+{
+  # Get API view URL and API Key to check status
+  Write-Host "Checking API review status"
+  $lang = MapLanguageName -language $language
+  if ($lang -eq $null) {
+    return
+  }
+  $headers = @{ "ApiKey" = $apiKey }
+  $body = @{
+    language = $lang
+    packageName = $packageName
+    packageVersion = $packageVersion
+  }
+
+  try
+  {
+    $response = Invoke-WebRequest $url -Method 'GET' -Headers $headers -Body $body
+    if ($response.StatusCode -eq '200')
+    {
+      Write-Host "API Review is approved for package $($packageName)"
+    }
+    else
+    {
+      Write-Warning "API Review is not approved for package $($packageName). Release pipeline will fail if API review is not approved."
+      Write-Host "You can check http://aka.ms/azsdk/engsys/apireview/faq for more details on API Approval."
+    }
+  }
+  catch
+  {
+    Write-Warning "Failed to check API review status for package $($PackageName). You can check http://aka.ms/azsdk/engsys/apireview/faq for more details on API Approval."
+  }
+}