Merge branch 'main' of https://github.com/Azure/azure-sdk-for-python into add_json_content

* 'main' of https://github.com/Azure/azure-sdk-for-python:
  [SchemaRegistry] fix logging_enable bug (Azure#21488)
  add ReplaceLatestEntryTitle parameter for updating changelog (Azure#21485)
  [rest] add kwargs to iter_bytes and iter_raw (Azure#21529)
  fix UnboundLocalError (Azure#19744)
  Use docker to do package validation. (Azure#21541)
  Use https for sparse checkout command (Azure#21535)
  [AutoRelease] t2-signalr-2021-11-01-50053 (Azure#21514)
  [textanalytics] release prep (Azure#21522)
  Increment package version after release of azure-core (Azure#21532)
  support charset_normalizer (Azure#21520)
  Sync eng/common directory with azure-sdk-tools for PR 2177 (Azure#21523)
  pin chardet to assist withfailures (Azure#21521)
  [formrecognizer] Initial work for adding get_children methods (Azure#21224)
  Revert "delete eventgrid" (Azure#21512)
  Eventgrid del temporarily (Azure#21510)
  delete eventgrid (Azure#21509)
  [AutoRelease] t2-eventhub-2021-10-25-83263 (Azure#21397)
  re-gen clu LLC client (Azure#21417)
  Improved some error behaviour (Azure#21499)
  Handling Proper Defaulting of `BuildTargetingString` within nested templates (Azure#21426)

Author: iscai-msft

eng/common/pipelines/templates/steps/docker-pull-image.yml

@@ -0,0 +1,15 @@
+parameters: 
+  - name: ContainerRegistryClientId
+    type: string
+  - name: ContainerRegistryClientSecret
+    type: string
+  - name: ImageId
+    type: string
+steps:
+- pwsh: |
+    $containerRegistry = ("${{parameters.ImageId}}" -split "\/")[0]
+    docker login $containerRegistry -u "${{ parameters.ContainerRegistryClientId }}" -p "${{ parameters.ContainerRegistryClientSecret }}"
+  displayName: Login container registry
+- pwsh: |
+    docker pull '${{ parameters.ImageId}}'
+  displayName: Pull docker image ${{ parameters.ImageId }}

eng/common/pipelines/templates/steps/sparse-checkout.yml

@@ -38,8 +38,8 @@ steps:
             } else {
               Write-Host "Repository $($repository.Name) is being initialized."
 
-              Write-Host "git clone --no-checkout --filter=tree:0 git://github.com/$($repository.Name) ."
-              git clone --no-checkout --filter=tree:0 git://github.com/$($repository.Name) .
+              Write-Host "git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) ."
+              git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) .
 
               Write-Host "git sparse-checkout init"
               git sparse-checkout init

eng/common/scripts/Invoke-GitHubAPI.ps1

@@ -157,6 +157,36 @@ function New-GitHubPullRequest {
           -MaximumRetryCount 3
 }
 
+function New-GitHubIssue {
+  param (
+    [Parameter(Mandatory = $true)]
+    $RepoOwner,
+    [Parameter(Mandatory = $true)]
+    $RepoName,
+    [Parameter(Mandatory = $true)]
+    $Title,
+    [Parameter(Mandatory = $true)]
+    $Description,
+    [ValidateNotNullOrEmpty()]
+    [Parameter(Mandatory = $true)]
+    $AuthToken
+  )
+
+  $uri = "$GithubAPIBaseURI/$RepoOwner/$RepoName/issues"
+
+  $parameters = @{
+    title = $Title
+    body = $Description
+  }
+
+  return Invoke-RestMethod `
+    -Method POST `
+    -Body ($parameters | ConvertTo-Json) `
+    -Uri $uri `
+    -Headers (Get-GitHubApiHeaders -token $AuthToken) `
+    -MaximumRetryCount 3
+}
+
 function Add-GitHubIssueComment {
   param (
     [Parameter(Mandatory = $true)]

eng/common/scripts/Update-DocsMsPackages.ps1

@@ -24,13 +24,20 @@ docs generation from pacakges which are not published to the default feed). This
 variable is meant to be used in the domain-specific business logic in
 &$UpdateDocsMsPackagesFn
 
+.PARAMETER ImageId
+Optional The docker image for package validation in format of '$containerRegistry/$imageName:$tag'. 
+e.g. azuresdkimages.azurecr.io/jsrefautocr:latest
+
 #>
 param (
   [Parameter(Mandatory = $true)]
   [string] $DocRepoLocation, # the location of the cloned doc repo
 
   [Parameter(Mandatory = $false)]
-  [string] $PackageSourceOverride
+  [string] $PackageSourceOverride,
+
+  [Parameter(Mandatory = $false)]
+  [string] $ImageId
 )
 
 . (Join-Path $PSScriptRoot common.ps1)

eng/pipelines/templates/jobs/ci.tests.yml

@@ -72,6 +72,10 @@ jobs:
       value: ${{ parameters.InjectedPackages }}
 
     steps:
+    - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
+      parameters:
+        BuildTargetingString: ${{ parameters.BuildTargetingString }}
+
     - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml
       parameters:
         AgentImage: $(OSVmImage)
@@ -99,7 +103,6 @@ jobs:
         OSVmImage: $(OSVmImage)
         CoverageArg: $(CoverageArg)
         PythonVersion: $(PythonVersion)
-        BuildTargetingString: ${{ parameters.BuildTargetingString }}
         ToxTestEnv: $(toxenv)
         ToxEnvParallel: ${{ parameters.ToxEnvParallel }}
         InjectedPackages: $(InjectedPackages)
@@ -112,7 +115,6 @@ jobs:
           - template: ../steps/set-dev-build.yml
             parameters:
               ServiceDirectory: ${{ parameters.ServiceDirectory }}
-              BuildTargetingString: ${{ parameters.BuildTargetingString }}
 
           - ${{ each step in parameters.BeforeTestSteps }}:
             -  ${{ step }}

eng/pipelines/templates/jobs/ci.yml

@@ -70,10 +70,13 @@ jobs:
       vmImage: MMSUbuntu20.04
 
     steps:
+    - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
+      parameters:
+        BuildTargetingString: ${{ parameters.BuildTargetingString }}
+
     - template: ../steps/build-artifacts.yml
       parameters:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
-        BuildTargetingString: ${{ parameters.BuildTargetingString }}
         BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
         BuildDocs: ${{ parameters.BuildDocs }}
         TestPipeline: ${{ parameters.TestPipeline }}
@@ -98,6 +101,10 @@ jobs:
       vmImage: MMSUbuntu20.04
 
     steps:
+    - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
+      parameters:
+        BuildTargetingString: ${{ parameters.BuildTargetingString }}
+
     - template: /eng/common/pipelines/templates/steps/check-spelling.yml
 
     - template: /eng/common/pipelines/templates/steps/verify-links.yml
@@ -112,7 +119,6 @@ jobs:
     - template: ../steps/analyze.yml
       parameters:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
-        BuildTargetingString: ${{ parameters.BuildTargetingString }}
         TestMarkArgument: ${{ parameters.TestMarkArgument }}
         AdditionalTestArgs: '--wheel_dir="$(Build.ArtifactStagingDirectory)"'
         TestPipeline: ${{ parameters.TestPipeline }}
@@ -180,7 +186,10 @@ jobs:
       vmImage: MMSUbuntu20.04
 
     steps:
+    - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
+      parameters:
+        BuildTargetingString: ${{ parameters.BuildTargetingString }}
+
     - template: ../steps/test_regression.yml
       parameters:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
-        BuildTargetingString: ${{ parameters.BuildTargetingString }}

eng/pipelines/templates/jobs/live.tests.yml

@@ -68,8 +68,6 @@ jobs:
         value: true
       - name: ArmTemplateParameters
         value: '@{}'
-      - name: BuildTargetingStringValue
-        value: $[ coalesce(variables['BuildTargetingString'], '${{ parameters.BuildTargetingString }}') ]
 
     timeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }}
     continueOnError: false
@@ -87,6 +85,10 @@ jobs:
         parameters:
           AgentImage: $(OSVmImage)
 
+      - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
+        parameters:
+          BuildTargetingString: ${{ parameters.BuildTargetingString }}
+
       - template: /eng/common/TestResources/build-test-resource-config.yml
         parameters:
           SubscriptionConfiguration: ${{ parameters.CloudConfig.SubscriptionConfiguration }}
@@ -101,7 +103,6 @@ jobs:
 
       - template: ../steps/build-test.yml
         parameters:
-          BuildTargetingString: $(BuildTargetingStringValue)
           ServiceDirectory: ${{ parameters.ServiceDirectory }}
           CloudName: ${{ parameters.CloudConfig.Cloud }}
           CoverageArg: $(CoverageArg)

eng/pipelines/templates/stages/archetype-sdk-client.yml

@@ -72,9 +72,6 @@ parameters:
 
 stages:
   - stage: Build
-    variables:
-      - name: BuildTargetingStringValue
-        value: $[ coalesce(variables['BuildTargetingString'], '${{ parameters.BuildTargetingString }}') ]
     jobs:
     - template: /eng/pipelines/templates/jobs/ci.yml
       parameters:
@@ -87,7 +84,7 @@ stages:
         AfterTestSteps: ${{ parameters.AfterTestSteps }}
         BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
         TestMarkArgument: ${{ parameters.TestMarkArgument }}
-        BuildTargetingString: $(BuildTargetingStringValue)
+        BuildTargetingString: '${{ parameters.BuildTargetingString }}'
         TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }}
         ToxEnvParallel: ${{ parameters.ToxEnvParallel }}
         InjectedPackages: ${{ parameters.InjectedPackages }}

eng/pipelines/templates/steps/analyze.yml

@@ -1,5 +1,4 @@
 parameters:
-  BuildTargetingString: 'azure-*'
   ServiceDirectory: ''
   TestMarkArgument: ''
   AdditionalTestArgs: ''
@@ -8,6 +7,12 @@ parameters:
   VerifyAutorest: false
   ValidateFormatting: false
 
+# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files:
+#     eng/pipelines/templates/jobs/ci.tests.yml
+#     eng/pipelines/templates/jobs/ci.yml
+#     eng/pipelines/templates/jobs/live.test.yml
+
+# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`.
 steps:
   - template: /eng/pipelines/templates/steps/analyze_dependency.yml
 
@@ -25,16 +30,6 @@ steps:
           ServiceName: ${{parameters.ServiceDirectory}}
           ForRelease: false
 
-# - template: /eng/common/pipelines/templates/steps/verify-samples.yml
-#   parameters:
-#     ServiceDirectory: ${{parameters.ServiceDirectory}}
-
-    # Using --always-succeed so as not to block the build. Once package
-    # target is based on data available per-package the --always-succeed should
-    # be removed so this script can help enforce correct practices
-    # (https://github.com/Azure/azure-sdk-for-python/issues/8697)
-
-
   - pwsh: |
      pip install -r eng/ci_tools.txt $(if($IsWindows) {"--user" })
     displayName: 'Install Necessary Dependencies'
@@ -83,31 +78,28 @@ steps:
     condition: and(succeededOrFailed(), ne(variables['Skip.VerifySdist'],'true'))
     inputs:
      scriptPath: 'scripts/devops_tasks/setup_execute_tests.py'
-     arguments: '"${{ parameters.BuildTargetingString }}" --service=${{parameters.ServiceDirectory}} --toxenv=verifysdist'
+     arguments: '"$(TargetingString)" --service=${{parameters.ServiceDirectory}} --toxenv=verifysdist'
 
   - task: PythonScript@0
     displayName: 'Verify whl'
     condition: and(succeededOrFailed(), ne(variables['Skip.VerifyWhl'],'true'))
     inputs:
      scriptPath: 'scripts/devops_tasks/setup_execute_tests.py'
-     arguments: '"${{ parameters.BuildTargetingString }}" --service=${{parameters.ServiceDirectory}} --toxenv=verifywhl'
+     arguments: '"$(TargetingString)" --service=${{parameters.ServiceDirectory}} --toxenv=verifywhl'
 
   - template: run_mypy.yml
     parameters:
       ServiceDirectory: ${{ parameters.ServiceDirectory }}
-      BuildTargetingString: ${{ parameters.BuildTargetingString }}
       TestMarkArgument: ${{ parameters.TestMarkArgument }}
 
   - template: run_pylint.yml
     parameters:
       ServiceDirectory: ${{ parameters.ServiceDirectory }}
-      BuildTargetingString: ${{ parameters.BuildTargetingString }}
 
   - ${{ if parameters.ValidateFormatting }}:
     - template: run_black.yml
       parameters:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
-        BuildTargetingString: ${{ parameters.BuildTargetingString }}
         ValidateFormatting: ${{ parameters.ValidateFormatting }}
 
   - task: DownloadPipelineArtifact@2
@@ -119,21 +111,18 @@ steps:
   - template: ../steps/run_apistub.yml
     parameters:
       ServiceDirectory: ${{ parameters.ServiceDirectory }}
-      BuildTargetingString: ${{ parameters.BuildTargetingString }}
       TestMarkArgument: ${{ parameters.TestMarkArgument }}
       AdditionalTestArgs: ${{parameters.AdditionalTestArgs}}
 
   - template: ../steps/run_bandit.yml
     parameters:
       ServiceDirectory: ${{ parameters.ServiceDirectory }}
-      BuildTargetingString: ${{ parameters.BuildTargetingString }}
       TestMarkArgument: ${{ parameters.TestMarkArgument }}
       AdditionalTestArgs: ${{parameters.AdditionalTestArgs}}
 
   - template: ../steps/run_breaking_changes.yml
     parameters:
       ServiceDirectory: ${{ parameters.ServiceDirectory }}
-      BuildTargetingString: ${{ parameters.BuildTargetingString }}
       TestMarkArgument: ${{ parameters.TestMarkArgument }}
       AdditionalTestArgs: ${{parameters.AdditionalTestArgs}}
 

eng/pipelines/templates/steps/build-artifacts.yml

@@ -5,9 +5,6 @@ parameters:
   - name: TestPipeline
     type: boolean
     default: false
-  - name: BuildTargetingString
-    type: string
-    default: 'azure-*'
   - name: ServiceDirectory
     type: string
     default: ''
@@ -18,13 +15,21 @@ parameters:
     type: object
     default: []
 
+# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files:
+#     eng/pipelines/templates/jobs/ci.tests.yml
+#     eng/pipelines/templates/jobs/ci.yml
+#     eng/pipelines/templates/jobs/live.test.yml
+
+# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`.
 steps:
   - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml
     parameters:
       PackageName: "azure-template"
       ServiceDirectory: "template"
       TestPipeline: ${{ parameters.TestPipeline }}
+
   - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
+
   - script: |
       echo "##vso[build.addbuildtag]Scheduled"
     displayName: 'Tag scheduled builds'
@@ -43,7 +48,7 @@ steps:
     displayName: 'Generate Python2 Applicable Namespace Packages'
     inputs:
       scriptPath: 'scripts/devops_tasks/build_packages.py'
-      arguments: '-d "$(Build.ArtifactStagingDirectory)" "${{ parameters.BuildTargetingString }}" --pkgfilter="nspkg" --service=${{parameters.ServiceDirectory}}'
+      arguments: '-d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --pkgfilter="nspkg" --service=${{parameters.ServiceDirectory}}'
 
   - task: UsePythonVersion@0
     displayName: 'Use Python $(PythonVersion)'
@@ -57,7 +62,7 @@ steps:
   - template: ../steps/set-dev-build.yml
     parameters:
       ServiceDirectory: ${{ parameters.ServiceDirectory }}
-      BuildTargetingString: ${{ parameters.BuildTargetingString }}
+
   - task: Powershell@2
     inputs:
       filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1
@@ -73,7 +78,7 @@ steps:
     displayName: 'Generate Packages'
     inputs:
       scriptPath: 'scripts/devops_tasks/build_packages.py'
-      arguments: '-d "$(Build.ArtifactStagingDirectory)" "${{ parameters.BuildTargetingString }}" --service=${{parameters.ServiceDirectory}} --devbuild="$(SetDevVersion)"'
+      arguments: '-d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --service=${{parameters.ServiceDirectory}} --devbuild="$(SetDevVersion)"'
 
   - script: |
       twine check $(Build.ArtifactStagingDirectory)/**/*.whl
@@ -86,7 +91,7 @@ steps:
     inputs:
       scriptPath: 'scripts/devops_tasks/setup_execute_tests.py'
       arguments: >-
-        "${{ parameters.BuildTargetingString }}" 
+        "$(TargetingString)" 
         --service="${{ parameters.ServiceDirectory }}" 
         --toxenv=sphinx