diff --git a/.azure-pipelines/templates/automation_test.yml b/.azure-pipelines/templates/automation_test.yml
index 6e0e542884d..c633bfe3c79 100644
--- a/.azure-pipelines/templates/automation_test.yml
+++ b/.azure-pipelines/templates/automation_test.yml
@@ -1,6 +1,16 @@
+# Azure DevOps task for automation test using azdev and pytest
+
 parameters:
-  pythonVersion: ''
-  profile: ''
+- name: pythonVersion
+  type: string
+  default: 3.8
+- name: profile
+  type: string
+  default: latest
+- name: fullTest
+  displayName: 'Run full test?'
+  type: boolean
+  default: false
 
 steps:
   - task: UsePythonVersion@0
@@ -13,10 +23,17 @@ steps:
 
       source env/bin/activate
 
-      if [[ "$(System.PullRequest.TargetBranch)" != "" ]]; then
-        azdev test --series --no-exitfirst --repo=./ --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch) --cli-ci --profile ${{ parameters.profile }}
+      echo pythonVersion: ${{ parameters.pythonVersion }}
+      echo profile: ${{ parameters.profile }}
+      echo fullTest: ${{ parameters.fullTest }}
+      echo Build.Reason: $(Build.Reason)
+
+      if [[ "$(Build.Reason)" == "PullRequest" && "${{ parameters.fullTest }}" == 'False' ]]; then
+        echo "Running incremental test"
+        azdev test --no-exitfirst --repo=./ --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch) --cli-ci --profile ${{ parameters.profile }} --verbose --series
       else
-        azdev test --series --no-exitfirst --profile ${{ parameters.profile }}
+        echo "Running full test"
+        azdev test --no-exitfirst --profile ${{ parameters.profile }} --verbose --series
       fi
     displayName: "Test on Profile ${{ parameters.profile }}"
     env:
diff --git a/azure-pipelines-full-tests.yml b/azure-pipelines-full-tests.yml
index 18a6ba50057..7058bf19864 100644
--- a/azure-pipelines-full-tests.yml
+++ b/azure-pipelines-full-tests.yml
@@ -13,144 +13,74 @@ pr:
       - '*'
 
 jobs:
-  - job: ExtractMetadata
-    displayName: Extract Metadata
+- job: AutomationTest
+  displayName: Automation Test (Profile Latest)
+  timeoutInMinutes: 120
+  pool:
+    vmImage: 'ubuntu-16.04'
+  strategy:
+    matrix:
+      Python36:
+        python.version: '3.6'
+      Python38:
+        python.version: '3.8'
+  steps:
+    - template: .azure-pipelines/templates/automation_test.yml
+      parameters:
+        pythonVersion: '$(python.version)'
+        profile: 'latest'
+        fullTest: true
 
-    condition: succeeded()
-    pool:
-      vmImage: 'ubuntu-16.04'
-    steps:
-      - task: Bash@3
-        displayName: 'Extract Version'
-        inputs:
-          targetType: 'filePath'
-          filePath: scripts/release/get_version.sh
-      - task: PublishPipelineArtifact@0
-        displayName: 'Publish Artifact: metadata'
-        inputs:
-          TargetPath: $(Build.ArtifactStagingDirectory)
-          ArtifactName: metadata
+- job: AutomationTest20200901
+  displayName: Automation Test (Profile 2020-09-01)
+  timeoutInMinutes: 120
+  pool:
+    vmImage: 'ubuntu-16.04'
+  strategy:
+    matrix:
+      Python36:
+        python.version: '3.6'
+      Python38:
+        python.version: '3.8'
+  steps:
+    - template: .azure-pipelines/templates/automation_test.yml
+      parameters:
+        pythonVersion: '$(python.version)'
+        profile: '2020-09-01-hybrid'
+        fullTest: true
 
-  - job: BuildPythonWheel
-    displayName: Build Python Wheels
-    dependsOn: ExtractMetadata
-    condition: succeeded()
-    pool:
-      vmImage: 'ubuntu-16.04'
-    steps:
-      - task: UsePythonVersion@0
-        displayName: 'Use Python 3.7'
-        inputs:
-          versionSpec: 3.7
+- job: AutomationTest20190301
+  displayName: Automation Test (Profile 2019-03-01)
+  timeoutInMinutes: 120
+  pool:
+    vmImage: 'ubuntu-16.04'
+  strategy:
+    matrix:
+      Python36:
+        python.version: '3.6'
+      Python38:
+        python.version: '3.8'
+  steps:
+    - template: .azure-pipelines/templates/automation_test.yml
+      parameters:
+        pythonVersion: '$(python.version)'
+        profile: '2019-03-01-hybrid'
+        fullTest: true
 
-      - script: |
-          if [[ "$(Build.Reason)" == "PullRequest" ]]; then
-            branch=$(System.PullRequest.TargetBranch)
-          else
-            branch=$(Build.SourceBranchName)
-          fi
-          scripts/release/pypi/build.sh $branch
-        displayName: 'Run Wheel Build Script'
-
-      - task: PublishPipelineArtifact@0
-        displayName: 'Publish Artifact: pypi'
-        inputs:
-          TargetPath: $(Build.ArtifactStagingDirectory)
-          ArtifactName: pypi
-
-  - job: AutomationTest
-    displayName: Automation Test
-    dependsOn: BuildPythonWheel
-    condition: succeeded()
-    timeoutInMinutes: 90
-
-    pool:
-      vmImage: 'ubuntu-16.04'
-    strategy:
-      matrix:
-        Python36:
-          python.version: '3.6'
-        Python38:
-          python.version: '3.8'
-    steps:
-      - task: UsePythonVersion@0
-        displayName: 'Use Python $(python.version)'
-        inputs:
-          versionSpec: '$(python.version)'
-      - task: DownloadPipelineArtifact@1
-        displayName: 'Download Python Wheels'
-        inputs:
-          TargetPath: '$(Build.ArtifactStagingDirectory)/pypi'
-          artifactName: pypi
-      - bash: |
-          pip install --force-reinstall wheel
-      - task: Bash@3
-        displayName: 'Run Test Automation'
-        inputs:
-          targetType: 'filePath'
-          filePath: ./scripts/ci/test_automation.sh
-
-  - job: RunAutomationReduced20190301
-    displayName: Run Automation Reduced, Profile 2019-03-01
-    dependsOn: BuildPythonWheel
-    condition: succeeded()
-    pool:
-      vmImage: 'ubuntu-16.04'
-    strategy:
-      matrix:
-        Python36:
-          python.version: '3.6'
-        Python38:
-          python.version: '3.8'
-    steps:
-      - task: UsePythonVersion@0
-        displayName: 'Use Python $(python.version)'
-        inputs:
-          versionSpec: '$(python.version)'
-      - task: DownloadPipelineArtifact@1
-        displayName: 'Download Python Wheels'
-        inputs:
-          TargetPath: '$(Build.ArtifactStagingDirectory)/pypi'
-          artifactName: pypi
-      - bash: |
-          pip install --force-reinstall wheel
-      - task: Bash@3
-        displayName: 'Run Test Automation'
-        inputs:
-          targetType: 'filePath'
-          filePath: ./scripts/ci/test_automation.sh
-        env:
-          REDUCE_SDK: 'True'
-          AZURE_CLI_TEST_TARGET_PROFILE: '2019-03-01'
-
-  - job: RunAutomation20180301
-    displayName: Run Automation, Profile 2018-03-01
-    dependsOn: BuildPythonWheel
-    condition: succeeded()
-    pool:
-      vmImage: 'ubuntu-16.04'
-    strategy:
-      matrix:
-        Python36:
-          python.version: '3.6'
-        Python38:
-          python.version: '3.8'
-    steps:
-      - task: UsePythonVersion@0
-        displayName: 'Use Python $(python.version)'
-        inputs:
-          versionSpec: '$(python.version)'
-      - task: DownloadPipelineArtifact@1
-        displayName: 'Download Python Wheels'
-        inputs:
-          TargetPath: '$(Build.ArtifactStagingDirectory)/pypi'
-          artifactName: pypi
-      - bash: |
-          pip install --force-reinstall wheel
-      - task: Bash@3
-        displayName: 'Run Test Automation'
-        inputs:
-          targetType: 'filePath'
-          filePath: ./scripts/ci/test_automation.sh
-        env:
-          AZURE_CLI_TEST_TARGET_PROFILE: '2018-03-01'
+- job: AutomationTest20180301
+  displayName: Automation Test (Profile 2018-03-01)
+  timeoutInMinutes: 120
+  pool:
+    vmImage: 'ubuntu-16.04'
+  strategy:
+    matrix:
+      Python36:
+        python.version: '3.6'
+      Python38:
+        python.version: '3.8'
+  steps:
+    - template: .azure-pipelines/templates/automation_test.yml
+      parameters:
+        pythonVersion: '$(python.version)'
+        profile: '2018-03-01-hybrid'
+        fullTest: true
diff --git a/scripts/ci/test_automation.sh b/scripts/ci/test_automation.sh
deleted file mode 100755
index 080b3e118fc..00000000000
--- a/scripts/ci/test_automation.sh
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/usr/bin/env bash
-
-set -ev
-
-. $(dirname ${BASH_SOURCE[0]})/artifacts.sh
-
-ls -la $share_folder/build
-
-ALL_MODULES=`find $share_folder/build/ -name "*.whl"`
-
-##############################################
-# Define colored output func
-function title {
-    LGREEN='\033[1;32m'
-    CLEAR='\033[0m'
-
-    echo -e ${LGREEN}$1${CLEAR}
-}
-
-title 'Install azdev'
-pip install -qqq -e ./tools
-
-title 'Install code coverage tools'
-pip install -qqq coverage codecov
-
-title 'Install private packages (optional)'
-[ -d privates ] && pip install -qqq privates/*.whl
-
-title 'Install products'
-pip install -qqq $ALL_MODULES
-
-title 'Installed packages'
-pip freeze
-
-if [ "$REDUCE_SDK" == "True" ]
-then
-    title 'azure.mgmt file counts'
-    (find $(dirname $(which python))/../lib -name '*.py' | grep azure/mgmt/ | wc)
-
-    python $(dirname ${BASH_SOURCE[0]})/../sdk_process/patch_models.py
-
-    title 'azure.mgmt file counts after reduce'
-    (find $(dirname $(which python))/../lib -name '*.py' | grep azure/mgmt/ | wc)
-fi
-
-target_profile=${AZURE_CLI_TEST_TARGET_PROFILE:-latest}
-if [ "$target_profile" != "latest" ]; then
-    # example: 2019-03-01-hybrid
-    target_profile=$target_profile-hybrid
-fi
-
-# test basic az commands
-az -v
-az -h
-
-title 'Running tests'
-python -m automation test --ci --profile $target_profile