Update

.azure-pipelines/templates/automation_test.yml

@@ -8,9 +8,13 @@ parameters:
   type: string
   default: latest
 - name: fullTest
-  displayName: 'Run full test?'
+  displayName: Run full test?
   type: boolean
   default: false
+- name: module
+  displayName: Run test for specific module
+  type: string
+  default: ''
 
 steps:
   - task: UsePythonVersion@0
@@ -26,15 +30,24 @@ steps:
       echo pythonVersion: ${{ parameters.pythonVersion }}
       echo profile: ${{ parameters.profile }}
       echo fullTest: ${{ parameters.fullTest }}
+      echo module: ${{ parameters.module }}
       echo Build.Reason: $(Build.Reason)
 
+      # Test specific module
+      module="${{ parameters.module }}"
+      if [[ -n $module ]]; then
+        echo "Running test for module '$module'"
+        azdev test --no-exitfirst --verbose --series $module
+        exit 0
+      fi
+
       if [[ "$(Build.Reason)" == "PullRequest" && "${{ parameters.fullTest }}" == 'False' ]]; then
         echo "Running incremental test"
         azdev test --no-exitfirst --repo=./ --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch) --cli-ci --profile ${{ parameters.profile }} --verbose --series
       else
         echo "Running full test"
         azdev test --no-exitfirst --profile ${{ parameters.profile }} --verbose --series
       fi
-    displayName: "Test on Profile ${{ parameters.profile }}"
+    displayName: "azdev test"
     env:
       ADO_PULL_REQUEST_LATEST_COMMIT: $(System.PullRequest.TargetBranch)

.azure-pipelines/templates/azdev_setup.yml

@@ -21,7 +21,7 @@ steps:
       else
         azdev setup -c $CLI_REPO_PATH -r $CLI_EXT_REPO_PATH --debug
       fi
-    displayName: 'Azdev Setup'
+    displayName: 'azdev setup'
     env:
       CLI_REPO_PATH: ${{ parameters.CLIRepoPath }}
       CLI_EXT_REPO_PATH: ${{ parameters.CLIExtensionRepoPath }}

.github/CODEOWNERS

@@ -13,6 +13,10 @@
 /src/azure-cli-testsdk/ @jsntcy @jiasli @kairu-ms @qwordy
 /src/azure-cli-core/ @jiasli @Juliehzl @fengzhou-msft @evelyn-ys @jsntcy @houk-ms
 /src/azure-cli-core/azure/cli/core/extension/ @msyyc @fengzhou-msft
+/src/azure-cli-core/azure/cli/core/_profile.py @jiasli @evelyn-ys
+/src/azure-cli-core/azure/cli/core/adal_authentication.py @jiasli @evelyn-ys
+/src/azure-cli-core/azure/cli/core/msal_authentication.py @jiasli @evelyn-ys
+/src/azure-cli-core/azure/cli/core/style.py @jiasli @evelyn-ys @zhoxing-ms
 /src/azure-cli/azure/cli/command_modules/acr/ @djyou @fengzhou-msft @yungezz
 /src/azure-cli/azure/cli/command_modules/acs/ @rjtsdl @fengzhou-msft
 /src/azure-cli/azure/cli/command_modules/advisor/ @Prasanna-Padmanabhan

azure-pipelines.yml

@@ -189,7 +189,7 @@ jobs:
     inputs:
       TargetPath: '$(Build.ArtifactStagingDirectory)/metadata'
       artifactName: metadata
-  
+
   - task: DownloadPipelineArtifact@1
     displayName: 'Download Build Artifacts'
     inputs:
@@ -201,7 +201,7 @@ jobs:
     inputs:
       targetType: inline
       script: |
-        if (-NOT ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")) {  
+        if (-NOT ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")) {
           # Start another Powershell process as Admin and execute this script again
           $arguments = "& '" +$myinvocation.mycommand.definition + "'"
           Start-Process powershell -Verb runAs -ArgumentList $arguments
@@ -223,7 +223,7 @@ jobs:
         $to_be_installed_version=Get-Content $(System.ArtifactsDirectory)/metadata/version
         if ($pre_installed_version -eq $to_be_installed_version){
           # See https://docs.microsoft.com/windows/win32/msi/reinstallmode about options of REINSTALLMODE
-          $reinstall_option="REINSTALL=ALL REINSTALLMODE=emus" 
+          $reinstall_option="REINSTALL=ALL REINSTALLMODE=emus"
           $InstallArgs += $reinstall_option
         }
         Start-Process "msiexec.exe" -ArgumentList $InstallArgs -Wait -NoNewWindow
@@ -372,31 +372,39 @@ jobs:
 
     displayName: 'Test pip Install'
 
-- job: UnitTest
-  displayName: Unit Test for Core and Telemetry
+- job: TestCore
+  displayName: Unit Test for Core
   timeoutInMinutes: 10
+  pool:
+    vmImage: 'ubuntu-16.04'
+  strategy:
+    matrix:
+      Python36:
+        python.version: '3.6'
+      Python38:
+        python.version: '3.8'
+  steps:
+    - template: .azure-pipelines/templates/automation_test.yml
+      parameters:
+        pythonVersion: '$(python.version)'
+        module: 'azure-cli-core'
 
+- job: TestTelemetry
+  displayName: Unit Test for Telemetry
+  timeoutInMinutes: 10
   pool:
     vmImage: 'ubuntu-16.04'
   strategy:
     matrix:
       Python36:
         python.version: '3.6'
-        tox_env: 'py36'
       Python38:
         python.version: '3.8'
-        tox_env: 'py38'
   steps:
-    - task: UsePythonVersion@0
-      displayName: 'Use Python $(python.version)'
-      inputs:
-        versionSpec: '$(python.version)'
-    - bash: pip install --upgrade pip tox
-      displayName: 'Install pip and tox'
-    - bash: ./scripts/ci/unittest.sh
-      displayName: 'Run Unit Test'
-      env:
-        TOXENV: $(tox_env)
+    - template: .azure-pipelines/templates/automation_test.yml
+      parameters:
+        pythonVersion: '$(python.version)'
+        module: 'azure-cli-telemetry'
 
 - job: IntegrationTestAgainstProfiles
   displayName: Integration Test against Profiles
@@ -858,6 +866,7 @@ jobs:
   - bash: |
       set -ev
       . env/bin/activate
+      python scripts/ci/service_name.py
       if [[ "$(System.PullRequest.TargetBranch)" != "" ]]; then
         azdev linter --ci-exclusions --min-severity medium --repo=./ --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch)
       else

doc/authoring_command_modules/README.md

@@ -20,64 +20,35 @@ The document provides instructions and guidelines on how to author command modul
 <a name="heading_set_up"></a>Set Up
 ------
 
-Create you dev environment if you haven't already. This is how to do that.  
+Create your dev environment if you haven't already. This is how to do that.  
 
 Clone the repo, enter the repo directory then create your virtual environment.  
 
 For example:
 ```
 git clone https://github.com/Azure/azure-cli.git
-cd azure-cli
-virtualenv env
+git clone https://github.com/Azure/azure-cli-extensions.git
+python -m venv env
 source env/bin/activate
-python scripts/dev_setup.py
+azdev setup -c azure-cli -r azure-cli-extensions
 ```
+For more information, see https://github.com/Azure/azure-cli-dev-tools#setting-up-your-development-environment.
 
 After this, you should be able to run the CLI with `az`.
 
 [Author your command module...](#heading_author_command_mod)
 
-Now, install your command module into the environment with pip in editable mode.  
-Where `<path_to_your_command_module>` is the path to the directory containing your `setup.py` file.
-```
-pip install -e <path_to_your_command_module>
-```
-
-If installation was successful, you should be able to run `pip list` and see your command module.
-```
-$ pip list
-...
-azure-cli-example (0.0.1, /Users/myuser/Repos/azure-cli-example)
-...
-```
-
-Also, you can run `az` and if your command module contributes any commands, they should appear.
+If your command module contributes any commands, they should appear when running `az`.
 If your commands aren't showing with `az`, use `az --debug` to help debug. There could have been an exception
 thrown whilst attempting to load your module.
 
 
 <a name="heading_author_command_mod"></a>Authoring command modules
 ------
-Currently, all command modules should start with `azure-cli-`.  
-When the CLI loads, it search for packages installed that start with that prefix.
-
-The `example_module_template` directory gives an example command module with other useful examples.
-
-Command modules should have the following structure:
-```
-.
-|-- README.rst
-|-- azure
-|   |-- __init__.py
-|   `-- cli
-|       |-- __init__.py
-|       `-- command_modules
-|           |-- __init__.py
-|           `-- <MODULE_NAME>
-|               `-- __init__.py
-`-- setup.py
-`-- HISTORY.rst
-```
+
+There are two options to initialize a command module:
+1. Use [Azure CLI Code Generator tool](https://github.com/Azure/autorest.az#how-does-azure-cli-code-generator-work) to generate code automatically.
+2. [Create a module with `azdev cli create`](https://azurecliprod.blob.core.windows.net/videos/04%20-%20AzdevCliCreate.mp4).
 
 **Create an \_\_init__.py for your module**
 

doc/track_2_migration_guidance.md

@@ -1,12 +1,17 @@
 # Track 2 Migration Guidance
 
-Azure CLI is built upon Azure Python SDK. Recently Azure Python SDK announced next generation product. It is named Track 2 SDK. The old version of SDK is named Track 1. It claims that it has great advantages than Track 1 SDK. It is not compatible with Track 1 SDK. Azure CLI developers need to spend considerable time and do some work to migrate from Track 1 SDK to Track 2 SDK. Let's see an example of Track 2 SDK. [azure-mgmt-compute 17.0.0b1](https://pypi.org/project/azure-mgmt-compute/17.0.0b1/) introduces important breaking changes and important new features like unified authentication and asynchronous programming.
+Azure CLI is built on Azure Python SDKs. Recently, Azure Python SDK team announced the next generation product, named Track 2 SDK. The old version of SDK is called Track 1 SDK. It claims that it has some advantages than Track 1 SDK. It is not compatible with Track 1 SDK. A considerable number of work days are required for Azure CLI developers to migrate their modules from Track 1 SDK to Track 2 SDK. Let's see an example of Track 2 SDK. [azure-mgmt-compute 17.0.0b1](https://pypi.org/project/azure-mgmt-compute/17.0.0b1/) introduces important breaking changes and important new features like unified authentication and asynchronous programming.
 
 This document summarizes typical issues and solutions when adopting Track 2 SDK in Azure CLI.
 
 Example PRs:
 1. [Compute PR #15750](https://github.com/Azure/azure-cli/pull/15750)
 2. [Network PR #16245](https://github.com/Azure/azure-cli/pull/16245)
+3. [Network PR #16350](https://github.com/Azure/azure-cli/pull/16350)
+4. [Storage PR #15845](https://github.com/Azure/azure-cli/pull/15845)
+5. [KeyVault PR #14150](https://github.com/Azure/azure-cli/pull/14150)
+6. [AppConfig PR #16376](https://github.com/Azure/azure-cli/pull/16376)
+7. [AppService PR #17146](https://github.com/Azure/azure-cli/pull/17146)
 
 Below is a list of typical issues.
 
@@ -26,11 +31,12 @@ Below is a list of typical issues.
 
 Long running operations have changed their function names in Track 2 SDK. A `begin_` prefix is added. For example, `create_or_update` becomes `begin_create_or_update`. `delete` becomes `begin_delete`. It is a naming convention in Track 2 SDK to indicate that an operation is a long running operation. Test cases can reveal most instances, but if a command has no test, it may be missed. A reliable approach is going through all methods to see whether they are long running operations.
 
+
 ### Property name change
 
 Some of property names change in Track 2 SDK.
 
-Examples:
+Here are some examples. `->` is not Python code. It represents the left one is updated to the right one.
 
 ```
 hyperVgeneration -> hyperVGeneration
@@ -40,6 +46,7 @@ virtual_machine_extension_type -> type_properties_type
 type1 -> type_properties_type
 instance_ids -> vm_instance_i_ds
 diskMbpsReadWrite -> diskMBpsReadWrite
+ipv4 -> I_PV4
 ```
 
 Some changes are unnecessary, even wrong in English. I opened [Azure/autorest.python#850](https://github.com/Azure/autorest.python/issues/850) to track this problem.
@@ -48,7 +55,7 @@ Some changes are unnecessary, even wrong in English. I opened [Azure/autorest.py
 
 Some of class names change in Track 2 SDK.
 
-Examples:
+[Example](https://github.com/Azure/azure-cli/pull/15750/files#diff-fd5160263d5431e9cdbf0f83abad213589c44c4c2724ff66b1172218caeb8396R629):
 
 ```
 VirtualMachineIdentityUserAssignedIdentitiesValue -> UserAssignedIdentitiesValue
@@ -61,7 +68,9 @@ Error type changes in Track2 SDK.
 Examples:
 
 ```
-CloudError -> azure.core.exceptions.ResourceNotFoundError
+msrestazure.azure_exceptions.CloudError -> azure.core.exceptions.ResourceNotFoundError
+msrest.exceptions.ClientException -> azure.core.exceptions.HttpResponseError
+ErrorException -> HttpResponseError
 ```
 
 ### No enum type
@@ -70,13 +79,24 @@ Track 2 SDK removes enum type and adopts string type instead. It loses the valid
 
 ### Class hierarchy change
 
-The class hierarchy may change in Track 2 SDK. Some properties are not flattened. They are wrapped in classes. 
+The class hierarchy may change in Track 2 SDK. Some properties are not flattened. They are wrapped in classes. In Track 1 SDK, if the number of parameters is less than 3, it will be flattened.
 
 Examples:
 
-In VMSS `begin_update_instances`, a new type `VirtualMachineScaleSetVMInstanceRequiredIDs` is added.
+In [VMSS](https://github.com/Azure/azure-cli/pull/15750/files?file-filters%5B%5D=.py#diff-fd5160263d5431e9cdbf0f83abad213589c44c4c2724ff66b1172218caeb8396R2688) `begin_update_instances`, a new type `VirtualMachineScaleSetVMInstanceRequiredIDs` is added.
+
+In [DiskAccess](https://github.com/Azure/azure-cli/pull/15750/files#diff-fd5160263d5431e9cdbf0f83abad213589c44c4c2724ff66b1172218caeb8396R3602) `begin_create_or_update`, location and tags are moved to a nested structure `DiskAccess`, `disk_access = DiskAccess(location=location, tags=tags)`
+
+In [Storage](https://github.com/Azure/azure-cli/pull/15845/files#diff-4cfe9a680ae04774e116b45bc06a679db751bfad1de211c6d2b3bc471900d8bfR23),
+```
+client.check_name_availability(account_name) // account_name is string
+```
+turns into
+```
+account_name = StorageAccountCheckNameAvailabilityParameters(name=name) client.check_name_availability(account_name)
+```
 
-In DiskAccess `begin_create_or_update`, location and tags are moved to a nested structure `DiskAccess`, `disk_access = DiskAccess(location=location, tags=tags)`
+In [AppConfig](https://github.com/Azure/azure-cli/pull/16376/files#diff-1796b5bb574aca9235e83b02a207cb8a42aafab920f3aae1c46af22bf0ce5aa4R191), `id` cannot be passed directly to `regenerate_key method`. It needs to be wrapped in the new model `RegenerateKeyParameters`.
 
 ### Obtaining subscription
 
@@ -104,6 +124,8 @@ The problem I met is property name change. It is hard to find the line of code t
 
 It is only used in CI jobs. It patches some code to SDK. This file should be deprecated. It was written long time ago. But for now, just modify this file and add missing packages.
 
+[Example](https://github.com/Azure/azure-cli/pull/15750/files#diff-e1256a3d1d91aea524b252fa7dc4a64b83d183b7f57fb5c326b270a1c4b224a7)
+
 ### Missing external tenant authentication support
 
 External tenant authentication support is missing in Azure CLI Core and Azure Core package. In request header, we use x-ms-authorization-auxiliary to pass auxiliary authorization token. Compute module is the first customer to have this requirement in Track 2 SDK. In azure/core/pipeline/policies/_authentication.py, there is a class BearerTokenCredentialPolicy. It simplifies bearer token authorization header configuration. However, auxiliary bearer token authorization is not supported in Azure Core policies [1] yet. The current solution is getting tokens manually, setting headers in client constructor or in operation call time manually. They said they will support this policy in the future. Azure CLI Core also needs an upgrade to provide better interfaces for Track 2 SDK users.

doc/track_2_migration_roadmap.md

@@ -32,27 +32,28 @@ This document provides the roadmap for Track 2 SDK migration in Azure CLI. When
 | Compute     | Yes|**Completed** | 04/30/2021 |Catherine||GA (18.0.0) | GA (14.0.0) | High (Once a month) | Module |
 | KeyVault    | Yes|**Completed** | 04/30/2021 |Catherine||GA (8.0.0) | GA (2.2.0) | Medium (Once three months) | Module |
 | Monitor     |Yes |**Completed** | 04/30/2021 |Yong||Monitor: GA (2.0.0)<br> LogAnalytics: GA (8.0.0) | Monitor: Preview (0.12.0)<br> LogAnalytics: GA (2.0.0) | Medium (Once two months) | Module |
-| Resource   | Yes |**Doing** | 04/30/2021 |Yong ||GA (15.0.0) | GA (12.0.0) | Medium (Once two months) | Module |
-| AppService   |  Yes | **In progress** | 04/30/2021 |Catherine||GA (1.0.0) | Preview (0.48.0) | Medium (Once three months) | Module |
+| Resource   | Yes |**In progress** | 04/30/2021 |Yong ||GA (15.0.0) | GA (12.0.0) | Medium (Once two months) | Module |
+| AppService   |  Yes | **Completed** | 04/30/2021 |Catherine||GA (1.0.0) | Preview (0.48.0) | Medium (Once three months) | Module |
 | ACS   |  Yes |Not Started | 04/30/2021 |Catherine||ContainerService: GA (14.0.0)<br> ContainerInstance: GA (7.0.0) | ContainerService: GA (11.0.0)<br> ContainerInstance: GA (2.0.0) | Medium (Once two months) | Module |
 | ACR   | Yes |Not Started | 04/30/2021 |Catherine||Preview (8.0.0b1) | Preview (3.0.0rc16) | Medium (Once two months) | Module |
-| CosmosDB   | Yes |**Doing** | 04/30/2021 |Yong||GA (6.0.0) | Preview (2.0.0rc1) |  Medium (Once two months) |Module|
+| CosmosDB   | Yes |**Completed** | 04/30/2021 |Yong||GA (6.0.0) | Preview (2.0.0rc1) |  Medium (Once two months) |Module|
 | ServiceBus   | Yes |Not Started | 04/30/2021 |Catherine||GA (6.0.0) | GA (1.0.0) | Medium (Once four months) | Module |
-| CDN   | Yes |**Doing** | 04/30/2021 |Yong||GA (10.0.0) | GA (6.0.0) | Low (Once half year) | Module |
-| NetAppFiles |  Yes |**Doing** | 04/30/2021 |Yong||GA (1.0.0) | Preview (0.15.0) | Medium (Once two months) | Module|
-| PrivateDNS | Yes | **Doing** | 04/30/2021 |Yong||Preview (1.0.0b1) | Preview (0.1.0) |N/A | Module |
-| ApplicationInsights   |Yes | **Doing** | 04/30/2021 |Yong||GA (1.0.0) | Vendored_Preview (0.2.0) | N/A | Extension |
+| CDN   | Yes |**Completed** | 04/30/2021 |Yong||GA (10.0.0) | GA (6.0.0) | Low (Once half year) | Module |
+| NetAppFiles |  Yes |**Completed** | 04/30/2021 |Yong||GA (1.0.0) | Preview (0.15.0) | Medium (Once two months) | Module|
+| DNS |  Yes |**In progress** | 04/30/2021 |Yong||Preview (8.0.0b1) | GA (3.0.0) | Medium (Once two months) | Module|
+| PrivateDNS | Yes | **In progress** | 04/30/2021 |Yong||Preview (1.0.0b1) | Preview (0.1.0) |N/A | Module |
+| ApplicationInsights   |Yes | **Completed** | 04/30/2021 |Yong||GA (1.0.0) | Vendored_Preview (0.2.0) | N/A | Extension |
 | Firewall   |Yes| **Completed** | 04/30/2021 |Yong|| Network: GA (17.0.0) | Network: Vendored_GA (13.0.0) | N/A | Extension |
-| Frontdoor   | Yes | **Doing** | 04/30/2021 |Yong||Preview (1.0.0b1) | Vendored_Preview (0.3.1) | N/A | Extension |
-| VirtualWan   | Yes| **Doing** | 04/30/2021 |Yong||N/A |Vendored_Preview (0.1.0) | N/A | Extension |
+| Frontdoor   | Yes | **In progress** | 04/30/2021 |Yong||Preview (1.0.0b1) | Vendored_Preview (0.3.1) | N/A | Extension |
+| VirtualWan   | Yes| **In progress** | 04/30/2021 |Yong||N/A |Vendored_Preview (0.1.0) | N/A | Extension |
 |**Phase 2: Management&Governance&others** |  |  | |  | |  |||||
-| CostManagement   | Yes | **Completed** | 06/30/2021 |||Vendored_GA (1.2.0) | N/A | N/A | Extension |
-| DataFactory   | Yes | **Completed** | 06/30/2021 |||Vendored_SDK | N/A | N/A | Extension |
+| CostManagement   | Yes | **Completed** | 06/30/2021 |Yong||Vendored_GA (1.2.0) | N/A | N/A | Extension |
+| DataFactory   | Yes | **Completed** | 06/30/2021 |Catherine||Vendored_SDK | N/A | N/A | Extension |
 | RDBMS |  Yes|**Completed** | 06/30/2021 |Yong||GA (8.0.0) | Preview (3.1.0rc1) | Medium (Once two months) | Module |
 | EventGrid | Yes |Not Started | 06/30/2021 |Catherine|| GA (8.0.0) | Preview (3.0.0rc8) | Medium (Once three months) | Module |
 | HDInsight | Yes |Not Started | 06/30/2021 |Catherine|| GA (7.0.0) | GA (2.1.0) | Medium (Once two months) | Module |
 | Kusto |  Yes |Not Started | 06/30/2021 |Catherine||Preview (1.0.0b1) | Preview (0.10.0) | Medium (Once two months | Module |
-| SQL   | No |**Doing** | 06/30/2021 |Yong||GA (1.0.0) | Preview (0.25.0) | Medium (Once two months) | Module |
+| SQL   | No |**In progress** | 06/30/2021 |Yong||GA (1.0.0) | Preview (0.25.0) | Medium (Once two months) | Module |
 | Role   | No |Not Started | 06/30/2021 |Yong|| Authorization: GA (1.0.0)<br>MSI: N/A | Authorization: Preview (0.61.0)<br>MSI: GA (1.0.0) | Low (Once a year) | Module |
 |**Phase 3** |  |  | |  | |  ||||
 | NatGateway || **Completed** | N/A || |Network: GA (8.0.0) | Network: GA (2.2.0) | Use Network package| Module |