Update 5/7

.azure-pipelines/templates/azdev_setup.yml

@@ -21,6 +21,8 @@ steps:
       else
         azdev setup -c $CLI_REPO_PATH -r $CLI_EXT_REPO_PATH --debug
       fi
+      # This helps detect issues in CI if a used SDK API version is deleted by the below script.
+      python $CLI_REPO_PATH/build_scripts/windows/scripts/remove_unused_api_versions.py
     displayName: 'azdev setup'
     env:
       CLI_REPO_PATH: ${{ parameters.CLIRepoPath }}

.github/CODEOWNERS

@@ -17,8 +17,8 @@
 /src/azure-cli-core/azure/cli/core/adal_authentication.py @jiasli @evelyn-ys
 /src/azure-cli-core/azure/cli/core/msal_authentication.py @jiasli @evelyn-ys
 /src/azure-cli-core/azure/cli/core/style.py @jiasli @evelyn-ys @zhoxing-ms
-/src/azure-cli/azure/cli/command_modules/acr/ @djyou @fengzhou-msft @yungezz
-/src/azure-cli/azure/cli/command_modules/acs/ @rjtsdl @fengzhou-msft
+/src/azure-cli/azure/cli/command_modules/acr/ @fengzhou-msft @yungezz @northtyphoon @rosanch
+/src/azure-cli/azure/cli/command_modules/acs/ @fengzhou-msft @zqingqing1 @gtracer @xizhamsft @andyliuliming
 /src/azure-cli/azure/cli/command_modules/advisor/ @Prasanna-Padmanabhan
 /src/azure-cli/azure/cli/command_modules/apim/ @kevinhillinger @jonlester
 /src/azure-cli/azure/cli/command_modules/appconfig/ @shenmuxiaosen @avanigupta @qwordy

azure-pipelines.yml

@@ -227,15 +227,20 @@ jobs:
           $InstallArgs += $reinstall_option
         }
         Start-Process "msiexec.exe" -ArgumentList $InstallArgs -Wait -NoNewWindow
-        Get-Content .\install_logs.txt
+        $install_time=Measure-Command {Start-Process "msiexec.exe" -ArgumentList $InstallArgs -Wait -NoNewWindow} | select -expand TotalSeconds
         $installed_version=az version --query '\"azure-cli\"' -o tsv
         if ($installed_version -ne $to_be_installed_version){
           echo "The MSI failed to install."
           Exit 1
         }
+        echo 'Install time(seconds):' $install_time
         az --version
+        # Test bundled pip with extension installation
+        az extension add -n rdbms-connect
         az self-test
 
+        Get-Content .\install_logs.txt
+
 - job: BuildDockerImage
   displayName: Build Docker Image
 

build_scripts/windows/scripts/build.cmd

@@ -9,10 +9,10 @@ if "%CLI_VERSION%"=="" (
     echo Please set the CLI_VERSION environment variable, e.g. 2.0.13
     goto ERROR
 )
-set PYTHON_VERSION=3.6.8
+set PYTHON_VERSION=3.8.9
 
 set WIX_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/msi/wix310-binaries-mirror.zip"
-set PYTHON_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/Python368-32.zip"
+set PYTHON_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/Python389-32.zip"
 set PROPAGATE_ENV_CHANGE_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/propagate_env_change.zip"
 
 :: Set up the output directory and temp. directories
@@ -26,7 +26,7 @@ mkdir %ARTIFACTS_DIR%
 set TEMP_SCRATCH_FOLDER=%ARTIFACTS_DIR%\cli_scratch
 set BUILDING_DIR=%ARTIFACTS_DIR%\cli
 set WIX_DIR=%ARTIFACTS_DIR%\wix
-set PYTHON_DIR=%ARTIFACTS_DIR%\Python368-32
+set PYTHON_DIR=%ARTIFACTS_DIR%\Python389-32
 set PROPAGATE_ENV_CHANGE_DIR=%~dp0..\propagate_env_change
 
 set REPO_ROOT=%~dp0..\..\..
@@ -75,15 +75,14 @@ if not exist %PYTHON_DIR% (
     mkdir %PYTHON_DIR%
     pushd %PYTHON_DIR%
     echo Downloading Python.
-    curl -o Python368-32.zip %PYTHON_DOWNLOAD_URL% -k
-    unzip -q Python368-32.zip
+    curl -o Python389-32.zip %PYTHON_DOWNLOAD_URL% -k
+    unzip -q Python389-32.zip
     if %errorlevel% neq 0 goto ERROR
-    del Python368-32.zip
+    del Python389-32.zip
     echo Python downloaded and extracted successfully.
     popd
 )
 set PYTHON_EXE=%PYTHON_DIR%\python.exe
-%PYTHON_EXE% -m pip install --upgrade pip==21.0.1 setuptools==52.0.0
 
 robocopy %PYTHON_DIR% %BUILDING_DIR% /s /NFL /NDL
 
@@ -100,6 +99,7 @@ if %errorlevel% neq 0 goto ERROR
 
 pushd %BUILDING_DIR%
 %BUILDING_DIR%\python.exe %~dp0\patch_models_v2.py
+%BUILDING_DIR%\python.exe %~dp0\remove_unused_api_versions.py
 popd
 
 echo Creating the wbin (Windows binaries) folder that will be added to the path...
@@ -123,11 +123,6 @@ for /f %%a in ('dir /b /s *_py3.*.pyc') do (
 )
 popd
 
-:: Remove unused Network SDK API versions
-pushd %BUILDING_DIR%\Lib\site-packages\azure\mgmt\network
-rmdir /s /q v2016_09_01 v2016_12_01 v2017_03_01 v2017_06_01 v2017_08_01 v2017_09_01 v2017_11_01 v2018_02_01 v2018_04_01 v2018_06_01 v2018_10_01 v2018_12_01 v2019_04_01 v2019_08_01 v2019_09_01 v2019_11_01 v2019_12_01 v2020_03_01
-popd
-
 :: Remove .py and only deploy .pyc files
 pushd %BUILDING_DIR%\Lib\site-packages
 for /f %%f in ('dir /b /s *.pyc') do (

build_scripts/windows/scripts/patch_models_v2.py

@@ -245,7 +245,7 @@ def find_models_to_change(module_name):
         return [
             importlib.import_module('.'+label+'.models', main_module.__name__)
             for (_, label, ispkg) in pkgutil.iter_modules(main_module.__path__)
-            if ispkg
+            if ispkg and label != 'aio'
         ]
 
 

build_scripts/windows/scripts/remove_unused_api_versions.py

@@ -0,0 +1,59 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import logging
+import os
+import re
+import shutil
+import azure.mgmt.network
+
+from azure.cli.core.profiles import AD_HOC_API_VERSIONS, AZURE_API_PROFILES, ResourceType
+
+
+_LOGGER = logging.getLogger(__name__)
+
+
+def remove_unused_network_api_versions():
+    # Hard-coded API versions
+    used_network_api_versions = set(AD_HOC_API_VERSIONS[ResourceType.MGMT_NETWORK].values())
+
+    # API versions in profile
+    for _, profile in AZURE_API_PROFILES.items():
+        if ResourceType.MGMT_NETWORK in profile:
+            used_network_api_versions.add(profile[ResourceType.MGMT_NETWORK])
+
+    # Normalize API version: 2019-02-01 -> v2019_02_01
+    used_network_api_vers = {f"v{api.replace('-','_')}" for api in used_network_api_versions}
+
+    # Network SDK has a set of versions imported in models.py.
+    # Let's keep them before we figure out how to remove a version in all related SDK files.
+    path = azure.mgmt.network.__path__[0]
+    model_file = os.path.join(path, 'models.py')
+    with open(model_file, 'r', encoding='utf-8') as f:
+        content = f.read()
+    for m in re.finditer(r'from \.(v[_\d]*)\.models import \*', content):
+        used_network_api_vers.add(m.group(1))
+
+    _LOGGER.info('Used network API versions:')
+    _LOGGER.info(sorted(used_network_api_vers))
+
+    all_api_vers = {d for d in os.listdir(path) if os.path.isdir(os.path.join(path, d)) and d.startswith('v')}
+    _LOGGER.info('All network API versions:')
+    _LOGGER.info(sorted(all_api_vers))
+
+    remove_api_vers = sorted(all_api_vers - used_network_api_vers)
+    _LOGGER.info('Network API versions that will be removed:')
+    _LOGGER.info(remove_api_vers)
+
+    for ver in remove_api_vers:
+        shutil.rmtree(os.path.join(path, ver))
+
+
+def main():
+    remove_unused_network_api_versions()
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    main()

build_scripts/windows/scripts/setup_msi_test.ps1

@@ -16,6 +16,6 @@ $testsdk = Get-ChildItem -Path $PSScriptRoot\..\..\..\artifacts\build\azure_cli_
 & 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install $PSScriptRoot\..\..\..\artifacts\build\$($testsdk.Name)
 
 $fulltest = Get-ChildItem -Path $PSScriptRoot\..\..\..\artifacts\build\azure_cli_fulltest*.whl | Select-Object Name
-& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install $PSScriptRoot\..\..\..\artifacts\build\$($fulltest.Name)
+& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install --no-deps $PSScriptRoot\..\..\..\artifacts\build\$($fulltest.Name)
 
 & 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' $PSScriptRoot\test_msi_package.py

doc/sphinx/azhelpgen/doc_source_map.json

@@ -75,5 +75,6 @@
     "aro": "src/azure-cli/azure/cli/command_modules/aro/_help.py",
     "util": "src/azure-cli/azure/cli/command_modules/util/_help.py",
     "synapse": "src/azure-cli/azure/cli/command_modules/synapse/_help.py",
-    "databoxedge": "src/azure-cli/azure/cli/command_modules/databoxedge/_help.py"
+    "databoxedge": "src/azure-cli/azure/cli/command_modules/databoxedge/_help.py",
+    "term": "src/azure-cli/azure/cli/command_modules/marketplaceordering/_help.py"
 }

doc/use_cli_in_airgapped_clouds.md

@@ -8,7 +8,7 @@ Here is a list of known CLI features that are not supported in airgapped clouds:
 * `az upgrade` to upgrade Azure CLI to the latest version. Instead, you can follow the below [Install](#Install) instructions to upgrade CLI.
 * `az find` to find command examples.
 * Command recommendations based on Aladdin service when the command cannot be parsed correctly.
-* Commands that install another tool such as `az aks install-cli`, `az acr helm install-cli`, `az storage copy`.
+* Commands that install another tool such as `az aks install-cli`, `az storage copy`.
 * External channel operations in `az bot` are not available such as Facebook and Wechat.
 * Some links in help messages or error messages cannot be accessible.
 

linter_exclusions.yml

@@ -259,6 +259,9 @@ aks create:
     node_osdisk_diskencryptionset_id:
       rule_exclusions:
       - option_length_too_long
+    enable_encryption_at_host:
+      rule_exclusions:
+      - option_length_too_long
 aks enable-addons:
   parameters:
     workspace_resource_id:
@@ -277,6 +280,9 @@ aks nodepool add:
     node_public_ip_prefix_id:
       rule_exclusions:
       - option_length_too_long
+    enable_encryption_at_host:
+      rule_exclusions:
+      - option_length_too_long
 aks update:
   parameters:
     aad_admin_group_object_ids:
@@ -300,6 +306,9 @@ aks update:
     load_balancer_outbound_ports:
       rule_exclusions:
       - option_length_too_long
+    enable_managed_identity:
+      rule_exclusions:
+      - option_length_too_long
 aks update-credentials:
   parameters:
     aad_server_app_secret:

src/azure-cli-core/HISTORY.rst

@@ -3,6 +3,12 @@
 Release History
 ===============
 
+2.23.0
+++++++
+* Display allowed values in error message when enum validation fails (#17621)
+* Refactor AzCommandGroup (#17387)
+* Add a linter rule for `service_name.json` (#17428)
+
 2.22.1
 ++++++
 * Minor fixes

src/azure-cli-core/azure/cli/core/__init__.py

@@ -4,7 +4,7 @@
 # --------------------------------------------------------------------------------------------
 # pylint: disable=line-too-long
 
-__version__ = "2.22.1"
+__version__ = "2.23.0"
 
 import os
 import sys

src/azure-cli-core/azure/cli/core/profiles/__init__.py

@@ -5,7 +5,7 @@
 
 #  pylint: disable=unused-import
 from azure.cli.core.profiles._shared import AZURE_API_PROFILES, ResourceType, CustomResourceType, PROFILE_TYPE,\
-    SDKProfile
+    SDKProfile, AD_HOC_API_VERSIONS
 
 
 def get_api_version(cli_ctx, resource_type, as_sdk_profile=False):

src/azure-cli-core/azure/cli/core/profiles/_shared.py

@@ -336,6 +336,19 @@ def default_api_version(self):
 }
 
 
+# We should avoid using ad hoc API versions,
+# use the version in a profile as much as possible.
+AD_HOC_API_VERSIONS = {
+    ResourceType.MGMT_NETWORK: {
+        'vm_default_target_network': '2018-01-01',
+        'nw_connection_monitor': '2019-06-01',
+        'container_network': '2018-08-01',
+        'appservice_network': '2020-04-01',
+        'appservice_ensure_subnet': '2019-02-01'
+    }
+}
+
+
 class _ApiVersions:  # pylint: disable=too-few-public-methods
     def __init__(self, client_type, sdk_profile, post_process):
         self._client_type = client_type

src/azure-cli-core/setup.py

@@ -8,7 +8,7 @@
 from codecs import open
 from setuptools import setup, find_packages
 
-VERSION = "2.22.1"
+VERSION = "2.23.0"
 
 # If we have source, validate that our version numbers match
 # This should prevent uploading releases with mismatched versions.