[Extension] Support installing extensions in a system directory

src/azure-cli-core/azure/cli/core/extension/__init__.py

@@ -2,12 +2,12 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
+# pylint: disable=line-too-long
 
 import os
 import traceback
 import json
 import re
-import sys
 import pkginfo
 
 from azure.cli.core._config import GLOBAL_CONFIG_DIR, ENV_VAR_PREFIX
@@ -18,10 +18,11 @@
 az_config = CLIConfig(config_dir=GLOBAL_CONFIG_DIR, config_env_var_prefix=ENV_VAR_PREFIX)
 _CUSTOM_EXT_DIR = az_config.get('extension', 'dir', None)
 _DEV_EXTENSION_SOURCES = az_config.get('extension', 'dev_sources', None)
+_CUSTOM_EXT_SYS_DIR = az_config.get('extension', 'sys_dir', None)
 EXTENSIONS_DIR = os.path.expanduser(_CUSTOM_EXT_DIR) if _CUSTOM_EXT_DIR else os.path.join(GLOBAL_CONFIG_DIR,
                                                                                           'cliextensions')
 DEV_EXTENSION_SOURCES = _DEV_EXTENSION_SOURCES.split(',') if _DEV_EXTENSION_SOURCES else []
-EXTENSIONS_SYS_DIR = os.path.join(get_python_lib(), 'azure-cli-extensions') if sys.platform.startswith('linux') else ""
+EXTENSIONS_SYS_DIR = os.path.expanduser(_CUSTOM_EXT_SYS_DIR) if _CUSTOM_EXT_SYS_DIR else os.path.join(get_python_lib(), 'azure-cli-extensions')
 
 EXTENSIONS_MOD_PREFIX = 'azext_'
 
@@ -134,10 +135,10 @@ def get_version(self):
 
     def get_metadata(self):
         from glob import glob
-        if not extension_exists(self.name):
-            return None
         metadata = {}
         ext_dir = self.path or get_extension_path(self.name)
+        if not ext_dir or not os.path.isdir(ext_dir):
+            return None
         info_dirs = glob(os.path.join(ext_dir, '*.*-info'))
         azext_metadata = WheelExtension.get_azext_metadata(ext_dir)
         if azext_metadata:
@@ -204,11 +205,10 @@ def get_version(self):
         return self.metadata.get('version')
 
     def get_metadata(self):
-
-        if not extension_exists(self.name):
-            return None
         metadata = {}
         ext_dir = self.path
+        if not ext_dir or not os.path.isdir(ext_dir):
+            return None
         egg_info_dirs = [f for f in os.listdir(ext_dir) if f.endswith('.egg-info')]
         azext_metadata = DevExtension.get_azext_metadata(ext_dir)
         if azext_metadata:
@@ -303,7 +303,15 @@ def get_extension_modname(ext_name=None, ext_dir=None):
 
 def get_extension_path(ext_name):
     # This will simply form the path for a WHEEL extension.
-    return os.path.join(EXTENSIONS_DIR, ext_name)
+    ext_sys_path = os.path.join(EXTENSIONS_SYS_DIR, ext_name)
+    ext_path = os.path.join(EXTENSIONS_DIR, ext_name)
+    return ext_path if os.path.isdir(ext_path) else (
+        ext_sys_path if os.path.isdir(ext_sys_path) else None)
+
+
+def build_extension_path(ext_name, system=None):
+    # This will simply form the path for a WHEEL extension.
+    return os.path.join(EXTENSIONS_SYS_DIR, ext_name) if system else os.path.join(EXTENSIONS_DIR, ext_name)
 
 
 def get_extensions(ext_type=None):

src/azure-cli-core/azure/cli/core/extension/operations.py

@@ -2,6 +2,8 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
+# pylint: disable=line-too-long
+
 from collections import OrderedDict
 import sys
 import os
@@ -17,7 +19,7 @@
 from pkg_resources import parse_version
 
 from azure.cli.core.util import CLIError, reload_module
-from azure.cli.core.extension import (extension_exists, get_extension_path, get_extensions, get_extension_modname,
+from azure.cli.core.extension import (extension_exists, build_extension_path, get_extensions, get_extension_modname,
                                       get_extension, ext_compat_with_cli,
                                       EXT_METADATA_ISPREVIEW, EXT_METADATA_ISEXPERIMENTAL,
                                       WheelExtension, DevExtension, ExtensionNotInstalledException, WHEEL_INFO_RE)
@@ -37,6 +39,7 @@
 OUT_KEY_METADATA = 'metadata'
 OUT_KEY_PREVIEW = 'preview'
 OUT_KEY_EXPERIMENTAL = 'experimental'
+OUT_KEY_PATH = 'path'
 
 IS_WINDOWS = sys.platform.lower() in ['windows', 'win32']
 LIST_FILE_PATH = os.path.join(os.sep, 'etc', 'apt', 'sources.list.d', 'azure-cli.list')
@@ -79,7 +82,7 @@ def _validate_whl_extension(ext_file):
     check_version_compatibility(azext_metadata)
 
 
-def _add_whl_ext(cmd, source, ext_sha256=None, pip_extra_index_urls=None, pip_proxy=None):  # pylint: disable=too-many-statements
+def _add_whl_ext(cmd, source, ext_sha256=None, pip_extra_index_urls=None, pip_proxy=None, system=None):  # pylint: disable=too-many-statements
     cmd.cli_ctx.get_progress_controller().add(message='Analyzing')
     if not source.endswith('.whl'):
         raise ValueError('Unknown extension type. Only Python wheels are supported.')
@@ -135,7 +138,7 @@ def _add_whl_ext(cmd, source, ext_sha256=None, pip_extra_index_urls=None, pip_pr
     check_distro_consistency()
     cmd.cli_ctx.get_progress_controller().add(message='Installing')
     # Install with pip
-    extension_path = get_extension_path(extension_name)
+    extension_path = build_extension_path(extension_name, system)
     pip_args = ['install', '--target', extension_path, ext_file]
 
     if pip_proxy:
@@ -168,12 +171,12 @@ def is_valid_sha256sum(a_file, expected_sum):
     return expected_sum == computed_hash, computed_hash
 
 
-def _augment_telemetry_with_ext_info(extension_name):
+def _augment_telemetry_with_ext_info(extension_name, ext=None):
     # The extension must be available before calling this otherwise we can't get the version from metadata
     if not extension_name:
         return
     try:
-        ext = get_extension(extension_name)
+        ext = ext or get_extension(extension_name)
         ext_version = ext.version
         set_extension_management_detail(extension_name, ext_version)
     except Exception:  # nopa pylint: disable=broad-except
@@ -183,8 +186,8 @@ def _augment_telemetry_with_ext_info(extension_name):
 
 def check_version_compatibility(azext_metadata):
     is_compatible, cli_core_version, min_required, max_required = ext_compat_with_cli(azext_metadata)
-    logger.debug("Extension compatibility result: is_compatible=%s cli_core_version=%s min_required=%s "
-                 "max_required=%s", is_compatible, cli_core_version, min_required, max_required)
+    logger.debug("Extension %s compatibility result: is_compatible=%s cli_core_version=%s min_required=%s "
+                 "max_required=%s", azext_metadata.get('name'), is_compatible, cli_core_version, min_required, max_required)
     if not is_compatible:
         min_max_msg_fmt = "The '{}' extension is not compatible with this version of the CLI.\n" \
                           "You have CLI core version {} and this extension " \
@@ -200,7 +203,7 @@ def check_version_compatibility(azext_metadata):
 
 
 def add_extension(cmd, source=None, extension_name=None, index_url=None, yes=None,  # pylint: disable=unused-argument
-                  pip_extra_index_urls=None, pip_proxy=None):
+                  pip_extra_index_urls=None, pip_proxy=None, system=None):
     ext_sha256 = None
     if extension_name:
         cmd.cli_ctx.get_progress_controller().add(message='Searching')
@@ -220,13 +223,14 @@ def add_extension(cmd, source=None, extension_name=None, index_url=None, yes=Non
             logger.debug(err)
             raise CLIError("No matching extensions for '{}'. Use --debug for more information.".format(extension_name))
     extension_name = _add_whl_ext(cmd=cmd, source=source, ext_sha256=ext_sha256,
-                                  pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy)
-    _augment_telemetry_with_ext_info(extension_name)
+                                  pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy, system=system)
+    ext = get_extension(extension_name)
+    _augment_telemetry_with_ext_info(extension_name, ext)
     try:
-        if extension_name and get_extension(extension_name).experimental:
+        if extension_name and ext.experimental:
             logger.warning("The installed extension '%s' is experimental and not covered by customer support. "
                            "Please use with discretion.", extension_name)
-        elif extension_name and get_extension(extension_name).preview:
+        elif extension_name and ext.preview:
             logger.warning("The installed extension '%s' is in preview.", extension_name)
     except ExtensionNotInstalledException:
         pass
@@ -245,15 +249,15 @@ def log_err(func, path, exc_info):
                 "Extension '{name}' was installed in development mode. Remove using "
                 "`azdev extension remove {name}`".format(name=extension_name))
         # We call this just before we remove the extension so we can get the metadata before it is gone
-        _augment_telemetry_with_ext_info(extension_name)
-        shutil.rmtree(get_extension_path(extension_name), onerror=log_err)
+        _augment_telemetry_with_ext_info(extension_name, ext)
+        shutil.rmtree(ext.path, onerror=log_err)
     except ExtensionNotInstalledException as e:
         raise CLIError(e)
 
 
 def list_extensions():
     return [{OUT_KEY_NAME: ext.name, OUT_KEY_VERSION: ext.version, OUT_KEY_TYPE: ext.ext_type,
-             OUT_KEY_PREVIEW: ext.preview, OUT_KEY_EXPERIMENTAL: ext.experimental}
+             OUT_KEY_PREVIEW: ext.preview, OUT_KEY_EXPERIMENTAL: ext.experimental, OUT_KEY_PATH: ext.path}
             for ext in get_extensions()]
 
 
@@ -263,7 +267,8 @@ def show_extension(extension_name):
         return {OUT_KEY_NAME: extension.name,
                 OUT_KEY_VERSION: extension.version,
                 OUT_KEY_TYPE: extension.ext_type,
-                OUT_KEY_METADATA: extension.metadata}
+                OUT_KEY_METADATA: extension.metadata,
+                OUT_KEY_PATH: extension.path}
     except ExtensionNotInstalledException as e:
         raise CLIError(e)
 
@@ -279,7 +284,7 @@ def update_extension(cmd, extension_name, index_url=None, pip_extra_index_urls=N
             raise CLIError("No updates available for '{}'. Use --debug for more information.".format(extension_name))
         # Copy current version of extension to tmp directory in case we need to restore it after a failed install.
         backup_dir = os.path.join(tempfile.mkdtemp(), extension_name)
-        extension_path = get_extension_path(extension_name)
+        extension_path = ext.path
         logger.debug('Backing up the current extension: %s to %s', extension_path, backup_dir)
         shutil.copytree(extension_path, backup_dir)
         # Remove current version of the extension

src/azure-cli/azure/cli/command_modules/extension/__init__.py

@@ -64,6 +64,7 @@ def load_arguments(self, command):
             c.argument('pip_extra_index_urls', options_list=['--pip-extra-index-urls'], nargs='+',
                        help='Space-separated list of extra URLs of package indexes to use. This should point to a repository compliant with PEP 503 (the simple repository API) or a local directory laid out in the same format.', arg_group='Experimental Pip')
             c.ignore('_subscription')  # hide global subscription param
+            c.argument('system', action='store_true', help='Use a system directory for the extension. Default path is azure-cli-extensions folder under the CLI running python environment lib path, configurable by environment variable AZURE_EXTENSION_SYS_DIR.')
 
         with self.argument_context('extension add') as c:
             c.argument('extension_name', completer=extension_name_from_index_completion_list)

src/azure-cli/azure/cli/command_modules/extension/_help.py

@@ -24,6 +24,8 @@
     text: az extension add --source ~/anextension-0.0.1-py2.py3-none-any.whl
   - name: Add extension from local disk and use pip proxy for dependencies
     text: az extension add --source ~/anextension-0.0.1-py2.py3-none-any.whl --pip-proxy https://user:[email protected]:8080
+  - name: Add extension to system directory
+    text: az extension add --name anextension --system
 """
 
 helps['extension list'] = """

src/azure-cli/azure/cli/command_modules/extension/custom.py

@@ -12,9 +12,9 @@
 
 
 def add_extension_cmd(cmd, source=None, extension_name=None, index_url=None, yes=None,
-                      pip_extra_index_urls=None, pip_proxy=None):
+                      pip_extra_index_urls=None, pip_proxy=None, system=None):
     return add_extension(cmd=cmd, source=source, extension_name=extension_name, index_url=index_url, yes=yes,
-                         pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy)
+                         pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy, system=system)
 
 
 def remove_extension_cmd(extension_name):