diff --git a/src/azure-cli-core/azure/cli/core/__init__.py b/src/azure-cli-core/azure/cli/core/__init__.py index 341ff9ad8dd..44f6b52d934 100644 --- a/src/azure-cli-core/azure/cli/core/__init__.py +++ b/src/azure-cli-core/azure/cli/core/__init__.py @@ -36,7 +36,7 @@ def __init__(self, **kwargs): from azure.cli.core.commands.arm import ( register_ids_argument, register_global_subscription_argument) from azure.cli.core.cloud import get_active_cloud - from azure.cli.core.extensions import register_extensions + from azure.cli.core.commands.transform import register_global_transforms from azure.cli.core._session import ACCOUNT, CONFIG, SESSION from knack.util import ensure_dir @@ -55,7 +55,7 @@ def __init__(self, **kwargs): self.cloud = get_active_cloud(self) logger.debug('Current cloud config:\n%s', str(self.cloud.name)) - register_extensions(self) + register_global_transforms(self) register_global_subscription_argument(self) register_ids_argument(self) # global subscription must be registered first! @@ -110,7 +110,7 @@ def load_command_table(self, args): import traceback from azure.cli.core.commands import ( _load_module_command_loader, _load_extension_command_loader, BLACKLISTED_MODS, ExtensionCommandSource) - from azure.cli.core.extension import ( + from azure.cli.core.extensions import ( get_extensions, get_extension_path, get_extension_modname) def _update_command_table_from_modules(args): diff --git a/src/azure-cli-core/azure/cli/core/commands/__init__.py b/src/azure-cli-core/azure/cli/core/commands/__init__.py index 605c0b13a3c..2eff60bec5e 100644 --- a/src/azure-cli-core/azure/cli/core/commands/__init__.py +++ b/src/azure-cli-core/azure/cli/core/commands/__init__.py @@ -27,7 +27,7 @@ CLI_POSITIONAL_PARAM_KWARGS, CONFIRM_PARAM_NAME) from azure.cli.core.commands.parameters import ( AzArgumentContext, patch_arg_make_required, patch_arg_make_optional) -from azure.cli.core.extension import get_extension +from azure.cli.core.extensions import get_extension from azure.cli.core.util import get_command_type_kwarg, read_file_content, get_arg_list, poller_classes import azure.cli.core.telemetry as telemetry diff --git a/src/azure-cli-core/azure/cli/core/commands/client_factory.py b/src/azure-cli-core/azure/cli/core/commands/client_factory.py index 72c89c582d3..a3442494104 100644 --- a/src/azure-cli-core/azure/cli/core/commands/client_factory.py +++ b/src/azure-cli-core/azure/cli/core/commands/client_factory.py @@ -10,7 +10,7 @@ from azure.cli.core import __version__ as core_version import azure.cli.core._debug as _debug -from azure.cli.core.extension import EXTENSIONS_MOD_PREFIX +from azure.cli.core.extensions import EXTENSIONS_MOD_PREFIX from azure.cli.core.profiles._shared import get_client_class, SDKProfile from azure.cli.core.profiles import ResourceType, CustomResourceType, get_api_version, get_sdk diff --git a/src/azure-cli-core/azure/cli/core/extensions/transform.py b/src/azure-cli-core/azure/cli/core/commands/transform.py similarity index 98% rename from src/azure-cli-core/azure/cli/core/extensions/transform.py rename to src/azure-cli-core/azure/cli/core/commands/transform.py index 27b18a8c3cc..fc3f0e248b8 100644 --- a/src/azure-cli-core/azure/cli/core/extensions/transform.py +++ b/src/azure-cli-core/azure/cli/core/commands/transform.py @@ -10,7 +10,7 @@ from azure.cli.core.util import b64_to_hex -def register(cli_ctx): +def register_global_transforms(cli_ctx): cli_ctx.register_event(events.EVENT_INVOKER_TRANSFORM_RESULT, _resource_group_transform) cli_ctx.register_event(events.EVENT_INVOKER_TRANSFORM_RESULT, _x509_from_base64_to_hex_transform) diff --git a/src/azure-cli-core/azure/cli/core/extension.py b/src/azure-cli-core/azure/cli/core/extension.py deleted file mode 100644 index 1aeebf271f6..00000000000 --- a/src/azure-cli-core/azure/cli/core/extension.py +++ /dev/null @@ -1,205 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -import os -import traceback -import json - -from knack.log import get_logger - -from azure.cli.core._config import GLOBAL_CONFIG_DIR - - -_CUSTOM_EXT_DIR = os.environ.get('AZURE_EXTENSION_DIR') -EXTENSIONS_DIR = os.path.expanduser(_CUSTOM_EXT_DIR) if _CUSTOM_EXT_DIR else os.path.join(GLOBAL_CONFIG_DIR, - 'cliextensions') -EXTENSIONS_MOD_PREFIX = 'azext_' - -WHL_METADATA_FILENAME = 'metadata.json' -AZEXT_METADATA_FILENAME = 'azext_metadata.json' - -EXT_METADATA_MINCLICOREVERSION = 'azext.minCliCoreVersion' -EXT_METADATA_MAXCLICOREVERSION = 'azext.maxCliCoreVersion' -EXT_METADATA_ISPREVIEW = 'azext.isPreview' - -logger = get_logger(__name__) - - -class ExtensionNotInstalledException(Exception): - def __init__(self, extension_name): - super(ExtensionNotInstalledException, self).__init__(extension_name) - self.extension_name = extension_name - - def __str__(self): - return "The extension {} is not installed.".format(self.extension_name) - - -class Extension(object): - - def __init__(self, name, ext_type): - self.name = name - self.ext_type = ext_type - self._version = None - self._metadata = None - self._preview = None - - @property - def version(self): - """ - Lazy load version. - Returns the version as a string or None if not available. - """ - try: - self._version = self._version or self.get_version() - except Exception: # pylint: disable=broad-except - logger.debug("Unable to get extension version: %s", traceback.format_exc()) - return self._version - - @property - def metadata(self): - """ - Lazy load metadata. - Returns the metadata as a dictionary or None if not available. - """ - try: - self._metadata = self._metadata or self.get_metadata() - except Exception: # pylint: disable=broad-except - logger.debug("Unable to get extension metadata: %s", traceback.format_exc()) - return self._metadata - - @property - def preview(self): - """ - Lazy load preview status. - Returns the preview status of the extension. - """ - try: - if not isinstance(self._preview, bool): - self._preview = bool(self.metadata.get(EXT_METADATA_ISPREVIEW)) - except Exception: # pylint: disable=broad-except - logger.debug("Unable to get extension preview status: %s", traceback.format_exc()) - return self._preview - - def get_version(self): - raise NotImplementedError() - - def get_metadata(self): - raise NotImplementedError() - - @staticmethod - def get_all(): - raise NotImplementedError() - - -class WheelExtension(Extension): - - def __init__(self, name): - super(WheelExtension, self).__init__(name, 'whl') - - def get_version(self): - return self.metadata.get('version') - - def get_metadata(self): - from wheel.install import WHEEL_INFO_RE - if not extension_exists(self.name): - return None - metadata = {} - ext_dir = get_extension_path(self.name) - dist_info_dirs = [f for f in os.listdir(ext_dir) if f.endswith('.dist-info')] - azext_metadata = WheelExtension.get_azext_metadata(ext_dir) - if azext_metadata: - metadata.update(azext_metadata) - for dist_info_dirname in dist_info_dirs: - parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname) - if parsed_dist_info_dir and parsed_dist_info_dir.groupdict().get('name') == self.name.replace('-', '_'): - whl_metadata_filepath = os.path.join(ext_dir, dist_info_dirname, WHL_METADATA_FILENAME) - if os.path.isfile(whl_metadata_filepath): - with open(whl_metadata_filepath) as f: - metadata.update(json.load(f)) - return metadata - - @staticmethod - def get_azext_metadata(ext_dir): - azext_metadata = None - ext_modname = get_extension_modname(ext_dir=ext_dir) - azext_metadata_filepath = os.path.join(ext_dir, ext_modname, AZEXT_METADATA_FILENAME) - if os.path.isfile(azext_metadata_filepath): - with open(azext_metadata_filepath) as f: - azext_metadata = json.load(f) - return azext_metadata - - @staticmethod - def get_all(): - """ - Returns all wheel-based extensions. - """ - exts = [] - if os.path.isdir(EXTENSIONS_DIR): - for ext_name in os.listdir(EXTENSIONS_DIR): - ext_path = get_extension_path(ext_name) - if os.path.isdir(ext_path) and \ - next((f for f in os.listdir(ext_path) if f.endswith(('.dist-info', '.egg-info'))), None): - exts.append(WheelExtension(ext_name)) - return exts - - -EXTENSION_TYPES = [WheelExtension] - - -def ext_compat_with_cli(azext_metadata): - from azure.cli.core import __version__ as core_version - from pkg_resources import parse_version - is_compatible, min_required, max_required = (True, None, None) - if azext_metadata: - min_required = azext_metadata.get(EXT_METADATA_MINCLICOREVERSION) - max_required = azext_metadata.get(EXT_METADATA_MAXCLICOREVERSION) - parsed_cli_version = parse_version(core_version) - if min_required and parsed_cli_version < parse_version(min_required): - is_compatible = False - elif max_required and parsed_cli_version > parse_version(max_required): - is_compatible = False - return is_compatible, core_version, min_required, max_required - - -def get_extension_modname(ext_name=None, ext_dir=None): - ext_dir = ext_dir or get_extension_path(ext_name) - pos_mods = [n for n in os.listdir(ext_dir) - if n.startswith(EXTENSIONS_MOD_PREFIX) and os.path.isdir(os.path.join(ext_dir, n))] - if len(pos_mods) != 1: - raise AssertionError("Expected 1 module to load starting with " - "'{}': got {}".format(EXTENSIONS_MOD_PREFIX, pos_mods)) - return pos_mods[0] - - -def get_extension_path(ext_name): - return os.path.join(EXTENSIONS_DIR, ext_name) - - -def get_extensions(): - logger.debug("Extensions directory: '%s'", EXTENSIONS_DIR) - extensions = [] - for ext_type in EXTENSION_TYPES: - extensions.extend([ext for ext in ext_type.get_all()]) - return extensions - - -def get_extension(ext_name): - ext = next((ext for ext in get_extensions() if ext.name == ext_name), None) - if ext is None: - raise ExtensionNotInstalledException(ext_name) - return ext - - -def extension_exists(ext_name): - ext = next((ext for ext in get_extensions() if ext.name == ext_name), None) - return ext is not None - - -def get_extension_names(): - """ - Helper method to only get extension names. - Returns the extension names of extensions installed in the extensions directory. - """ - return [ext.name for ext in get_extensions()] diff --git a/src/azure-cli-core/azure/cli/core/extensions/__init__.py b/src/azure-cli-core/azure/cli/core/extensions/__init__.py index d7c33a11221..1aeebf271f6 100644 --- a/src/azure-cli-core/azure/cli/core/extensions/__init__.py +++ b/src/azure-cli-core/azure/cli/core/extensions/__init__.py @@ -3,8 +3,203 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -from azure.cli.core.extensions.transform import register as register_transform +import os +import traceback +import json +from knack.log import get_logger -def register_extensions(cli_ctx): - register_transform(cli_ctx) +from azure.cli.core._config import GLOBAL_CONFIG_DIR + + +_CUSTOM_EXT_DIR = os.environ.get('AZURE_EXTENSION_DIR') +EXTENSIONS_DIR = os.path.expanduser(_CUSTOM_EXT_DIR) if _CUSTOM_EXT_DIR else os.path.join(GLOBAL_CONFIG_DIR, + 'cliextensions') +EXTENSIONS_MOD_PREFIX = 'azext_' + +WHL_METADATA_FILENAME = 'metadata.json' +AZEXT_METADATA_FILENAME = 'azext_metadata.json' + +EXT_METADATA_MINCLICOREVERSION = 'azext.minCliCoreVersion' +EXT_METADATA_MAXCLICOREVERSION = 'azext.maxCliCoreVersion' +EXT_METADATA_ISPREVIEW = 'azext.isPreview' + +logger = get_logger(__name__) + + +class ExtensionNotInstalledException(Exception): + def __init__(self, extension_name): + super(ExtensionNotInstalledException, self).__init__(extension_name) + self.extension_name = extension_name + + def __str__(self): + return "The extension {} is not installed.".format(self.extension_name) + + +class Extension(object): + + def __init__(self, name, ext_type): + self.name = name + self.ext_type = ext_type + self._version = None + self._metadata = None + self._preview = None + + @property + def version(self): + """ + Lazy load version. + Returns the version as a string or None if not available. + """ + try: + self._version = self._version or self.get_version() + except Exception: # pylint: disable=broad-except + logger.debug("Unable to get extension version: %s", traceback.format_exc()) + return self._version + + @property + def metadata(self): + """ + Lazy load metadata. + Returns the metadata as a dictionary or None if not available. + """ + try: + self._metadata = self._metadata or self.get_metadata() + except Exception: # pylint: disable=broad-except + logger.debug("Unable to get extension metadata: %s", traceback.format_exc()) + return self._metadata + + @property + def preview(self): + """ + Lazy load preview status. + Returns the preview status of the extension. + """ + try: + if not isinstance(self._preview, bool): + self._preview = bool(self.metadata.get(EXT_METADATA_ISPREVIEW)) + except Exception: # pylint: disable=broad-except + logger.debug("Unable to get extension preview status: %s", traceback.format_exc()) + return self._preview + + def get_version(self): + raise NotImplementedError() + + def get_metadata(self): + raise NotImplementedError() + + @staticmethod + def get_all(): + raise NotImplementedError() + + +class WheelExtension(Extension): + + def __init__(self, name): + super(WheelExtension, self).__init__(name, 'whl') + + def get_version(self): + return self.metadata.get('version') + + def get_metadata(self): + from wheel.install import WHEEL_INFO_RE + if not extension_exists(self.name): + return None + metadata = {} + ext_dir = get_extension_path(self.name) + dist_info_dirs = [f for f in os.listdir(ext_dir) if f.endswith('.dist-info')] + azext_metadata = WheelExtension.get_azext_metadata(ext_dir) + if azext_metadata: + metadata.update(azext_metadata) + for dist_info_dirname in dist_info_dirs: + parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname) + if parsed_dist_info_dir and parsed_dist_info_dir.groupdict().get('name') == self.name.replace('-', '_'): + whl_metadata_filepath = os.path.join(ext_dir, dist_info_dirname, WHL_METADATA_FILENAME) + if os.path.isfile(whl_metadata_filepath): + with open(whl_metadata_filepath) as f: + metadata.update(json.load(f)) + return metadata + + @staticmethod + def get_azext_metadata(ext_dir): + azext_metadata = None + ext_modname = get_extension_modname(ext_dir=ext_dir) + azext_metadata_filepath = os.path.join(ext_dir, ext_modname, AZEXT_METADATA_FILENAME) + if os.path.isfile(azext_metadata_filepath): + with open(azext_metadata_filepath) as f: + azext_metadata = json.load(f) + return azext_metadata + + @staticmethod + def get_all(): + """ + Returns all wheel-based extensions. + """ + exts = [] + if os.path.isdir(EXTENSIONS_DIR): + for ext_name in os.listdir(EXTENSIONS_DIR): + ext_path = get_extension_path(ext_name) + if os.path.isdir(ext_path) and \ + next((f for f in os.listdir(ext_path) if f.endswith(('.dist-info', '.egg-info'))), None): + exts.append(WheelExtension(ext_name)) + return exts + + +EXTENSION_TYPES = [WheelExtension] + + +def ext_compat_with_cli(azext_metadata): + from azure.cli.core import __version__ as core_version + from pkg_resources import parse_version + is_compatible, min_required, max_required = (True, None, None) + if azext_metadata: + min_required = azext_metadata.get(EXT_METADATA_MINCLICOREVERSION) + max_required = azext_metadata.get(EXT_METADATA_MAXCLICOREVERSION) + parsed_cli_version = parse_version(core_version) + if min_required and parsed_cli_version < parse_version(min_required): + is_compatible = False + elif max_required and parsed_cli_version > parse_version(max_required): + is_compatible = False + return is_compatible, core_version, min_required, max_required + + +def get_extension_modname(ext_name=None, ext_dir=None): + ext_dir = ext_dir or get_extension_path(ext_name) + pos_mods = [n for n in os.listdir(ext_dir) + if n.startswith(EXTENSIONS_MOD_PREFIX) and os.path.isdir(os.path.join(ext_dir, n))] + if len(pos_mods) != 1: + raise AssertionError("Expected 1 module to load starting with " + "'{}': got {}".format(EXTENSIONS_MOD_PREFIX, pos_mods)) + return pos_mods[0] + + +def get_extension_path(ext_name): + return os.path.join(EXTENSIONS_DIR, ext_name) + + +def get_extensions(): + logger.debug("Extensions directory: '%s'", EXTENSIONS_DIR) + extensions = [] + for ext_type in EXTENSION_TYPES: + extensions.extend([ext for ext in ext_type.get_all()]) + return extensions + + +def get_extension(ext_name): + ext = next((ext for ext in get_extensions() if ext.name == ext_name), None) + if ext is None: + raise ExtensionNotInstalledException(ext_name) + return ext + + +def extension_exists(ext_name): + ext = next((ext for ext in get_extensions() if ext.name == ext_name), None) + return ext is not None + + +def get_extension_names(): + """ + Helper method to only get extension names. + Returns the extension names of extensions installed in the extensions directory. + """ + return [ext.name for ext in get_extensions()] diff --git a/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/_homebrew_patch.py b/src/azure-cli-core/azure/cli/core/extensions/_homebrew_patch.py similarity index 100% rename from src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/_homebrew_patch.py rename to src/azure-cli-core/azure/cli/core/extensions/_homebrew_patch.py diff --git a/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/_index.py b/src/azure-cli-core/azure/cli/core/extensions/_index.py similarity index 100% rename from src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/_index.py rename to src/azure-cli-core/azure/cli/core/extensions/_index.py diff --git a/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/_resolve.py b/src/azure-cli-core/azure/cli/core/extensions/_resolve.py similarity index 96% rename from src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/_resolve.py rename to src/azure-cli-core/azure/cli/core/extensions/_resolve.py index 383d481f406..6cbac714a41 100644 --- a/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/_resolve.py +++ b/src/azure-cli-core/azure/cli/core/extensions/_resolve.py @@ -7,9 +7,8 @@ from knack.log import get_logger -from azure.cli.core.extension import ext_compat_with_cli - -from azure.cli.command_modules.extension._index import get_index_extensions +from azure.cli.core.extensions import ext_compat_with_cli +from azure.cli.core.extensions._index import get_index_extensions logger = get_logger(__name__) diff --git a/src/azure-cli-core/azure/cli/core/extensions/experimental.py b/src/azure-cli-core/azure/cli/core/extensions/experimental.py deleted file mode 100644 index 21938b8fea5..00000000000 --- a/src/azure-cli-core/azure/cli/core/extensions/experimental.py +++ /dev/null @@ -1,37 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - - -def register(event_dispatcher): - def _enable_experimental_handlers(_, event_data): - """The user can opt in to experimental event handlers by - passing an --experimental flag on the command line - """ - try: - event_data['args'].remove('--experimental') - event_dispatcher.register(event_dispatcher.PARSING_PARAMETERS, - file_argument_value) - except ValueError: - pass - - def file_argument_value(_, event_data): - """Replace provided parameter value with value from file - if the parameter value starts with a '@' - """ - def load_file(path): - with open(path, 'r') as f: - return f.read() - - args = event_data['args'] - for name in args.keys(): - value = args[name] - try: - if str.startswith(value, '@'): - args[name] = load_file(value[1:]) - except TypeError: - pass - - event_dispatcher.register(event_dispatcher.REGISTER_GLOBAL_PARAMETERS, - _enable_experimental_handlers) diff --git a/src/azure-cli-core/azure/cli/core/extensions/operations.py b/src/azure-cli-core/azure/cli/core/extensions/operations.py new file mode 100644 index 00000000000..c2bdeae7ff7 --- /dev/null +++ b/src/azure-cli-core/azure/cli/core/extensions/operations.py @@ -0,0 +1,348 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +import sys +import os +import tempfile +import shutil +import zipfile +import traceback +import hashlib +from subprocess import check_output, STDOUT, CalledProcessError +from six.moves.urllib.parse import urlparse # pylint: disable=import-error +from collections import OrderedDict + +import requests +from wheel.install import WHEEL_INFO_RE +from pkg_resources import parse_version + +from knack.log import get_logger + +from azure.cli.core.util import CLIError +from azure.cli.core.extensions import (extension_exists, get_extension_path, get_extensions, + get_extension, ext_compat_with_cli, EXT_METADATA_ISPREVIEW, + WheelExtension, ExtensionNotInstalledException) +from azure.cli.core.telemetry import set_extension_management_detail + +from ._homebrew_patch import HomebrewPipPatch +from ._index import get_index_extensions +from ._resolve import resolve_from_index, NoExtensionCandidatesError + +logger = get_logger(__name__) + +OUT_KEY_NAME = 'name' +OUT_KEY_VERSION = 'version' +OUT_KEY_TYPE = 'extensionType' +OUT_KEY_METADATA = 'metadata' + +IS_WINDOWS = sys.platform.lower() in ['windows', 'win32'] +LIST_FILE_PATH = os.path.join(os.sep, 'etc', 'apt', 'sources.list.d', 'azure-cli.list') +LSB_RELEASE_FILE = os.path.join(os.sep, 'etc', 'lsb-release') + + +def _run_pip(pip_exec_args): + cmd = [sys.executable, '-m', 'pip'] + pip_exec_args + ['-vv', '--disable-pip-version-check', '--no-cache-dir'] + logger.debug('Running: %s', cmd) + try: + log_output = check_output(cmd, stderr=STDOUT, universal_newlines=True) + logger.debug(log_output) + returncode = 0 + except CalledProcessError as e: + logger.debug(e.output) + logger.debug(e) + returncode = e.returncode + return returncode + + +def _whl_download_from_url(url_parse_result, ext_file): + from azure.cli.core.util import should_disable_connection_verify + url = url_parse_result.geturl() + r = requests.get(url, stream=True, verify=(not should_disable_connection_verify())) + if r.status_code != 200: + raise CLIError("Request to {} failed with {}".format(url, r.status_code)) + with open(ext_file, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: # ignore keep-alive new chunks + f.write(chunk) + + +def _validate_whl_cli_compat(azext_metadata): + is_compatible, cli_core_version, min_required, max_required = ext_compat_with_cli(azext_metadata) + logger.debug("Extension compatibility result: is_compatible=%s cli_core_version=%s min_required=%s " + "max_required=%s", is_compatible, cli_core_version, min_required, max_required) + if not is_compatible: + min_max_msg_fmt = "The extension is not compatible with this version of the CLI.\n" \ + "You have CLI core version {} and this extension " \ + "requires ".format(cli_core_version) + if min_required and max_required: + min_max_msg_fmt += 'a min of {} and max of {}.'.format(min_required, max_required) + elif min_required: + min_max_msg_fmt += 'a min of {}.'.format(min_required) + elif max_required: + min_max_msg_fmt += 'a max of {}.'.format(max_required) + raise CLIError(min_max_msg_fmt) + + +def _validate_whl_extension(ext_file): + tmp_dir = tempfile.mkdtemp() + zip_ref = zipfile.ZipFile(ext_file, 'r') + zip_ref.extractall(tmp_dir) + zip_ref.close() + azext_metadata = WheelExtension.get_azext_metadata(tmp_dir) + shutil.rmtree(tmp_dir) + _validate_whl_cli_compat(azext_metadata) + + +def _add_whl_ext(source, ext_sha256=None, pip_extra_index_urls=None, pip_proxy=None): # pylint: disable=too-many-statements + if not source.endswith('.whl'): + raise ValueError('Unknown extension type. Only Python wheels are supported.') + url_parse_result = urlparse(source) + is_url = (url_parse_result.scheme == 'http' or url_parse_result.scheme == 'https') + logger.debug('Extension source is url? %s', is_url) + whl_filename = os.path.basename(url_parse_result.path) if is_url else os.path.basename(source) + parsed_filename = WHEEL_INFO_RE(whl_filename) + # Extension names can have - but .whl format changes it to _ (PEP 0427). Undo this. + extension_name = parsed_filename.groupdict().get('name').replace('_', '-') if parsed_filename else None + if not extension_name: + raise CLIError('Unable to determine extension name from {}. Is the file name correct?'.format(source)) + if extension_exists(extension_name): + raise CLIError('The extension {} already exists.'.format(extension_name)) + ext_file = None + if is_url: + # Download from URL + tmp_dir = tempfile.mkdtemp() + ext_file = os.path.join(tmp_dir, whl_filename) + logger.debug('Downloading %s to %s', source, ext_file) + try: + _whl_download_from_url(url_parse_result, ext_file) + except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err: + raise CLIError('Please ensure you have network connection. Error detail: {}'.format(str(err))) + logger.debug('Downloaded to %s', ext_file) + else: + # Get file path + ext_file = os.path.realpath(os.path.expanduser(source)) + if not os.path.isfile(ext_file): + raise CLIError("File {} not found.".format(source)) + # Validate the extension + logger.debug('Validating the extension %s', ext_file) + if ext_sha256: + valid_checksum, computed_checksum = is_valid_sha256sum(ext_file, ext_sha256) + if valid_checksum: + logger.debug("Checksum of %s is OK", ext_file) + else: + logger.debug("Invalid checksum for %s. Expected '%s', computed '%s'.", + ext_file, ext_sha256, computed_checksum) + raise CLIError("The checksum of the extension does not match the expected value. " + "Use --debug for more information.") + try: + _validate_whl_extension(ext_file) + except AssertionError: + logger.debug(traceback.format_exc()) + raise CLIError('The extension is invalid. Use --debug for more information.') + except CLIError as e: + raise e + logger.debug('Validation successful on %s', ext_file) + # Check for distro consistency + check_distro_consistency() + # Install with pip + extension_path = get_extension_path(extension_name) + pip_args = ['install', '--target', extension_path, ext_file] + + if pip_proxy: + pip_args = pip_args + ['--proxy', pip_proxy] + if pip_extra_index_urls: + for extra_index_url in pip_extra_index_urls: + pip_args = pip_args + ['--extra-index-url', extra_index_url] + + logger.debug('Executing pip with args: %s', pip_args) + with HomebrewPipPatch(): + pip_status_code = _run_pip(pip_args) + if pip_status_code > 0: + logger.debug('Pip failed so deleting anything we might have installed at %s', extension_path) + shutil.rmtree(extension_path, ignore_errors=True) + raise CLIError('An error occurred. Pip failed with status code {}. ' + 'Use --debug for more information.'.format(pip_status_code)) + # Save the whl we used to install the extension in the extension dir. + dst = os.path.join(extension_path, whl_filename) + shutil.copyfile(ext_file, dst) + logger.debug('Saved the whl to %s', dst) + + +def is_valid_sha256sum(a_file, expected_sum): + sha256 = hashlib.sha256() + with open(a_file, 'rb') as f: + sha256.update(f.read()) + computed_hash = sha256.hexdigest() + return expected_sum == computed_hash, computed_hash + + +def _augment_telemetry_with_ext_info(extension_name): + # The extension must be available before calling this otherwise we can't get the version from metadata + if not extension_name: + return + try: + ext = get_extension(extension_name) + ext_version = ext.version + set_extension_management_detail(extension_name, ext_version) + except Exception: # nopa pylint: disable=broad-except + # Don't error on telemetry + pass + + +def add_extension(source=None, extension_name=None, index_url=None, yes=None, # pylint: disable=unused-argument + pip_extra_index_urls=None, pip_proxy=None): + ext_sha256 = None + if extension_name: + if extension_exists(extension_name): + logger.warning("The extension '%s' already exists.", extension_name) + return + try: + source, ext_sha256 = resolve_from_index(extension_name, index_url=index_url) + except NoExtensionCandidatesError as err: + logger.debug(err) + raise CLIError("No matching extensions for '{}'. Use --debug for more information.".format(extension_name)) + _add_whl_ext(source, ext_sha256=ext_sha256, pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy) + _augment_telemetry_with_ext_info(extension_name) + try: + if extension_name and get_extension(extension_name).preview: + logger.warning("The installed extension '%s' is in preview.", extension_name) + except ExtensionNotInstalledException: + pass + + +def remove_extension(extension_name): + def log_err(func, path, exc_info): + logger.debug("Error occurred attempting to delete item from the extension '%s'.", extension_name) + logger.debug("%s: %s - %s", func, path, exc_info) + + try: + # Get the extension and it will raise an error if it doesn't exist + get_extension(extension_name) + # We call this just before we remove the extension so we can get the metadata before it is gone + _augment_telemetry_with_ext_info(extension_name) + shutil.rmtree(get_extension_path(extension_name), onerror=log_err) + except ExtensionNotInstalledException as e: + raise CLIError(e) + + +def list_extensions(): + return [{OUT_KEY_NAME: ext.name, OUT_KEY_VERSION: ext.version, OUT_KEY_TYPE: ext.ext_type} + for ext in get_extensions()] + + +def show_extension(extension_name): + try: + extension = get_extension(extension_name) + return {OUT_KEY_NAME: extension.name, + OUT_KEY_VERSION: extension.version, + OUT_KEY_TYPE: extension.ext_type, + OUT_KEY_METADATA: extension.metadata} + except ExtensionNotInstalledException as e: + raise CLIError(e) + + +def update_extension(extension_name, index_url=None, pip_extra_index_urls=None, pip_proxy=None): + try: + ext = get_extension(extension_name) + cur_version = ext.get_version() + try: + download_url, ext_sha256 = resolve_from_index(extension_name, cur_version=cur_version, index_url=index_url) + except NoExtensionCandidatesError as err: + logger.debug(err) + raise CLIError("No updates available for '{}'. Use --debug for more information.".format(extension_name)) + # Copy current version of extension to tmp directory in case we need to restore it after a failed install. + backup_dir = os.path.join(tempfile.mkdtemp(), extension_name) + extension_path = get_extension_path(extension_name) + logger.debug('Backing up the current extension: %s to %s', extension_path, backup_dir) + shutil.copytree(extension_path, backup_dir) + # Remove current version of the extension + shutil.rmtree(extension_path) + # Install newer version + try: + _add_whl_ext(download_url, ext_sha256=ext_sha256, + pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy) + logger.debug('Deleting backup of old extension at %s', backup_dir) + shutil.rmtree(backup_dir) + # This gets the metadata for the extension *after* the update + _augment_telemetry_with_ext_info(extension_name) + except Exception as err: + logger.error('An error occurred whilst updating.') + logger.error(err) + logger.debug('Copying %s to %s', backup_dir, extension_path) + shutil.copytree(backup_dir, extension_path) + raise CLIError('Failed to update. Rolled {} back to {}.'.format(extension_name, cur_version)) + except ExtensionNotInstalledException as e: + raise CLIError(e) + + +def list_available_extensions(index_url=None, show_details=False): + index_data = get_index_extensions(index_url=index_url) + if show_details: + return index_data + installed_extensions = get_extensions() + installed_extension_names = [e.name for e in installed_extensions] + results = [] + for name, items in OrderedDict(sorted(index_data.items())).items(): + # exclude extensions/versions incompatible with current CLI version + items = [item for item in items if ext_compat_with_cli(item['metadata'])[0]] + if not items: + continue + + latest = max(items, key=lambda c: parse_version(c['metadata']['version'])) + installed = False + if name in installed_extension_names: + installed = True + ext_version = get_extension(name).version + if ext_version and parse_version(latest['metadata']['version']) > parse_version(ext_version): + installed = str(True) + ' (upgrade available)' + results.append({ + 'name': name, + 'version': latest['metadata']['version'], + 'summary': latest['metadata']['summary'], + 'preview': latest['metadata'].get(EXT_METADATA_ISPREVIEW, False), + 'installed': installed + }) + return results + + +def get_lsb_release(): + try: + with open(LSB_RELEASE_FILE, 'r') as lr: + lsb = lr.readlines() + desc = lsb[2] + desc_split = desc.split('=') + rel = desc_split[1] + return rel.strip() + except Exception: # pylint: disable=broad-except + return None + + +def check_distro_consistency(): + if IS_WINDOWS: + return + + try: + logger.debug('Linux distro check: Reading from: %s', LIST_FILE_PATH) + + with open(LIST_FILE_PATH, 'r') as list_file: + package_source = list_file.read() + stored_linux_dist_name = package_source.split(" ")[3] + logger.debug('Linux distro check: Found in list file: %s', stored_linux_dist_name) + current_linux_dist_name = get_lsb_release() + logger.debug('Linux distro check: Reported by API: %s', current_linux_dist_name) + + except Exception as err: # pylint: disable=broad-except + current_linux_dist_name = None + stored_linux_dist_name = None + logger.debug('Linux distro check: An error occurred while checking ' + 'linux distribution version source list consistency.') + logger.debug(err) + + if current_linux_dist_name != stored_linux_dist_name: + logger.debug("Linux distro check: Mismatch distribution " + "name in %s file", LIST_FILE_PATH) + logger.debug("Linux distro check: If command fails, install the appropriate package " + "for your distribution or change the above file accordingly.") + logger.debug("Linux distro check: %s has '%s', current distro is '%s'", + LIST_FILE_PATH, stored_linux_dist_name, current_linux_dist_name) diff --git a/src/azure-cli-core/azure/cli/core/extensions/tests/__init__.py b/src/azure-cli-core/azure/cli/core/extensions/tests/__init__.py new file mode 100644 index 00000000000..34913fb394d --- /dev/null +++ b/src/azure-cli-core/azure/cli/core/extensions/tests/__init__.py @@ -0,0 +1,4 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- diff --git a/src/azure-cli-core/azure/cli/core/extensions/tests/latest/__init__.py b/src/azure-cli-core/azure/cli/core/extensions/tests/latest/__init__.py new file mode 100644 index 00000000000..34913fb394d --- /dev/null +++ b/src/azure-cli-core/azure/cli/core/extensions/tests/latest/__init__.py @@ -0,0 +1,4 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- diff --git a/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/tests/latest/data/my_second_cli_extension-0.0.1+dev-py2.py3-none-any.whl b/src/azure-cli-core/azure/cli/core/extensions/tests/latest/data/my_second_cli_extension-0.0.1+dev-py2.py3-none-any.whl similarity index 100% rename from src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/tests/latest/data/my_second_cli_extension-0.0.1+dev-py2.py3-none-any.whl rename to src/azure-cli-core/azure/cli/core/extensions/tests/latest/data/my_second_cli_extension-0.0.1+dev-py2.py3-none-any.whl diff --git a/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/tests/latest/data/myfirstcliextension-0.0.3+dev-py2.py3-none-any.whl b/src/azure-cli-core/azure/cli/core/extensions/tests/latest/data/myfirstcliextension-0.0.3+dev-py2.py3-none-any.whl similarity index 100% rename from src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/tests/latest/data/myfirstcliextension-0.0.3+dev-py2.py3-none-any.whl rename to src/azure-cli-core/azure/cli/core/extensions/tests/latest/data/myfirstcliextension-0.0.3+dev-py2.py3-none-any.whl diff --git a/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/tests/latest/data/myfirstcliextension-0.0.4+dev-py2.py3-none-any.whl b/src/azure-cli-core/azure/cli/core/extensions/tests/latest/data/myfirstcliextension-0.0.4+dev-py2.py3-none-any.whl similarity index 100% rename from src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/tests/latest/data/myfirstcliextension-0.0.4+dev-py2.py3-none-any.whl rename to src/azure-cli-core/azure/cli/core/extensions/tests/latest/data/myfirstcliextension-0.0.4+dev-py2.py3-none-any.whl diff --git a/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/tests/latest/data/notanextension.txt b/src/azure-cli-core/azure/cli/core/extensions/tests/latest/data/notanextension.txt similarity index 100% rename from src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/tests/latest/data/notanextension.txt rename to src/azure-cli-core/azure/cli/core/extensions/tests/latest/data/notanextension.txt diff --git a/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/tests/latest/test_extension_commands.py b/src/azure-cli-core/azure/cli/core/extensions/tests/latest/test_extension_commands.py similarity index 75% rename from src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/tests/latest/test_extension_commands.py rename to src/azure-cli-core/azure/cli/core/extensions/tests/latest/test_extension_commands.py index 5fed2931f71..b7ad2a3c848 100644 --- a/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/tests/latest/test_extension_commands.py +++ b/src/azure-cli-core/azure/cli/core/extensions/tests/latest/test_extension_commands.py @@ -10,10 +10,10 @@ import mock from azure.cli.core.util import CLIError -from azure.cli.command_modules.extension.custom import (list_extensions, add_extension, show_extension, - remove_extension, update_extension, - list_available_extensions, OUT_KEY_NAME, OUT_KEY_VERSION, OUT_KEY_METADATA) -from azure.cli.command_modules.extension._resolve import NoExtensionCandidatesError +from azure.cli.core.extensions.operations import (list_extensions, add_extension, show_extension, + remove_extension, update_extension, + list_available_extensions, OUT_KEY_NAME, OUT_KEY_VERSION, OUT_KEY_METADATA) +from azure.cli.core.extensions._resolve import NoExtensionCandidatesError def _get_test_data_file(filename): @@ -38,7 +38,7 @@ class TestExtensionCommands(unittest.TestCase): def setUp(self): self.ext_dir = tempfile.mkdtemp() - self.patcher = mock.patch('azure.cli.core.extension.EXTENSIONS_DIR', self.ext_dir) + self.patcher = mock.patch('azure.cli.core.extensions.EXTENSIONS_DIR', self.ext_dir) self.patcher.start() def tearDown(self): @@ -106,9 +106,9 @@ def test_add_extension_with_pip_proxy(self): proxy_param = '--proxy' proxy_endpoint = "https://user:pass@proxy.microsoft.com" computed_extension_sha256 = _compute_file_hash(MY_EXT_SOURCE) - with mock.patch('azure.cli.command_modules.extension.custom.resolve_from_index', return_value=(MY_EXT_SOURCE, computed_extension_sha256)), \ - mock.patch('azure.cli.command_modules.extension.custom.shutil'), \ - mock.patch('azure.cli.command_modules.extension.custom.check_output') as check_output: + with mock.patch('azure.cli.core.extensions.operations.resolve_from_index', return_value=(MY_EXT_SOURCE, computed_extension_sha256)), \ + mock.patch('azure.cli.core.extensions.operations.shutil'), \ + mock.patch('azure.cli.core.extensions.operations.check_output') as check_output: add_extension(extension_name=extension_name, pip_proxy=proxy_endpoint) args = check_output.call_args pip_cmd = args[0][0] @@ -118,9 +118,9 @@ def test_add_extension_with_pip_proxy(self): def test_add_extension_verify_no_pip_proxy(self): extension_name = MY_EXT_NAME computed_extension_sha256 = _compute_file_hash(MY_EXT_SOURCE) - with mock.patch('azure.cli.command_modules.extension.custom.resolve_from_index', return_value=(MY_EXT_SOURCE, computed_extension_sha256)), \ - mock.patch('azure.cli.command_modules.extension.custom.shutil'), \ - mock.patch('azure.cli.command_modules.extension.custom.check_output') as check_output: + with mock.patch('azure.cli.core.extensions.operations.resolve_from_index', return_value=(MY_EXT_SOURCE, computed_extension_sha256)), \ + mock.patch('azure.cli.core.extensions.operations.shutil'), \ + mock.patch('azure.cli.core.extensions.operations.check_output') as check_output: add_extension(extension_name=extension_name) args = check_output.call_args pip_cmd = args[0][0] @@ -130,7 +130,7 @@ def test_add_extension_verify_no_pip_proxy(self): def test_add_extension_with_name_valid_checksum(self): extension_name = MY_EXT_NAME computed_extension_sha256 = _compute_file_hash(MY_EXT_SOURCE) - with mock.patch('azure.cli.command_modules.extension.custom.resolve_from_index', return_value=(MY_EXT_SOURCE, computed_extension_sha256)): + with mock.patch('azure.cli.core.extensions.operations.resolve_from_index', return_value=(MY_EXT_SOURCE, computed_extension_sha256)): add_extension(extension_name=extension_name) ext = show_extension(MY_EXT_NAME) self.assertEqual(ext[OUT_KEY_NAME], MY_EXT_NAME) @@ -138,14 +138,14 @@ def test_add_extension_with_name_valid_checksum(self): def test_add_extension_with_name_invalid_checksum(self): extension_name = MY_EXT_NAME bad_sha256 = 'thishashisclearlywrong' - with mock.patch('azure.cli.command_modules.extension.custom.resolve_from_index', return_value=(MY_EXT_SOURCE, bad_sha256)): + with mock.patch('azure.cli.core.extensions.operations.resolve_from_index', return_value=(MY_EXT_SOURCE, bad_sha256)): with self.assertRaises(CLIError) as err: add_extension(extension_name=extension_name) self.assertTrue('The checksum of the extension does not match the expected value.' in str(err.exception)) def test_add_extension_with_name_source_not_whl(self): extension_name = 'myextension' - with mock.patch('azure.cli.command_modules.extension.custom.resolve_from_index', return_value=('{}.notwhl'.format(extension_name), None)): + with mock.patch('azure.cli.core.extensions.operations.resolve_from_index', return_value=('{}.notwhl'.format(extension_name), None)): with self.assertRaises(ValueError) as err: add_extension(extension_name=extension_name) self.assertTrue('Unknown extension type. Only Python wheels are supported.' in str(err.exception)) @@ -157,8 +157,8 @@ def test_add_extension_with_name_but_it_already_exists(self): self.assertEqual(ext[OUT_KEY_NAME], MY_EXT_NAME) # Now add using name computed_extension_sha256 = _compute_file_hash(MY_EXT_SOURCE) - with mock.patch('azure.cli.command_modules.extension.custom.resolve_from_index', return_value=(MY_EXT_SOURCE, computed_extension_sha256)): - with mock.patch('azure.cli.command_modules.extension.custom.logger') as mock_logger: + with mock.patch('azure.cli.core.extensions.operations.resolve_from_index', return_value=(MY_EXT_SOURCE, computed_extension_sha256)): + with mock.patch('azure.cli.core.extensions.operations.logger') as mock_logger: add_extension(extension_name=MY_EXT_NAME) call_args = mock_logger.warning.call_args self.assertEqual("The extension '%s' already exists.", call_args[0][0]) @@ -171,7 +171,7 @@ def test_update_extension(self): self.assertEqual(ext[OUT_KEY_VERSION], '0.0.3+dev') newer_extension = _get_test_data_file('myfirstcliextension-0.0.4+dev-py2.py3-none-any.whl') computed_extension_sha256 = _compute_file_hash(newer_extension) - with mock.patch('azure.cli.command_modules.extension.custom.resolve_from_index', return_value=(newer_extension, computed_extension_sha256)): + with mock.patch('azure.cli.core.extensions.operations.resolve_from_index', return_value=(newer_extension, computed_extension_sha256)): update_extension(MY_EXT_NAME) ext = show_extension(MY_EXT_NAME) self.assertEqual(ext[OUT_KEY_VERSION], '0.0.4+dev') @@ -185,11 +185,11 @@ def test_update_extension_with_pip_proxy(self): proxy_param = '--proxy' proxy_endpoint = "https://user:pass@proxy.microsoft.com" - with mock.patch('azure.cli.command_modules.extension.custom.resolve_from_index', return_value=(MY_EXT_SOURCE, computed_extension_sha256)), \ - mock.patch('azure.cli.command_modules.extension.custom.shutil'), \ - mock.patch('azure.cli.command_modules.extension.custom.is_valid_sha256sum', return_value=(True, computed_extension_sha256)), \ - mock.patch('azure.cli.command_modules.extension.custom.extension_exists', return_value=None), \ - mock.patch('azure.cli.command_modules.extension.custom.check_output') as check_output: + with mock.patch('azure.cli.core.extensions.operations.resolve_from_index', return_value=(MY_EXT_SOURCE, computed_extension_sha256)), \ + mock.patch('azure.cli.core.extensions.operations.shutil'), \ + mock.patch('azure.cli.core.extensions.operations.is_valid_sha256sum', return_value=(True, computed_extension_sha256)), \ + mock.patch('azure.cli.core.extensions.operations.extension_exists', return_value=None), \ + mock.patch('azure.cli.core.extensions.operations.check_output') as check_output: update_extension(MY_EXT_NAME, pip_proxy=proxy_endpoint) args = check_output.call_args @@ -204,11 +204,11 @@ def test_update_extension_verify_no_pip_proxy(self): newer_extension = _get_test_data_file('myfirstcliextension-0.0.4+dev-py2.py3-none-any.whl') computed_extension_sha256 = _compute_file_hash(newer_extension) - with mock.patch('azure.cli.command_modules.extension.custom.resolve_from_index', return_value=(MY_EXT_SOURCE, computed_extension_sha256)), \ - mock.patch('azure.cli.command_modules.extension.custom.shutil'), \ - mock.patch('azure.cli.command_modules.extension.custom.is_valid_sha256sum', return_value=(True, computed_extension_sha256)), \ - mock.patch('azure.cli.command_modules.extension.custom.extension_exists', return_value=None), \ - mock.patch('azure.cli.command_modules.extension.custom.check_output') as check_output: + with mock.patch('azure.cli.core.extensions.operations.resolve_from_index', return_value=(MY_EXT_SOURCE, computed_extension_sha256)), \ + mock.patch('azure.cli.core.extensions.operations.shutil'), \ + mock.patch('azure.cli.core.extensions.operations.is_valid_sha256sum', return_value=(True, computed_extension_sha256)), \ + mock.patch('azure.cli.core.extensions.operations.extension_exists', return_value=None), \ + mock.patch('azure.cli.core.extensions.operations.check_output') as check_output: update_extension(MY_EXT_NAME) args = check_output.call_args @@ -225,7 +225,7 @@ def test_update_extension_no_updates(self): add_extension(source=MY_EXT_SOURCE) ext = show_extension(MY_EXT_NAME) self.assertEqual(ext[OUT_KEY_VERSION], '0.0.3+dev') - with mock.patch('azure.cli.command_modules.extension.custom.resolve_from_index', side_effect=NoExtensionCandidatesError()): + with mock.patch('azure.cli.core.extensions.operations.resolve_from_index', side_effect=NoExtensionCandidatesError()): with self.assertRaises(CLIError) as err: update_extension(MY_EXT_NAME) self.assertTrue("No updates available for '{}'.".format(MY_EXT_NAME) in str(err.exception)) @@ -236,7 +236,7 @@ def test_update_extension_exception_in_update_and_rolled_back(self): self.assertEqual(ext[OUT_KEY_VERSION], '0.0.3+dev') newer_extension = _get_test_data_file('myfirstcliextension-0.0.4+dev-py2.py3-none-any.whl') bad_sha256 = 'thishashisclearlywrong' - with mock.patch('azure.cli.command_modules.extension.custom.resolve_from_index', return_value=(newer_extension, bad_sha256)): + with mock.patch('azure.cli.core.extensions.operations.resolve_from_index', return_value=(newer_extension, bad_sha256)): with self.assertRaises(CLIError) as err: update_extension(MY_EXT_NAME) self.assertTrue('Failed to update. Rolled {} back to {}.'.format(ext['name'], ext[OUT_KEY_VERSION]) in str(err.exception)) @@ -244,18 +244,18 @@ def test_update_extension_exception_in_update_and_rolled_back(self): self.assertEqual(ext[OUT_KEY_VERSION], '0.0.3+dev') def test_list_available_extensions_default(self): - with mock.patch('azure.cli.command_modules.extension.custom.get_index_extensions', autospec=True) as c: + with mock.patch('azure.cli.core.extensions.operations.get_index_extensions', autospec=True) as c: list_available_extensions() c.assert_called_once_with(None) - def test_list_available_extensions_custom_index_url(self): - with mock.patch('azure.cli.command_modules.extension.custom.get_index_extensions', autospec=True) as c: + def test_list_available_extensions_operations_index_url(self): + with mock.patch('azure.cli.core.extensions.operations.get_index_extensions', autospec=True) as c: index_url = 'http://contoso.com' list_available_extensions(index_url=index_url) c.assert_called_once_with(index_url) def test_list_available_extensions_show_details(self): - with mock.patch('azure.cli.command_modules.extension.custom.get_index_extensions', autospec=True) as c: + with mock.patch('azure.cli.core.extensions.operations.get_index_extensions', autospec=True) as c: list_available_extensions(show_details=True) c.assert_called_once_with(None) @@ -268,7 +268,7 @@ def test_list_available_extensions_no_show_details(self): 'version': '0.1.0' }}] } - with mock.patch('azure.cli.command_modules.extension.custom.get_index_extensions', return_value=sample_index_extensions): + with mock.patch('azure.cli.core.extensions.operations.get_index_extensions', return_value=sample_index_extensions): res = list_available_extensions() self.assertIsInstance(res, list) self.assertEqual(len(res), len(sample_index_extensions)) @@ -287,7 +287,7 @@ def test_list_available_extensions_incompatible_cli_version(self): 'version': '0.1.0' }}] } - with mock.patch('azure.cli.command_modules.extension.custom.get_index_extensions', return_value=sample_index_extensions): + with mock.patch('azure.cli.core.extensions.operations.get_index_extensions', return_value=sample_index_extensions): res = list_available_extensions() self.assertIsInstance(res, list) self.assertEqual(len(res), 0) @@ -320,7 +320,7 @@ def test_update_extension_extra_index_url(self): self.assertEqual(ext[OUT_KEY_VERSION], '0.0.3+dev') newer_extension = _get_test_data_file('myfirstcliextension-0.0.4+dev-py2.py3-none-any.whl') computed_extension_sha256 = _compute_file_hash(newer_extension) - with mock.patch('azure.cli.command_modules.extension.custom.resolve_from_index', return_value=(newer_extension, computed_extension_sha256)): + with mock.patch('azure.cli.core.extensions.operations.resolve_from_index', return_value=(newer_extension, computed_extension_sha256)): update_extension(MY_EXT_NAME, pip_extra_index_urls=extra_index_urls) ext = show_extension(MY_EXT_NAME) self.assertEqual(ext[OUT_KEY_VERSION], '0.0.4+dev') diff --git a/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/tests/latest/test_index_get.py b/src/azure-cli-core/azure/cli/core/extensions/tests/latest/test_index_get.py similarity index 87% rename from src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/tests/latest/test_index_get.py rename to src/azure-cli-core/azure/cli/core/extensions/tests/latest/test_index_get.py index 645d1691ede..50e864ab10f 100644 --- a/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/tests/latest/test_index_get.py +++ b/src/azure-cli-core/azure/cli/core/extensions/tests/latest/test_index_get.py @@ -7,9 +7,9 @@ import unittest from requests.exceptions import ConnectionError, HTTPError from azure.cli.core.util import CLIError -from azure.cli.command_modules.extension._index import (get_index, get_index_extensions, DEFAULT_INDEX_URL, - ERR_TMPL_NON_200, ERR_TMPL_NO_NETWORK, ERR_TMPL_BAD_JSON, - ERR_UNABLE_TO_GET_EXTENSIONS) +from azure.cli.core.extensions._index import (get_index, get_index_extensions, DEFAULT_INDEX_URL, + ERR_TMPL_NON_200, ERR_TMPL_NO_NETWORK, ERR_TMPL_BAD_JSON, + ERR_UNABLE_TO_GET_EXTENSIONS) class MockResponse(object): @@ -75,12 +75,12 @@ def test_get_index_extensions(self): with mock.patch('requests.get', side_effect=mock_index_get_generator(DEFAULT_INDEX_URL, data)): self.assertEqual(get_index_extensions().get('myext'), obj) - with mock.patch('azure.cli.command_modules.extension._index.logger.warning', autospec=True) as logger_mock: + with mock.patch('azure.cli.core.extensions._index.logger.warning', autospec=True) as logger_mock: with mock.patch('requests.get', side_effect=mock_index_get_generator(DEFAULT_INDEX_URL, {})): self.assertEqual(get_index_extensions(), None) logger_mock.assert_called_once_with(ERR_UNABLE_TO_GET_EXTENSIONS) - with mock.patch('azure.cli.command_modules.extension._index.logger.warning', autospec=True) as logger_mock: + with mock.patch('azure.cli.core.extensions._index.logger.warning', autospec=True) as logger_mock: with mock.patch('requests.get', side_effect=mock_index_get_generator(DEFAULT_INDEX_URL, {'v2extensions': []})): self.assertEqual(get_index_extensions(), None) logger_mock.assert_called_once_with(ERR_UNABLE_TO_GET_EXTENSIONS) diff --git a/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/tests/latest/test_resolve.py b/src/azure-cli-core/azure/cli/core/extensions/tests/latest/test_resolve.py similarity index 93% rename from src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/tests/latest/test_resolve.py rename to src/azure-cli-core/azure/cli/core/extensions/tests/latest/test_resolve.py index 38cb83b929e..1ffc35e50ee 100644 --- a/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/tests/latest/test_resolve.py +++ b/src/azure-cli-core/azure/cli/core/extensions/tests/latest/test_resolve.py @@ -5,13 +5,13 @@ import unittest import mock -from azure.cli.command_modules.extension._resolve import (resolve_from_index, NoExtensionCandidatesError, - _is_not_platform_specific, _is_greater_than_cur_version) +from azure.cli.core.extensions._resolve import (resolve_from_index, NoExtensionCandidatesError, + _is_not_platform_specific, _is_greater_than_cur_version) class IndexPatch(object): def __init__(self, data=None): - self.patcher = mock.patch('azure.cli.command_modules.extension._resolve.get_index_extensions', return_value=data) + self.patcher = mock.patch('azure.cli.core.extensions._resolve.get_index_extensions', return_value=data) def __enter__(self): self.patcher.start() diff --git a/src/azure-cli-core/azure/cli/core/parser.py b/src/azure-cli-core/azure/cli/core/parser.py index 8f606c7998c..61858d4ccf3 100644 --- a/src/azure-cli-core/azure/cli/core/parser.py +++ b/src/azure-cli-core/azure/cli/core/parser.py @@ -17,7 +17,7 @@ from knack.util import CLIError import azure.cli.core.telemetry as telemetry -from azure.cli.core.extension import get_extension +from azure.cli.core.extensions import get_extension from azure.cli.core.commands import ExtensionCommandSource from azure.cli.core.commands.events import EVENT_INVOKER_ON_TAB_COMPLETION diff --git a/src/azure-cli-core/azure/cli/core/tests/test_add_resourcegroup_transform.py b/src/azure-cli-core/azure/cli/core/tests/test_add_resourcegroup_transform.py index 6b8cd34c250..453bb49df6e 100644 --- a/src/azure-cli-core/azure/cli/core/tests/test_add_resourcegroup_transform.py +++ b/src/azure-cli-core/azure/cli/core/tests/test_add_resourcegroup_transform.py @@ -5,7 +5,7 @@ import unittest from six import StringIO -from azure.cli.core.extensions.transform import _parse_id, _add_resource_group +from azure.cli.core.commands.transform import _parse_id, _add_resource_group class TestResourceGroupTransform(unittest.TestCase): diff --git a/src/azure-cli-core/azure/cli/core/tests/test_command_registration.py b/src/azure-cli-core/azure/cli/core/tests/test_command_registration.py index e7395d205cd..d261c0305b5 100644 --- a/src/azure-cli-core/azure/cli/core/tests/test_command_registration.py +++ b/src/azure-cli-core/azure/cli/core/tests/test_command_registration.py @@ -11,7 +11,7 @@ from azure.cli.core import AzCommandsLoader, MainCommandsLoader from azure.cli.core.commands import ExtensionCommandSource -from azure.cli.core.extension import EXTENSIONS_MOD_PREFIX +from azure.cli.core.extensions import EXTENSIONS_MOD_PREFIX from azure.cli.core.mock import DummyCli from knack.arguments import CLICommandArgument, CLIArgumentType @@ -201,8 +201,8 @@ def load_command_table(self, args): @mock.patch('importlib.import_module', _mock_import_lib) @mock.patch('pkgutil.iter_modules', _mock_iter_modules) @mock.patch('azure.cli.core.commands._load_command_loader', _mock_load_command_loader) - @mock.patch('azure.cli.core.extension.get_extension_modname', _mock_extension_modname) - @mock.patch('azure.cli.core.extension.get_extensions', _mock_get_extensions) + @mock.patch('azure.cli.core.extensions.get_extension_modname', _mock_extension_modname) + @mock.patch('azure.cli.core.extensions.get_extensions', _mock_get_extensions) def test_register_command_from_extension(self): from azure.cli.core.commands import _load_command_loader diff --git a/src/azure-cli-core/azure/cli/core/tests/test_extension.py b/src/azure-cli-core/azure/cli/core/tests/test_extension.py index aa7e730e574..49caf1d2a1c 100644 --- a/src/azure-cli-core/azure/cli/core/tests/test_extension.py +++ b/src/azure-cli-core/azure/cli/core/tests/test_extension.py @@ -10,10 +10,10 @@ import mock -from azure.cli.core.extension import (get_extensions, get_extension_path, extension_exists, - get_extension, get_extension_names, get_extension_modname, ext_compat_with_cli, - ExtensionNotInstalledException, WheelExtension, - EXTENSIONS_MOD_PREFIX, EXT_METADATA_MINCLICOREVERSION, EXT_METADATA_MAXCLICOREVERSION) +from azure.cli.core.extensions import (get_extensions, get_extension_path, extension_exists, + get_extension, get_extension_names, get_extension_modname, ext_compat_with_cli, + ExtensionNotInstalledException, WheelExtension, + EXTENSIONS_MOD_PREFIX, EXT_METADATA_MINCLICOREVERSION, EXT_METADATA_MAXCLICOREVERSION) # The test extension name @@ -45,7 +45,7 @@ class TestExtensionsBase(unittest.TestCase): def setUp(self): self.ext_dir = tempfile.mkdtemp() - self.patcher = mock.patch('azure.cli.core.extension.EXTENSIONS_DIR', self.ext_dir) + self.patcher = mock.patch('azure.cli.core.extensions.EXTENSIONS_DIR', self.ext_dir) self.patcher.start() def tearDown(self): diff --git a/src/azure-cli-core/azure/cli/core/tests/test_parser.py b/src/azure-cli-core/azure/cli/core/tests/test_parser.py index 7b95890fbba..7a2ede53fd5 100644 --- a/src/azure-cli-core/azure/cli/core/tests/test_parser.py +++ b/src/azure-cli-core/azure/cli/core/tests/test_parser.py @@ -193,8 +193,8 @@ def load_command_table(self, args): @mock.patch('importlib.import_module', _mock_import_lib) @mock.patch('pkgutil.iter_modules', _mock_iter_modules) @mock.patch('azure.cli.core.commands._load_command_loader', _mock_load_command_loader) - @mock.patch('azure.cli.core.extension.get_extension_modname', _mock_extension_modname) - @mock.patch('azure.cli.core.extension.get_extensions', _mock_get_extensions) + @mock.patch('azure.cli.core.extensions.get_extension_modname', _mock_extension_modname) + @mock.patch('azure.cli.core.extensions.get_extensions', _mock_get_extensions) def test_parser_error_spellchecker(self): cli = DummyCli() main_loader = MainCommandsLoader(cli) diff --git a/src/azure-cli-core/azure/cli/core/util.py b/src/azure-cli-core/azure/cli/core/util.py index 020bc0e6972..05c1ffa3fb0 100644 --- a/src/azure-cli-core/azure/cli/core/util.py +++ b/src/azure-cli-core/azure/cli/core/util.py @@ -72,7 +72,7 @@ def get_installed_cli_distributions(): def get_az_version_string(): import platform - from azure.cli.core.extension import get_extensions, EXTENSIONS_DIR + from azure.cli.core.extensions import get_extensions, EXTENSIONS_DIR output = six.StringIO() installed_dists = get_installed_cli_distributions() diff --git a/src/command_modules/azure-cli-acs/HISTORY.rst b/src/command_modules/azure-cli-acs/HISTORY.rst index d186044bdde..1202c145ac2 100644 --- a/src/command_modules/azure-cli-acs/HISTORY.rst +++ b/src/command_modules/azure-cli-acs/HISTORY.rst @@ -2,6 +2,9 @@ Release History =============== +2.3.10 +++++++ +* Minor fixes 2.3.9 +++++ diff --git a/src/command_modules/azure-cli-acs/azure/cli/command_modules/acs/custom.py b/src/command_modules/azure-cli-acs/azure/cli/command_modules/acs/custom.py index d72c7ceaf2d..f7e177f8016 100644 --- a/src/command_modules/azure-cli-acs/azure/cli/command_modules/acs/custom.py +++ b/src/command_modules/azure-cli-acs/azure/cli/command_modules/acs/custom.py @@ -1739,7 +1739,7 @@ def _update_addons(cmd, instance, subscription_id, resource_group_name, addons, def _get_azext_module(extension_name, module_name): try: # Adding the installed extension in the path - from azure.cli.core.extension import get_extension_path + from azure.cli.core.extensions import get_extension_path ext_dir = get_extension_path(extension_name) sys.path.append(ext_dir) # Import the extension module @@ -1785,18 +1785,18 @@ def _handle_addons_args(cmd, addons_str, subscription_id, resource_group_name, a def _install_dev_spaces_extension(extension_name): try: - from azure.cli.command_modules.extension import custom - custom.add_extension(extension_name=extension_name) + from azure.cli.core.extensions import operations + operations.add_extension(extension_name=extension_name) except Exception: # nopa pylint: disable=broad-except return False return True def _update_dev_spaces_extension(extension_name, extension_module): - from azure.cli.core.extension import ExtensionNotInstalledException + from azure.cli.core.extensions import ExtensionNotInstalledException try: - from azure.cli.command_modules.extension import custom - custom.update_extension(extension_name=extension_name) + from azure.cli.core.extensions import operations + operations.update_extension(extension_name=extension_name) # reloading the imported module to update try: @@ -1817,7 +1817,7 @@ def _update_dev_spaces_extension(extension_name, extension_module): def _get_or_add_extension(extension_name, extension_module, update=False): - from azure.cli.core.extension import (ExtensionNotInstalledException, get_extension) + from azure.cli.core.extensions import (ExtensionNotInstalledException, get_extension) try: get_extension(extension_name) if update: diff --git a/src/command_modules/azure-cli-acs/setup.py b/src/command_modules/azure-cli-acs/setup.py index 85e6f3098cb..3aaa6ed8eaa 100644 --- a/src/command_modules/azure-cli-acs/setup.py +++ b/src/command_modules/azure-cli-acs/setup.py @@ -14,7 +14,7 @@ logger.warn("Wheel is not available, disabling bdist_wheel hook") cmdclass = {} -VERSION = "2.3.9" +VERSION = "2.3.10" CLASSIFIERS = [ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', diff --git a/src/command_modules/azure-cli-extension/HISTORY.rst b/src/command_modules/azure-cli-extension/HISTORY.rst index 07630ea1c5c..b25f0b3cec5 100644 --- a/src/command_modules/azure-cli-extension/HISTORY.rst +++ b/src/command_modules/azure-cli-extension/HISTORY.rst @@ -3,6 +3,10 @@ Release History =============== +0.2.3 ++++++ +* Minor fixes. + 0.2.2 +++++ * Attempting to add an extension that is already installed will not raise an exception. diff --git a/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/_completers.py b/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/_completers.py index cddbc5ecab5..b39d8907ed3 100644 --- a/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/_completers.py +++ b/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/_completers.py @@ -4,9 +4,9 @@ # -------------------------------------------------------------------------------------------- from azure.cli.core.decorators import Completer -from azure.cli.core.extension import get_extension_names +from azure.cli.core.extensions import get_extension_names -from azure.cli.command_modules.extension.custom import get_index_extensions +from azure.cli.core.extensions.operations import get_index_extensions @Completer diff --git a/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/custom.py b/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/custom.py index bf7de487fad..f37c6ab5b9e 100644 --- a/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/custom.py +++ b/src/command_modules/azure-cli-extension/azure/cli/command_modules/extension/custom.py @@ -2,347 +2,37 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -import sys -import os -import tempfile -import shutil -import zipfile -import traceback -import hashlib -from subprocess import check_output, STDOUT, CalledProcessError -from six.moves.urllib.parse import urlparse # pylint: disable=import-error -from collections import OrderedDict - -import requests -from wheel.install import WHEEL_INFO_RE -from pkg_resources import parse_version - from knack.log import get_logger -from azure.cli.core.util import CLIError -from azure.cli.core.extension import (extension_exists, get_extension_path, get_extensions, - get_extension, ext_compat_with_cli, EXT_METADATA_ISPREVIEW, - WheelExtension, ExtensionNotInstalledException) -from azure.cli.core.telemetry import set_extension_management_detail - -from ._homebrew_patch import HomebrewPipPatch -from ._index import get_index_extensions -from ._resolve import resolve_from_index, NoExtensionCandidatesError +from azure.cli.core.extensions.operations import ( + add_extension, remove_extension, list_extensions, show_extension, + list_available_extensions, update_extension) logger = get_logger(__name__) -OUT_KEY_NAME = 'name' -OUT_KEY_VERSION = 'version' -OUT_KEY_TYPE = 'extensionType' -OUT_KEY_METADATA = 'metadata' - -IS_WINDOWS = sys.platform.lower() in ['windows', 'win32'] -LIST_FILE_PATH = os.path.join(os.sep, 'etc', 'apt', 'sources.list.d', 'azure-cli.list') -LSB_RELEASE_FILE = os.path.join(os.sep, 'etc', 'lsb-release') - - -def _run_pip(pip_exec_args): - cmd = [sys.executable, '-m', 'pip'] + pip_exec_args + ['-vv', '--disable-pip-version-check', '--no-cache-dir'] - logger.debug('Running: %s', cmd) - try: - log_output = check_output(cmd, stderr=STDOUT, universal_newlines=True) - logger.debug(log_output) - returncode = 0 - except CalledProcessError as e: - logger.debug(e.output) - logger.debug(e) - returncode = e.returncode - return returncode - - -def _whl_download_from_url(url_parse_result, ext_file): - from azure.cli.core.util import should_disable_connection_verify - url = url_parse_result.geturl() - r = requests.get(url, stream=True, verify=(not should_disable_connection_verify())) - if r.status_code != 200: - raise CLIError("Request to {} failed with {}".format(url, r.status_code)) - with open(ext_file, 'wb') as f: - for chunk in r.iter_content(chunk_size=1024): - if chunk: # ignore keep-alive new chunks - f.write(chunk) - - -def _validate_whl_cli_compat(azext_metadata): - is_compatible, cli_core_version, min_required, max_required = ext_compat_with_cli(azext_metadata) - logger.debug("Extension compatibility result: is_compatible=%s cli_core_version=%s min_required=%s " - "max_required=%s", is_compatible, cli_core_version, min_required, max_required) - if not is_compatible: - min_max_msg_fmt = "The extension is not compatible with this version of the CLI.\n" \ - "You have CLI core version {} and this extension " \ - "requires ".format(cli_core_version) - if min_required and max_required: - min_max_msg_fmt += 'a min of {} and max of {}.'.format(min_required, max_required) - elif min_required: - min_max_msg_fmt += 'a min of {}.'.format(min_required) - elif max_required: - min_max_msg_fmt += 'a max of {}.'.format(max_required) - raise CLIError(min_max_msg_fmt) - - -def _validate_whl_extension(ext_file): - tmp_dir = tempfile.mkdtemp() - zip_ref = zipfile.ZipFile(ext_file, 'r') - zip_ref.extractall(tmp_dir) - zip_ref.close() - azext_metadata = WheelExtension.get_azext_metadata(tmp_dir) - shutil.rmtree(tmp_dir) - _validate_whl_cli_compat(azext_metadata) - - -def _add_whl_ext(source, ext_sha256=None, pip_extra_index_urls=None, pip_proxy=None): # pylint: disable=too-many-statements - if not source.endswith('.whl'): - raise ValueError('Unknown extension type. Only Python wheels are supported.') - url_parse_result = urlparse(source) - is_url = (url_parse_result.scheme == 'http' or url_parse_result.scheme == 'https') - logger.debug('Extension source is url? %s', is_url) - whl_filename = os.path.basename(url_parse_result.path) if is_url else os.path.basename(source) - parsed_filename = WHEEL_INFO_RE(whl_filename) - # Extension names can have - but .whl format changes it to _ (PEP 0427). Undo this. - extension_name = parsed_filename.groupdict().get('name').replace('_', '-') if parsed_filename else None - if not extension_name: - raise CLIError('Unable to determine extension name from {}. Is the file name correct?'.format(source)) - if extension_exists(extension_name): - raise CLIError('The extension {} already exists.'.format(extension_name)) - ext_file = None - if is_url: - # Download from URL - tmp_dir = tempfile.mkdtemp() - ext_file = os.path.join(tmp_dir, whl_filename) - logger.debug('Downloading %s to %s', source, ext_file) - try: - _whl_download_from_url(url_parse_result, ext_file) - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err: - raise CLIError('Please ensure you have network connection. Error detail: {}'.format(str(err))) - logger.debug('Downloaded to %s', ext_file) - else: - # Get file path - ext_file = os.path.realpath(os.path.expanduser(source)) - if not os.path.isfile(ext_file): - raise CLIError("File {} not found.".format(source)) - # Validate the extension - logger.debug('Validating the extension %s', ext_file) - if ext_sha256: - valid_checksum, computed_checksum = is_valid_sha256sum(ext_file, ext_sha256) - if valid_checksum: - logger.debug("Checksum of %s is OK", ext_file) - else: - logger.debug("Invalid checksum for %s. Expected '%s', computed '%s'.", - ext_file, ext_sha256, computed_checksum) - raise CLIError("The checksum of the extension does not match the expected value. " - "Use --debug for more information.") - try: - _validate_whl_extension(ext_file) - except AssertionError: - logger.debug(traceback.format_exc()) - raise CLIError('The extension is invalid. Use --debug for more information.') - except CLIError as e: - raise e - logger.debug('Validation successful on %s', ext_file) - # Check for distro consistency - check_distro_consistency() - # Install with pip - extension_path = get_extension_path(extension_name) - pip_args = ['install', '--target', extension_path, ext_file] - - if pip_proxy: - pip_args = pip_args + ['--proxy', pip_proxy] - if pip_extra_index_urls: - for extra_index_url in pip_extra_index_urls: - pip_args = pip_args + ['--extra-index-url', extra_index_url] - - logger.debug('Executing pip with args: %s', pip_args) - with HomebrewPipPatch(): - pip_status_code = _run_pip(pip_args) - if pip_status_code > 0: - logger.debug('Pip failed so deleting anything we might have installed at %s', extension_path) - shutil.rmtree(extension_path, ignore_errors=True) - raise CLIError('An error occurred. Pip failed with status code {}. ' - 'Use --debug for more information.'.format(pip_status_code)) - # Save the whl we used to install the extension in the extension dir. - dst = os.path.join(extension_path, whl_filename) - shutil.copyfile(ext_file, dst) - logger.debug('Saved the whl to %s', dst) - -def is_valid_sha256sum(a_file, expected_sum): - sha256 = hashlib.sha256() - with open(a_file, 'rb') as f: - sha256.update(f.read()) - computed_hash = sha256.hexdigest() - return expected_sum == computed_hash, computed_hash - - -def _augment_telemetry_with_ext_info(extension_name): - # The extension must be available before calling this otherwise we can't get the version from metadata - if not extension_name: - return - try: - ext = get_extension(extension_name) - ext_version = ext.version - set_extension_management_detail(extension_name, ext_version) - except Exception: # nopa pylint: disable=broad-except - # Don't error on telemetry - pass - - -def add_extension(source=None, extension_name=None, index_url=None, yes=None, # pylint: disable=unused-argument - pip_extra_index_urls=None, pip_proxy=None): - ext_sha256 = None - if extension_name: - if extension_exists(extension_name): - logger.warning("The extension '%s' already exists.", extension_name) - return - try: - source, ext_sha256 = resolve_from_index(extension_name, index_url=index_url) - except NoExtensionCandidatesError as err: - logger.debug(err) - raise CLIError("No matching extensions for '{}'. Use --debug for more information.".format(extension_name)) - _add_whl_ext(source, ext_sha256=ext_sha256, pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy) - _augment_telemetry_with_ext_info(extension_name) - try: - if extension_name and get_extension(extension_name).preview: - logger.warning("The installed extension '%s' is in preview.", extension_name) - except ExtensionNotInstalledException: - pass - - -def remove_extension(extension_name): - def log_err(func, path, exc_info): - logger.debug("Error occurred attempting to delete item from the extension '%s'.", extension_name) - logger.debug("%s: %s - %s", func, path, exc_info) - - try: - # Get the extension and it will raise an error if it doesn't exist - get_extension(extension_name) - # We call this just before we remove the extension so we can get the metadata before it is gone - _augment_telemetry_with_ext_info(extension_name) - shutil.rmtree(get_extension_path(extension_name), onerror=log_err) - except ExtensionNotInstalledException as e: - raise CLIError(e) - - -def list_extensions(): - return [{OUT_KEY_NAME: ext.name, OUT_KEY_VERSION: ext.version, OUT_KEY_TYPE: ext.ext_type} - for ext in get_extensions()] - - -def show_extension(extension_name): - try: - extension = get_extension(extension_name) - return {OUT_KEY_NAME: extension.name, - OUT_KEY_VERSION: extension.version, - OUT_KEY_TYPE: extension.ext_type, - OUT_KEY_METADATA: extension.metadata} - except ExtensionNotInstalledException as e: - raise CLIError(e) - - -def update_extension(extension_name, index_url=None, pip_extra_index_urls=None, pip_proxy=None): - try: - ext = get_extension(extension_name) - cur_version = ext.get_version() - try: - download_url, ext_sha256 = resolve_from_index(extension_name, cur_version=cur_version, index_url=index_url) - except NoExtensionCandidatesError as err: - logger.debug(err) - raise CLIError("No updates available for '{}'. Use --debug for more information.".format(extension_name)) - # Copy current version of extension to tmp directory in case we need to restore it after a failed install. - backup_dir = os.path.join(tempfile.mkdtemp(), extension_name) - extension_path = get_extension_path(extension_name) - logger.debug('Backing up the current extension: %s to %s', extension_path, backup_dir) - shutil.copytree(extension_path, backup_dir) - # Remove current version of the extension - shutil.rmtree(extension_path) - # Install newer version - try: - _add_whl_ext(download_url, ext_sha256=ext_sha256, +def add_extension_cmd(source=None, extension_name=None, index_url=None, yes=None, + pip_extra_index_urls=None, pip_proxy=None): + return add_extension(source=source, extension_name=extension_name, index_url=index_url, yes=yes, pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy) - logger.debug('Deleting backup of old extension at %s', backup_dir) - shutil.rmtree(backup_dir) - # This gets the metadata for the extension *after* the update - _augment_telemetry_with_ext_info(extension_name) - except Exception as err: - logger.error('An error occurred whilst updating.') - logger.error(err) - logger.debug('Copying %s to %s', backup_dir, extension_path) - shutil.copytree(backup_dir, extension_path) - raise CLIError('Failed to update. Rolled {} back to {}.'.format(extension_name, cur_version)) - except ExtensionNotInstalledException as e: - raise CLIError(e) - -def list_available_extensions(index_url=None, show_details=False): - index_data = get_index_extensions(index_url=index_url) - if show_details: - return index_data - installed_extensions = get_extensions() - installed_extension_names = [e.name for e in installed_extensions] - results = [] - for name, items in OrderedDict(sorted(index_data.items())).items(): - # exclude extensions/versions incompatible with current CLI version - items = [item for item in items if ext_compat_with_cli(item['metadata'])[0]] - if not items: - continue - latest = max(items, key=lambda c: parse_version(c['metadata']['version'])) - installed = False - if name in installed_extension_names: - installed = True - ext_version = get_extension(name).version - if ext_version and parse_version(latest['metadata']['version']) > parse_version(ext_version): - installed = str(True) + ' (upgrade available)' - results.append({ - 'name': name, - 'version': latest['metadata']['version'], - 'summary': latest['metadata']['summary'], - 'preview': latest['metadata'].get(EXT_METADATA_ISPREVIEW, False), - 'installed': installed - }) - return results +def remove_extension_cmd(extension_name): + return remove_extension(extension_name) -def get_lsb_release(): - try: - with open(LSB_RELEASE_FILE, 'r') as lr: - lsb = lr.readlines() - desc = lsb[2] - desc_split = desc.split('=') - rel = desc_split[1] - return rel.strip() - except Exception: # pylint: disable=broad-except - return None +def list_extensions_cmd(): + return list_extensions() -def check_distro_consistency(): - if IS_WINDOWS: - return +def show_extension_cmd(extension_name): + return show_extension(extension_name) - try: - logger.debug('Linux distro check: Reading from: %s', LIST_FILE_PATH) - with open(LIST_FILE_PATH, 'r') as list_file: - package_source = list_file.read() - stored_linux_dist_name = package_source.split(" ")[3] - logger.debug('Linux distro check: Found in list file: %s', stored_linux_dist_name) - current_linux_dist_name = get_lsb_release() - logger.debug('Linux distro check: Reported by API: %s', current_linux_dist_name) +def update_extension_cmd(extension_name, index_url=None, pip_extra_index_urls=None, pip_proxy=None): + return update_extension(extension_name, index_url=index_url, pip_extra_index_urls=pip_extra_index_urls, + pip_proxy=pip_proxy) - except Exception as err: # pylint: disable=broad-except - current_linux_dist_name = None - stored_linux_dist_name = None - logger.debug('Linux distro check: An error occurred while checking ' - 'linux distribution version source list consistency.') - logger.debug(err) - if current_linux_dist_name != stored_linux_dist_name: - logger.debug("Linux distro check: Mismatch distribution " - "name in %s file", LIST_FILE_PATH) - logger.debug("Linux distro check: If command fails, install the appropriate package " - "for your distribution or change the above file accordingly.") - logger.debug("Linux distro check: %s has '%s', current distro is '%s'", - LIST_FILE_PATH, stored_linux_dist_name, current_linux_dist_name) +def list_available_extensions_cmd(index_url=None, show_details=False): + return list_available_extensions(index_url=index_url, show_details=show_details) diff --git a/src/command_modules/azure-cli-extension/setup.py b/src/command_modules/azure-cli-extension/setup.py index 9be1694a9e2..63c63145383 100644 --- a/src/command_modules/azure-cli-extension/setup.py +++ b/src/command_modules/azure-cli-extension/setup.py @@ -14,7 +14,7 @@ logger.warn("Wheel is not available, disabling bdist_wheel hook") cmdclass = {} -VERSION = "0.2.2" +VERSION = "0.2.3" CLASSIFIERS = [ 'Development Status :: 5 - Production/Stable', diff --git a/src/command_modules/azure-cli-iot/HISTORY.rst b/src/command_modules/azure-cli-iot/HISTORY.rst index ddb4a85e2e0..cac42ae881e 100644 --- a/src/command_modules/azure-cli-iot/HISTORY.rst +++ b/src/command_modules/azure-cli-iot/HISTORY.rst @@ -2,9 +2,13 @@ Release History =============== +0.3.4 ++++++ +* Minor fixes + 0.3.3 +++++ -* Added extension installation comand to first-run banner +* Added extension installation command to first-run banner 0.3.2 +++++ diff --git a/src/command_modules/azure-cli-iot/azure/cli/command_modules/iot/__init__.py b/src/command_modules/azure-cli-iot/azure/cli/command_modules/iot/__init__.py index 149c8b81ff8..34c5ed32b61 100644 --- a/src/command_modules/azure-cli-iot/azure/cli/command_modules/iot/__init__.py +++ b/src/command_modules/azure-cli-iot/azure/cli/command_modules/iot/__init__.py @@ -8,7 +8,7 @@ from azure.cli.core import AzCommandsLoader from azure.cli.core.commands import CliCommandType import azure.cli.command_modules.iot._help # pylint: disable=unused-import -from azure.cli.core.extension import extension_exists +from azure.cli.core.extensions import extension_exists def handler(ctx, **kwargs): diff --git a/src/command_modules/azure-cli-iot/setup.py b/src/command_modules/azure-cli-iot/setup.py index e913c4cd95e..e9603c1058b 100644 --- a/src/command_modules/azure-cli-iot/setup.py +++ b/src/command_modules/azure-cli-iot/setup.py @@ -14,7 +14,7 @@ logger.warn("Wheel is not available, disabling bdist_wheel hook") cmdclass = {} -VERSION = "0.3.3" +VERSION = "0.3.4" # The full list of classifiers is available at # https://pypi.python.org/pypi?%3Aaction=list_classifiers CLASSIFIERS = [ diff --git a/tools/automation/tests/__init__.py b/tools/automation/tests/__init__.py index bd31dcc741c..8b174b06d6d 100644 --- a/tools/automation/tests/__init__.py +++ b/tools/automation/tests/__init__.py @@ -148,7 +148,7 @@ def get_test_index(args): def get_extension_modules(): from importlib import import_module import pkgutil - from azure.cli.core.extension import get_extensions, get_extension_path, get_extension_modname + from azure.cli.core.extensions import get_extensions, get_extension_path, get_extension_modname extension_whls = get_extensions() ext_modules = [] if extension_whls: