Skip to content

[Extension] Add azmirror cloud endpoint for installing extensions in air gapped clouds and support config of index url #15128

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fengzhou-msft merged 10 commits into from
Feb 23, 2021
Merged

[Extension] Add azmirror cloud endpoint for installing extensions in air gapped clouds and support config of index url #15128

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
5a15e31
add extension_storage_account_resource_id
fengzhou-msft Sep 10, 2020
45993c0
use extension_storage_account_resource_id
fengzhou-msft Sep 10, 2020
c252d84
use endpoint for extension command tree
fengzhou-msft Sep 11, 2020
5ce9bc7
fix style
fengzhou-msft Sep 11, 2020
20690bd
fix tests
fengzhou-msft Sep 11, 2020
5eb1a1f
merge dev
fengzhou-msft Feb 4, 2021
93c24c2
use a broader azmirror endpoint
fengzhou-msft Feb 4, 2021
75c9dd9
update endpoint
fengzhou-msft Feb 5, 2021
19d1c2f
fix line-too-long
fengzhou-msft Feb 5, 2021
17f0e6e
process azmirror_storage_account_resource_id for ARM metadata
fengzhou-msft Feb 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/azure-cli-core/azure/cli/core/cloud.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,8 @@ def __init__(self,
app_insights_resource_id=None,
app_insights_telemetry_channel_resource_id=None,
synapse_analytics_resource_id=None,
attestation_resource_id=None):
attestation_resource_id=None,
extension_storage_account_resource_id=None):

@yungezz yungezz Sep 11, 2020

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

extension_pacakges_url?

# Attribute names are significant. They are used when storing/retrieving clouds from config
self.management = management
self.resource_manager = resource_manager
Expand All @@ -94,6 +95,7 @@ def __init__(self,
self.app_insights_telemetry_channel_resource_id = app_insights_telemetry_channel_resource_id
self.synapse_analytics_resource_id = synapse_analytics_resource_id
self.attestation_resource_id = attestation_resource_id
self.extension_storage_account_resource_id = extension_storage_account_resource_id

def has_endpoint_set(self, endpoint_name):
try:
Expand Down
19 changes: 15 additions & 4 deletions src/azure-cli-core/azure/cli/core/extension/_index.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,22 @@
TRIES = 3


def get_index_url(cli_ctx=None):
import posixpath
if cli_ctx:
url = cli_ctx.config.get('extension', 'index_url', None)
if url:
return url
ext_endpoint = cli_ctx.cloud.endpoints.extension_storage_account_resource_id if cli_ctx and \

@yungezz yungezz Sep 11, 2020

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

config overrides cloud setting

cli_ctx.cloud.endpoints.has_endpoint_set('extension_storage_account_resource_id') else None
return posixpath.join(ext_endpoint, 'index.json') if ext_endpoint else DEFAULT_INDEX_URL


# pylint: disable=inconsistent-return-statements
def get_index(index_url=None):
def get_index(index_url=None, cli_ctx=None):
import requests
from azure.cli.core.util import should_disable_connection_verify
index_url = index_url or DEFAULT_INDEX_URL
index_url = index_url or get_index_url(cli_ctx=cli_ctx)

for try_number in range(TRIES):
try:
Expand All @@ -45,8 +56,8 @@ def get_index(index_url=None):
continue


def get_index_extensions(index_url=None):
index = get_index(index_url=index_url)
def get_index_extensions(index_url=None, cli_ctx=None):
index = get_index(index_url=index_url, cli_ctx=cli_ctx)
extensions = index.get('extensions')
if extensions is None:
logger.warning(ERR_UNABLE_TO_GET_EXTENSIONS)
Expand Down
11 changes: 9 additions & 2 deletions src/azure-cli-core/azure/cli/core/extension/_resolve.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,13 +53,13 @@ def filter_func(item):
return filter_func


def resolve_from_index(extension_name, cur_version=None, index_url=None, target_version=None):
def resolve_from_index(extension_name, cur_version=None, index_url=None, target_version=None, cli_ctx=None):
"""
Gets the download Url and digest for the matching extension

:param cur_version: threshold verssion to filter out extensions.
"""
candidates = get_index_extensions(index_url=index_url).get(extension_name, [])
candidates = get_index_extensions(index_url=index_url, cli_ctx=cli_ctx).get(extension_name, [])

if not candidates:
raise NoExtensionCandidatesError("No extension found with name '{}'".format(extension_name))
Expand Down Expand Up @@ -88,6 +88,13 @@ def resolve_from_index(extension_name, cur_version=None, index_url=None, target_
download_url, digest = chosen.get('downloadUrl'), chosen.get('sha256Digest')
if not download_url:
raise NoExtensionCandidatesError("No download url found.")
ext_endpoint = cli_ctx.cloud.endpoints.extension_storage_account_resource_id if cli_ctx and \
cli_ctx.cloud.endpoints.has_endpoint_set('extension_storage_account_resource_id') else None
config_index_url = cli_ctx.config.get('extension', 'index_url', None) if cli_ctx else None

@jiasli jiasli Sep 15, 2020

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we can put these 2 URLs in one place?

@jiasli jiasli Sep 15, 2020

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, isn't the whl location hard-coded in the index?

{
    "downloadUrl": "https://azurecliprod.blob.core.windows.net/cli-extensions/account-0.2.0-py2.py3-none-any.whl",
    "filename": "account-0.2.0-py2.py3-none-any.whl",
    "metadata": {
        "azext.isExperimental": true,

@fengzhou-msft fengzhou-msft Feb 8, 2021 •
edited
Loading

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The index will be teleported into different storage accounts in air gapped clouds, we do not have permission to modify the whl downloadUrl based on the destination storage accounts after it gets copied in air gapped clouds. So we need to modify the downloadUrl here in azure-cli when we need to download it in an air gapped cloud.

@jiasli jiasli Feb 8, 2021

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we really need to put one URL in cloud config, one in az config?

@fengzhou-msft fengzhou-msft Feb 9, 2021

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The one in cloud config is now a general support for downloading extensions and other tools/files needed in air gapped clouds. It's always a storage account endpoint and controlled by Microsoft.

The one in az config is to support customers that want to use their own customized index.json, which may have their own private extensions and some allowed public extensions. You can use any url of the index file, such as the raw content url on a github branch.

if ext_endpoint and not config_index_url:
import posixpath
whl_name = download_url.split('/')[-1]
download_url = posixpath.join(ext_endpoint, whl_name)
return download_url, digest


Expand Down
12 changes: 6 additions & 6 deletions src/azure-cli-core/azure/cli/core/extension/operations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ def add_extension(cmd=None, source=None, extension_name=None, index_url=None, ye
return
logger.warning("Overriding development version of '%s' with production version.", extension_name)
try:
source, ext_sha256 = resolve_from_index(extension_name, index_url=index_url, target_version=version)
source, ext_sha256 = resolve_from_index(extension_name, index_url=index_url, target_version=version, cli_ctx=cmd_cli_ctx)
except NoExtensionCandidatesError as err:
logger.debug(err)

Expand Down Expand Up @@ -295,7 +295,7 @@ def update_extension(cmd=None, extension_name=None, index_url=None, pip_extra_in
ext = get_extension(extension_name, ext_type=WheelExtension)
cur_version = ext.get_version()
try:
download_url, ext_sha256 = resolve_from_index(extension_name, cur_version=cur_version, index_url=index_url)
download_url, ext_sha256 = resolve_from_index(extension_name, cur_version=cur_version, index_url=index_url, cli_ctx=cmd_cli_ctx)
except NoExtensionCandidatesError as err:
logger.debug(err)
msg = "No updates available for '{}'. Use --debug for more information.".format(extension_name)
Expand Down Expand Up @@ -327,8 +327,8 @@ def update_extension(cmd=None, extension_name=None, index_url=None, pip_extra_in
raise CLIError(e)


def list_available_extensions(index_url=None, show_details=False):
index_data = get_index_extensions(index_url=index_url)
def list_available_extensions(index_url=None, show_details=False, cli_ctx=None):
index_data = get_index_extensions(index_url=index_url, cli_ctx=cli_ctx)
if show_details:
return index_data
installed_extensions = get_extensions(ext_type=WheelExtension)
Expand Down Expand Up @@ -358,8 +358,8 @@ def list_available_extensions(index_url=None, show_details=False):
return results


def list_versions(extension_name, index_url=None):
index_data = get_index_extensions(index_url=index_url)
def list_versions(extension_name, index_url=None, cli_ctx=None):
index_data = get_index_extensions(index_url=index_url, cli_ctx=cli_ctx)

try:
exts = index_data[extension_name]
Expand Down
18 changes: 9 additions & 9 deletions src/azure-cli-core/azure/cli/core/extension/tests/latest/test_extension_commands.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,19 +332,19 @@ def test_update_extension_exception_in_update_and_rolled_back(self):

def test_list_available_extensions_default(self):
with mock.patch('azure.cli.core.extension.operations.get_index_extensions', autospec=True) as c:
list_available_extensions()
c.assert_called_once_with(None)
list_available_extensions(cli_ctx=self.cmd.cli_ctx)
c.assert_called_once_with(None, self.cmd.cli_ctx)

def test_list_available_extensions_operations_index_url(self):
with mock.patch('azure.cli.core.extension.operations.get_index_extensions', autospec=True) as c:
index_url = 'http://contoso.com'
list_available_extensions(index_url=index_url)
c.assert_called_once_with(index_url)
list_available_extensions(index_url=index_url, cli_ctx=self.cmd.cli_ctx)
c.assert_called_once_with(index_url, self.cmd.cli_ctx)

def test_list_available_extensions_show_details(self):
with mock.patch('azure.cli.core.extension.operations.get_index_extensions', autospec=True) as c:
list_available_extensions(show_details=True)
c.assert_called_once_with(None)
list_available_extensions(show_details=True, cli_ctx=self.cmd.cli_ctx)
c.assert_called_once_with(None, self.cmd.cli_ctx)

def test_list_available_extensions_no_show_details(self):
sample_index_extensions = {
Expand All @@ -364,7 +364,7 @@ def test_list_available_extensions_no_show_details(self):
}}]
}
with mock.patch('azure.cli.core.extension.operations.get_index_extensions', return_value=sample_index_extensions):
res = list_available_extensions()
res = list_available_extensions(cli_ctx=self.cmd.cli_ctx)
self.assertIsInstance(res, list)
self.assertEqual(len(res), len(sample_index_extensions))
self.assertEqual(res[0]['name'], 'test_sample_extension1')
Expand All @@ -373,7 +373,7 @@ def test_list_available_extensions_no_show_details(self):
self.assertEqual(res[0]['preview'], False)
self.assertEqual(res[0]['experimental'], False)
with mock.patch('azure.cli.core.extension.operations.get_index_extensions', return_value=sample_index_extensions):
res = list_available_extensions()
res = list_available_extensions(cli_ctx=self.cmd.cli_ctx)
self.assertIsInstance(res, list)
self.assertEqual(len(res), len(sample_index_extensions))
self.assertEqual(res[1]['name'], 'test_sample_extension2')
Expand All @@ -393,7 +393,7 @@ def test_list_available_extensions_incompatible_cli_version(self):
}}]
}
with mock.patch('azure.cli.core.extension.operations.get_index_extensions', return_value=sample_index_extensions):
res = list_available_extensions()
res = list_available_extensions(cli_ctx=self.cmd.cli_ctx)
self.assertIsInstance(res, list)
self.assertEqual(len(res), 0)

Expand Down
21 changes: 21 additions & 0 deletions src/azure-cli-core/azure/cli/core/extension/tests/latest/test_index_get.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,27 @@ def test_get_index_extensions(self):
self.assertEqual(get_index_extensions(), None)
logger_mock.assert_called_once_with(ERR_UNABLE_TO_GET_EXTENSIONS)

# pylint: disable=line-too-long
def test_get_index_cloud(self):

from azure.cli.core.mock import DummyCli
cli_ctx = DummyCli()

default_data = {'extensions': {}}
obj = object()
cloud_data = {'extensions': {'myext': obj}}
# cli_ctx not passed
with mock.patch('requests.get', side_effect=mock_index_get_generator(DEFAULT_INDEX_URL, default_data)):
self.assertEqual(get_index_extensions(), {})
# cli_ctx passed but endpoint not set
delattr(cli_ctx.cloud.endpoints, 'extension_storage_account_resource_id')
with mock.patch('requests.get', side_effect=mock_index_get_generator(DEFAULT_INDEX_URL, default_data)):
self.assertEqual(get_index_extensions(cli_ctx=cli_ctx), {})
# cli_ctx passed and the endpoint is set
cli_ctx.cloud.endpoints.extension_storage_account_resource_id = 'http://contoso.com/cli-index'
with mock.patch('requests.get', side_effect=mock_index_get_generator('http://contoso.com/cli-index/index.json', cloud_data)):
self.assertEqual(get_index_extensions(cli_ctx=cli_ctx).get('myext'), obj)


if __name__ == '__main__':
unittest.main()
9 changes: 7 additions & 2 deletions src/azure-cli-core/azure/cli/core/parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -291,13 +291,18 @@ def _get_extension_command_tree(self):
return None
EXT_CMD_TREE.load(os.path.join(cli_ctx.config.config_dir, 'extensionCommandTree.json'), VALID_SECOND)
if not EXT_CMD_TREE.data:
import posixpath
import requests
from azure.cli.core.util import should_disable_connection_verify
try:
ext_endpoint = cli_ctx.cloud.endpoints.extension_storage_account_resource_id if cli_ctx and \
cli_ctx.cloud.endpoints.has_endpoint_set('extension_storage_account_resource_id') else None
url = posixpath.join(ext_endpoint, 'extensionCommandTree.json') if ext_endpoint else \
'https://azurecliextensionsync.blob.core.windows.net/cmd-index/extensionCommandTree.json'
response = requests.get(
'https://azurecliextensionsync.blob.core.windows.net/cmd-index/extensionCommandTree.json',
url,
verify=(not should_disable_connection_verify()),
timeout=300)
timeout=10)
except Exception as ex: # pylint: disable=broad-except
logger.info("Request failed for extension command tree: %s", str(ex))
return None
Expand Down
2 changes: 1 addition & 1 deletion src/azure-cli/azure/cli/command_modules/extension/_completers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,4 @@ def extension_name_completion_list(cmd, prefix, namespace, **kwargs): # pylint:

@Completer
def extension_name_from_index_completion_list(cmd, prefix, namespace, **kwargs): # pylint: disable=unused-argument
return get_index_extensions().keys()
return get_index_extensions(cli_ctx=cmd.cli_ctx).keys()
8 changes: 4 additions & 4 deletions src/azure-cli/azure/cli/command_modules/extension/custom.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,9 @@ def update_extension_cmd(cmd, extension_name, index_url=None, pip_extra_index_ur
pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy)


def list_available_extensions_cmd(index_url=None, show_details=False):
return list_available_extensions(index_url=index_url, show_details=show_details)
def list_available_extensions_cmd(cmd, index_url=None, show_details=False):
return list_available_extensions(index_url=index_url, show_details=show_details, cli_ctx=cmd.cli_ctx)


def list_versions_cmd(extension_name, index_url=None):
return list_versions(extension_name, index_url=index_url)
def list_versions_cmd(cmd, extension_name, index_url=None):
return list_versions(extension_name, index_url=index_url, cli_ctx=cmd.cli_ctx)