From cdddaf1f6694a1f76296062fbf3350dd114a5751 Mon Sep 17 00:00:00 2001 From: Shenglong Li Date: Thu, 1 Apr 2021 13:59:47 -0700 Subject: [PATCH 1/5] Update params for az bicep build/decompile --- .../cli/command_modules/resource/_help.py | 22 +++++++++---------- .../cli/command_modules/resource/_params.py | 13 +++++++---- .../cli/command_modules/resource/custom.py | 16 +++++++++----- 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/src/azure-cli/azure/cli/command_modules/resource/_help.py b/src/azure-cli/azure/cli/command_modules/resource/_help.py index e3654ab0bb8..1b5525083e2 100644 --- a/src/azure-cli/azure/cli/command_modules/resource/_help.py +++ b/src/azure-cli/azure/cli/command_modules/resource/_help.py @@ -2334,26 +2334,24 @@ helps['bicep build'] = """ type: command -short-summary: Build one or more Bicep files. +short-summary: Build a Bicep file. examples: - name: Build a Bicep file. - text: az bicep build --files {bicep_file} - - name: Build multiple Bicep files. - text: az bicep build --files {bicep_file1} {bicep_file2} - - name: Build a Bicep file and prints all output to stdout. - text: az bicep build --files {bicep_file} --stdout - - name: Build multiple Bicep files and prints all output to stdout. - text: az bicep build --files {bicep_file1} {bicep_file2} --stdout + text: az bicep build --file {bicep_file} + - name: Build a Bicep file and print all output to stdout. + text: az bicep build --file {bicep_file} --stdout + - name: Build a Bicep file and save the result to the specified directory. + text: az bicep build --file {bicep_file} --outdir {out_dir} + - name: Build a Bicep file and save the result to the specified file. + text: az bicep build --file {bicep_file} --outfile {out_file} """ helps['bicep decompile'] = """ type: command -short-summary: Attempt to decompile one or more ARM template files to Bicep files +short-summary: Attempt to decompile an ARM template file to a Bicep file. examples: - name: Decompile an ARM template file. - text: az bicep decompile --files {json_template_file} - - name: Decompile multiple ARM template files. - text: az bicep decompile --files {json_template_file1} {json_template_file2} + text: az bicep decompile --file {json_template_file} """ helps['bicep version'] = """ diff --git a/src/azure-cli/azure/cli/command_modules/resource/_params.py b/src/azure-cli/azure/cli/command_modules/resource/_params.py index 09008cc7e82..36bc44081b5 100644 --- a/src/azure-cli/azure/cli/command_modules/resource/_params.py +++ b/src/azure-cli/azure/cli/command_modules/resource/_params.py @@ -7,6 +7,7 @@ # pylint: disable=too-many-locals, too-many-statements, line-too-long def load_arguments(self, _): from argcomplete.completers import FilesCompleter + from argcomplete.completers import DirectoriesCompleter from azure.mgmt.resource.locks.models import LockLevel from azure.mgmt.resource.managedapplications.models import ApplicationLockLevel @@ -572,14 +573,18 @@ def load_arguments(self, _): c.argument('resource_group', arg_type=resource_group_name_type) with self.argument_context('bicep build') as c: - c.argument('files', arg_type=CLIArgumentType(nargs="+", options_list=['--files', '-f'], completer=FilesCompleter(), - type=file_type, help="Space separated Bicep file paths in the file system.")) + c.argument('files', arg_type=CLIArgumentType(options_list=['--file', '-f'], completer=FilesCompleter(), + type=file_type, help="The path to the Bicep file to build in the file system.")) + c.argument('outdir', arg_type=CLIArgumentType(options_list=['--outdir'], completer=DirectoriesCompleter(), + help="When set, saves the output at the specified directory.")) + c.argument('outfile', arg_type=CLIArgumentType(options_list=['--outfile'], completer=FilesCompleter(), + help="When set, saves the output as the specified file path.")) c.argument('stdout', arg_type=CLIArgumentType(options_list=['--stdout'], action='store_true', help="When set, prints all output to stdout instead of corresponding files.")) with self.argument_context('bicep decompile') as c: - c.argument('files', arg_type=CLIArgumentType(nargs="+", options_list=['--files', '-f'], completer=FilesCompleter(), - type=file_type, help="Space separated ARM template paths in the file system.")) + c.argument('files', arg_type=CLIArgumentType(options_list=['--file', '-f'], completer=FilesCompleter(), + type=file_type, help="The path to the ARM template to decompile in the file system.")) with self.argument_context('bicep install') as c: c.argument('version', options_list=['--version', '-v'], help='The version of Bicep CLI to be installed. Default to the latest if not specified.') diff --git a/src/azure-cli/azure/cli/command_modules/resource/custom.py b/src/azure-cli/azure/cli/command_modules/resource/custom.py index 8cfd914b203..4bc7e121c27 100644 --- a/src/azure-cli/azure/cli/command_modules/resource/custom.py +++ b/src/azure-cli/azure/cli/command_modules/resource/custom.py @@ -3194,15 +3194,19 @@ def upgrade_bicep_cli(cmd): ensure_bicep_installation(release_tag=latest_release_tag) -def build_bicep_file(cmd, files, stdout=None): +def build_bicep_file(cmd, file, stdout=None, outdir=None, outfile=None): + args = ["build", file] + if outdir: + args += ["--outdir", outdir] + if outfile: + args += ["--outfile", outfile] if stdout: - print(run_bicep_command(["build"] + files + ["--stdout"])) - else: - run_bicep_command(["build"] + files) + args += ["--stdout"] + run_bicep_command(args) -def decompile_bicep_file(cmd, files): - run_bicep_command(["decompile"] + files) +def decompile_bicep_file(cmd, file): + run_bicep_command(["decompile", file]) def show_bicep_cli_version(cmd): From af76e8d62d163a6503f34176016a477467c059ea Mon Sep 17 00:00:00 2001 From: Shenglong Li Date: Thu, 1 Apr 2021 15:40:06 -0700 Subject: [PATCH 2/5] Validate target scope --- .../cli/command_modules/resource/_bicep.py | 39 ++++++++++++++++++- .../cli/command_modules/resource/custom.py | 29 ++++++++------ .../tests/latest/test_resource_bicep.py | 30 +++++++++++++- 3 files changed, 83 insertions(+), 15 deletions(-) diff --git a/src/azure-cli/azure/cli/command_modules/resource/_bicep.py b/src/azure-cli/azure/cli/command_modules/resource/_bicep.py index 6caf507a447..d1ac505a974 100644 --- a/src/azure-cli/azure/cli/command_modules/resource/_bicep.py +++ b/src/azure-cli/azure/cli/command_modules/resource/_bicep.py @@ -17,13 +17,31 @@ from six.moves.urllib.request import urlopen from knack.log import get_logger -from azure.cli.core.azclierror import FileOperationError, ValidationError, UnclassifiedUserFault, ClientRequestError +from azure.cli.core.azclierror import ( + FileOperationError, + ValidationError, + UnclassifiedUserFault, + ClientRequestError, + InvalidTemplateError, +) # See: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string _semver_pattern = r"(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?" # pylint: disable=line-too-long + +# See: https://docs.microsoft.com/en-us/azure/azure-resource-manager/templates/template-syntax#template-format +_template_schema_pattern = r"https?://schema\.management\.azure\.com/schemas/[0-9a-zA-Z-]+/(?P[a-zA-Z]+)Template\.json#?" # pylint: disable=line-too-long + _logger = get_logger(__name__) +def validate_bicep_target_scope(template_schema, deployment_scope): + target_scope = _template_schema_to_target_scope(template_schema) + if target_scope != deployment_scope: + raise InvalidTemplateError( + f'The target scope "{target_scope}" does not match the deployment scope "{deployment_scope}".' + ) + + def run_bicep_command(args, auto_install=True, check_upgrade=True): installation_path = _get_bicep_installation_path(platform.system()) installed = os.path.isfile(installation_path) @@ -84,7 +102,7 @@ def ensure_bicep_installation(release_tag=None, stdout=True): print(f'Successfully installed Bicep CLI to "{installation_path}".') else: _logger.info( - 'Successfully installed Bicep CLI to %s', + "Successfully installed Bicep CLI to %s", installation_path, ) except IOError as err: @@ -158,3 +176,20 @@ def _run_command(bicep_installation_path, args): return process.stdout.decode("utf-8") except subprocess.CalledProcessError: raise UnclassifiedUserFault(process.stderr.decode("utf-8")) + + +def _template_schema_to_target_scope(template_schema): + template_schema_match = re.search(_template_schema_pattern, template_schema) + template_type = template_schema_match.group('templateType') if template_schema_match else None + template_type_lower = template_type.lower() if template_type else None + print(template_type) + + if template_type_lower == "deployment": + return "resourceGroup" + if template_type_lower == "subscriptiondeployment": + return "subscription" + if template_type_lower == "managementgroupdeployment": + return "managementGroup" + if template_type_lower == "tenantdeployment": + return "tenant" + return None diff --git a/src/azure-cli/azure/cli/command_modules/resource/custom.py b/src/azure-cli/azure/cli/command_modules/resource/custom.py index 4bc7e121c27..71836153122 100644 --- a/src/azure-cli/azure/cli/command_modules/resource/custom.py +++ b/src/azure-cli/azure/cli/command_modules/resource/custom.py @@ -49,7 +49,8 @@ is_bicep_file, ensure_bicep_installation, get_bicep_latest_release_tag, - get_bicep_available_release_tags + get_bicep_available_release_tags, + validate_bicep_target_scope ) logger = get_logger(__name__) @@ -479,7 +480,7 @@ def _deploy_arm_template_at_subscription_scope(cmd, template_file=None, template_uri=None, parameters=None, deployment_name=None, deployment_location=None, validate_only=False, no_wait=False, no_prompt=False, template_spec=None, query_string=None): - deployment_properties = _prepare_deployment_properties_unmodified(cmd, template_file=template_file, + deployment_properties = _prepare_deployment_properties_unmodified(cmd, 'subscription', template_file=template_file, template_uri=template_uri, parameters=parameters, mode='Incremental', no_prompt=no_prompt, @@ -562,7 +563,7 @@ def _deploy_arm_template_at_resource_group(cmd, deployment_name=None, mode=None, rollback_on_error=None, validate_only=False, no_wait=False, aux_subscriptions=None, aux_tenants=None, no_prompt=False, template_spec=None, query_string=None): - deployment_properties = _prepare_deployment_properties_unmodified(cmd, template_file=template_file, + deployment_properties = _prepare_deployment_properties_unmodified(cmd, 'resourceGroup', template_file=template_file, template_uri=template_uri, parameters=parameters, mode=mode, rollback_on_error=rollback_on_error, @@ -644,7 +645,7 @@ def _deploy_arm_template_at_management_group(cmd, template_file=None, template_uri=None, parameters=None, deployment_name=None, deployment_location=None, validate_only=False, no_wait=False, no_prompt=False, template_spec=None, query_string=None): - deployment_properties = _prepare_deployment_properties_unmodified(cmd, template_file=template_file, + deployment_properties = _prepare_deployment_properties_unmodified(cmd, 'managementGroup', template_file=template_file, template_uri=template_uri, parameters=parameters, mode='Incremental', no_prompt=no_prompt, template_spec=template_spec, query_string=query_string) @@ -720,7 +721,7 @@ def _deploy_arm_template_at_tenant_scope(cmd, template_file=None, template_uri=None, parameters=None, deployment_name=None, deployment_location=None, validate_only=False, no_wait=False, no_prompt=False, template_spec=None, query_string=None): - deployment_properties = _prepare_deployment_properties_unmodified(cmd, template_file=template_file, + deployment_properties = _prepare_deployment_properties_unmodified(cmd, 'tenant', template_file=template_file, template_uri=template_uri, parameters=parameters, mode='Incremental', no_prompt=no_prompt, template_spec=template_spec, query_string=query_string,) @@ -759,7 +760,7 @@ def what_if_deploy_arm_template_at_resource_group(cmd, resource_group_name, aux_tenants=None, result_format=None, no_pretty_print=None, no_prompt=False, exclude_change_types=None, template_spec=None, query_string=None): - what_if_properties = _prepare_deployment_what_if_properties(cmd, template_file, template_uri, + what_if_properties = _prepare_deployment_what_if_properties(cmd, 'resourceGroup', template_file, template_uri, parameters, mode, result_format, no_prompt, template_spec, query_string) mgmt_client = _get_deployment_management_client(cmd.cli_ctx, aux_tenants=aux_tenants, plug_pipeline=(template_uri is None and template_spec is None)) @@ -773,7 +774,7 @@ def what_if_deploy_arm_template_at_subscription_scope(cmd, deployment_name=None, deployment_location=None, result_format=None, no_pretty_print=None, no_prompt=False, exclude_change_types=None, template_spec=None, query_string=None): - what_if_properties = _prepare_deployment_what_if_properties(cmd, template_file, template_uri, parameters, + what_if_properties = _prepare_deployment_what_if_properties(cmd, 'subscription', template_file, template_uri, parameters, DeploymentMode.incremental, result_format, no_prompt, template_spec, query_string) mgmt_client = _get_deployment_management_client(cmd.cli_ctx, plug_pipeline=(template_uri is None and template_spec is None)) what_if_poller = mgmt_client.what_if_at_subscription_scope(deployment_name, what_if_properties, deployment_location) @@ -786,7 +787,7 @@ def what_if_deploy_arm_template_at_management_group(cmd, management_group_id=Non deployment_name=None, deployment_location=None, result_format=None, no_pretty_print=None, no_prompt=False, exclude_change_types=None, template_spec=None, query_string=None): - what_if_properties = _prepare_deployment_what_if_properties(cmd, template_file, template_uri, parameters, + what_if_properties = _prepare_deployment_what_if_properties(cmd, 'managementGroup', template_file, template_uri, parameters, DeploymentMode.incremental, result_format, no_prompt, template_spec=template_spec, query_string=query_string) mgmt_client = _get_deployment_management_client(cmd.cli_ctx, plug_pipeline=(template_uri is None and template_spec is None)) what_if_poller = mgmt_client.what_if_at_management_group_scope(management_group_id, deployment_name, @@ -800,7 +801,7 @@ def what_if_deploy_arm_template_at_tenant_scope(cmd, deployment_name=None, deployment_location=None, result_format=None, no_pretty_print=None, no_prompt=False, exclude_change_types=None, template_spec=None, query_string=None): - what_if_properties = _prepare_deployment_what_if_properties(cmd, template_file, template_uri, parameters, + what_if_properties = _prepare_deployment_what_if_properties(cmd, 'tenant', template_file, template_uri, parameters, DeploymentMode.incremental, result_format, no_prompt, template_spec, query_string) mgmt_client = _get_deployment_management_client(cmd.cli_ctx, plug_pipeline=(template_uri is None and template_spec is None)) what_if_poller = mgmt_client.what_if_at_tenant_scope(deployment_name, deployment_location, what_if_properties) @@ -869,7 +870,7 @@ def _prepare_template_uri_with_query_string(template_uri, input_query_string): raise InvalidArgumentValueError('Unable to parse parameter: {} .Make sure the value is formed correctly.'.format(input_query_string)) -def _prepare_deployment_properties_unmodified(cmd, template_file=None, template_uri=None, parameters=None, +def _prepare_deployment_properties_unmodified(cmd, deployment_scope, template_file=None, template_uri=None, parameters=None, mode=None, rollback_on_error=None, no_prompt=False, template_spec=None, query_string=None): cli_ctx = cmd.cli_ctx DeploymentProperties, TemplateLink, OnErrorDeployment = get_sdk(cli_ctx, ResourceType.MGMT_RESOURCE_RESOURCES, @@ -901,6 +902,10 @@ def _prepare_deployment_properties_unmodified(cmd, template_file=None, template_ ) template_obj = _remove_comments_from_json(template_content, file_path=template_file) + if is_bicep_file(template_file): + template_schema = template_obj.get('$schema', '') + validate_bicep_target_scope(template_schema, deployment_scope) + if rollback_on_error == '': on_error_deployment = OnErrorDeployment(type='LastSuccessful') elif rollback_on_error: @@ -917,13 +922,13 @@ def _prepare_deployment_properties_unmodified(cmd, template_file=None, template_ return properties -def _prepare_deployment_what_if_properties(cmd, template_file, template_uri, parameters, +def _prepare_deployment_what_if_properties(cmd, deployment_scope, template_file, template_uri, parameters, mode, result_format, no_prompt, template_spec, query_string): DeploymentWhatIfProperties, DeploymentWhatIfSettings = get_sdk(cmd.cli_ctx, ResourceType.MGMT_RESOURCE_RESOURCES, 'DeploymentWhatIfProperties', 'DeploymentWhatIfSettings', mod='models') - deployment_properties = _prepare_deployment_properties_unmodified(cmd=cmd, template_file=template_file, template_uri=template_uri, + deployment_properties = _prepare_deployment_properties_unmodified(cmd, deployment_scope, template_file=template_file, template_uri=template_uri, parameters=parameters, mode=mode, no_prompt=no_prompt, template_spec=template_spec, query_string=query_string) deployment_what_if_properties = DeploymentWhatIfProperties(template=deployment_properties.template, template_link=deployment_properties.template_link, parameters=deployment_properties.parameters, mode=deployment_properties.mode, diff --git a/src/azure-cli/azure/cli/command_modules/resource/tests/latest/test_resource_bicep.py b/src/azure-cli/azure/cli/command_modules/resource/tests/latest/test_resource_bicep.py index 9262923c1da..f184aa62adc 100644 --- a/src/azure-cli/azure/cli/command_modules/resource/tests/latest/test_resource_bicep.py +++ b/src/azure-cli/azure/cli/command_modules/resource/tests/latest/test_resource_bicep.py @@ -7,7 +7,12 @@ import mock from knack.util import CLIError -from azure.cli.command_modules.resource._bicep import ensure_bicep_installation, run_bicep_command +from azure.cli.command_modules.resource._bicep import ( + ensure_bicep_installation, + run_bicep_command, + validate_bicep_target_scope, +) +from azure.cli.core.azclierror import InvalidTemplateError class TestBicep(unittest.TestCase): @@ -56,3 +61,26 @@ def test_ensure_bicep_installation_skip_download_if_installed_version_matches_re ensure_bicep_installation(release_tag="v0.1.0") dirname_mock.assert_not_called() + + def test_validate_target_scope_raise_error_if_target_scope_does_not_match_deployment_scope(self): + with self.assertRaisesRegex( + InvalidTemplateError, 'The target scope "tenant" does not match the deployment scope "subscription".' + ): + validate_bicep_target_scope( + "https://schema.management.azure.com/schemas/2019-08-01/tenantDeploymentTemplate.json#", "subscription" + ) + + def test_validate_target_scope_success_if_target_scope_matches_deployment_scope(self): + for template_schema, deployment_scope in [ + ("https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", "resourceGroup"), + ("https://schema.management.azure.com/schemas/2019-08-01/subscriptionDeploymentTemplate.json#", "subscription"), + ("https://schema.management.azure.com/schemas/2019-08-01/managementGroupDeploymentTemplate.json#", "managementGroup"), + ("https://schema.management.azure.com/schemas/2019-08-01/tenantDeploymentTemplate.json#", "tenant"), + ]: + with self.subTest(template_schema=template_schema, deployment_scope=deployment_scope): + try: + validate_bicep_target_scope(template_schema, deployment_scope) + except InvalidTemplateError as e: + self.fail(e.error_msg) + except: + self.fail("Encountered an unexpected exception.") From acb94d5ffbd67f43eb0b7ee63a281ada6eddfdb4 Mon Sep 17 00:00:00 2001 From: Shenglong Li Date: Thu, 1 Apr 2021 15:49:22 -0700 Subject: [PATCH 3/5] Fix live scenario tests --- .../latest/policy_definition_deploy_mg.bicep | 17 +++++++++++++++++ ...bicep => policy_definition_deploy_sub.bicep} | 2 ++ ...icep => role_definition_deploy_tenant.bicep} | 4 +++- .../resource/tests/latest/test_resource.py | 6 +++--- 4 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 src/azure-cli/azure/cli/command_modules/resource/tests/latest/policy_definition_deploy_mg.bicep rename src/azure-cli/azure/cli/command_modules/resource/tests/latest/{policy_definition_deploy.bicep => policy_definition_deploy_sub.bicep} (90%) rename src/azure-cli/azure/cli/command_modules/resource/tests/latest/{role_definition_deploy.bicep => role_definition_deploy_tenant.bicep} (94%) diff --git a/src/azure-cli/azure/cli/command_modules/resource/tests/latest/policy_definition_deploy_mg.bicep b/src/azure-cli/azure/cli/command_modules/resource/tests/latest/policy_definition_deploy_mg.bicep new file mode 100644 index 00000000000..2e6b0abfe3b --- /dev/null +++ b/src/azure-cli/azure/cli/command_modules/resource/tests/latest/policy_definition_deploy_mg.bicep @@ -0,0 +1,17 @@ +targetScope = 'managementGroup' + +resource policyDef 'Microsoft.Authorization/policyDefinitions@2018-05-01' = { + name: 'policy-for-bicep-test' + properties: { + policyType: 'Custom' + policyRule: { + if: { + field: 'location' + equals: 'westus2' + } + then: { + effect: 'deny' + } + } + } +} diff --git a/src/azure-cli/azure/cli/command_modules/resource/tests/latest/policy_definition_deploy.bicep b/src/azure-cli/azure/cli/command_modules/resource/tests/latest/policy_definition_deploy_sub.bicep similarity index 90% rename from src/azure-cli/azure/cli/command_modules/resource/tests/latest/policy_definition_deploy.bicep rename to src/azure-cli/azure/cli/command_modules/resource/tests/latest/policy_definition_deploy_sub.bicep index bf2f0906427..dd6faab2519 100644 --- a/src/azure-cli/azure/cli/command_modules/resource/tests/latest/policy_definition_deploy.bicep +++ b/src/azure-cli/azure/cli/command_modules/resource/tests/latest/policy_definition_deploy_sub.bicep @@ -1,3 +1,5 @@ +targetScope = 'subscription' + resource policyDef 'Microsoft.Authorization/policyDefinitions@2018-05-01' = { name: 'policy-for-bicep-test' properties: { diff --git a/src/azure-cli/azure/cli/command_modules/resource/tests/latest/role_definition_deploy.bicep b/src/azure-cli/azure/cli/command_modules/resource/tests/latest/role_definition_deploy_tenant.bicep similarity index 94% rename from src/azure-cli/azure/cli/command_modules/resource/tests/latest/role_definition_deploy.bicep rename to src/azure-cli/azure/cli/command_modules/resource/tests/latest/role_definition_deploy_tenant.bicep index 9d4e921487d..9905490ad87 100644 --- a/src/azure-cli/azure/cli/command_modules/resource/tests/latest/role_definition_deploy.bicep +++ b/src/azure-cli/azure/cli/command_modules/resource/tests/latest/role_definition_deploy_tenant.bicep @@ -1,3 +1,5 @@ +targetScope = 'tenant' + resource roleDef 'Microsoft.Authorization/roleDefinitions@2018-01-01-preview' = { name: '0cb07228-4614-4814-ac1a-c4e39793ce58' properties: { @@ -15,4 +17,4 @@ resource roleDef 'Microsoft.Authorization/roleDefinitions@2018-01-01-preview' = '/providers/Microsoft.Management/managementGroups/cli_tenant_level_deployment_mg' ] } -} \ No newline at end of file +} diff --git a/src/azure-cli/azure/cli/command_modules/resource/tests/latest/test_resource.py b/src/azure-cli/azure/cli/command_modules/resource/tests/latest/test_resource.py index 1c9d2637a8c..c1cbe91fe1d 100644 --- a/src/azure-cli/azure/cli/command_modules/resource/tests/latest/test_resource.py +++ b/src/azure-cli/azure/cli/command_modules/resource/tests/latest/test_resource.py @@ -3302,7 +3302,7 @@ def test_resource_group_level_deployment_with_bicep(self): def test_subscription_level_deployment_with_bicep(self): curr_dir = os.path.dirname(os.path.realpath(__file__)) self.kwargs.update({ - 'tf': os.path.join(curr_dir, 'policy_definition_deploy.bicep').replace('\\', '\\\\'), + 'tf': os.path.join(curr_dir, 'policy_definition_deploy_sub.bicep').replace('\\', '\\\\'), }) self.cmd('deployment sub validate --location westus --template-file "{tf}"', checks=[ @@ -3320,7 +3320,7 @@ def test_subscription_level_deployment_with_bicep(self): def test_management_group_level_deployment_with_bicep(self): curr_dir = os.path.dirname(os.path.realpath(__file__)) self.kwargs.update({ - 'tf': os.path.join(curr_dir, 'policy_definition_deploy.bicep').replace('\\', '\\\\'), + 'tf': os.path.join(curr_dir, 'policy_definition_deploy_mg.bicep').replace('\\', '\\\\'), 'mg': self.create_random_name('azure-cli-management', 30) }) @@ -3341,7 +3341,7 @@ def test_management_group_level_deployment_with_bicep(self): def test_tenent_level_deployment_with_bicep(self): curr_dir = os.path.dirname(os.path.realpath(__file__)) self.kwargs.update({ - 'tf': os.path.join(curr_dir, 'role_definition_deploy.bicep').replace('\\', '\\\\') + 'tf': os.path.join(curr_dir, 'role_definition_deploy_tenant.bicep').replace('\\', '\\\\') }) self.cmd('deployment tenant validate --location WestUS --template-file "{tf}"', checks=[ From f943c4a69b874a8fe72c749173790180d4dc73d3 Mon Sep 17 00:00:00 2001 From: Shenglong Li Date: Thu, 1 Apr 2021 17:22:37 -0700 Subject: [PATCH 4/5] Improve bicep version check logic --- .../cli/command_modules/resource/_bicep.py | 54 +++++++++++++++---- .../cli/command_modules/resource/_params.py | 6 +-- .../tests/latest/test_resource_bicep.py | 4 +- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/src/azure-cli/azure/cli/command_modules/resource/_bicep.py b/src/azure-cli/azure/cli/command_modules/resource/_bicep.py index d1ac505a974..d8e4caef7cc 100644 --- a/src/azure-cli/azure/cli/command_modules/resource/_bicep.py +++ b/src/azure-cli/azure/cli/command_modules/resource/_bicep.py @@ -8,15 +8,18 @@ import stat import platform import subprocess +import json -from pathlib import Path +from json.decoder import JSONDecodeError from contextlib import suppress +from datetime import datetime, timedelta import requests import semver from six.moves.urllib.request import urlopen from knack.log import get_logger +from azure.cli.core.api import get_config_dir from azure.cli.core.azclierror import ( FileOperationError, ValidationError, @@ -31,6 +34,11 @@ # See: https://docs.microsoft.com/en-us/azure/azure-resource-manager/templates/template-syntax#template-format _template_schema_pattern = r"https?://schema\.management\.azure\.com/schemas/[0-9a-zA-Z-]+/(?P[a-zA-Z]+)Template\.json#?" # pylint: disable=line-too-long +_config_dir = get_config_dir() +_bicep_installation_dir = os.path.join(_config_dir, "bin") +_bicep_version_check_file_path = os.path.join(_config_dir, "bicepVersionCheck.json") +_bicep_version_check_cache_ttl = timedelta(minutes=10) + _logger = get_logger(__name__) @@ -42,7 +50,7 @@ def validate_bicep_target_scope(template_schema, deployment_scope): ) -def run_bicep_command(args, auto_install=True, check_upgrade=True): +def run_bicep_command(args, auto_install=True, check_version=True): installation_path = _get_bicep_installation_path(platform.system()) installed = os.path.isfile(installation_path) @@ -51,19 +59,25 @@ def run_bicep_command(args, auto_install=True, check_upgrade=True): ensure_bicep_installation(stdout=False) else: raise FileOperationError('Bicep CLI not found. Install it now by running "az bicep install".') - elif check_upgrade: + elif check_version: + latest_release_tag, cache_expired = _load_bicep_version_check_result_from_cache() + with suppress(ClientRequestError): # Checking upgrade should ignore connection issues. # Users may continue using the current installed version. installed_version = _get_bicep_installed_version(installation_path) - latest_release_tag = get_bicep_latest_release_tag() + latest_release_tag = get_bicep_latest_release_tag() if cache_expired else latest_release_tag latest_version = _extract_semver(latest_release_tag) + if installed_version and latest_version and semver.compare(installed_version, latest_version) < 0: _logger.warning( 'A new Bicep release is available: %s. Upgrade now by running "az bicep upgrade".', latest_release_tag, ) + if cache_expired: + _refresh_bicep_version_check_cache(latest_release_tag) + return _run_command(installation_path, args) @@ -124,11 +138,34 @@ def get_bicep_available_release_tags(): def get_bicep_latest_release_tag(): try: response = requests.get("https://api.github.com/repos/Azure/bicep/releases/latest") + response.raise_for_status() return response.json()["tag_name"] except IOError as err: raise ClientRequestError(f"Error while attempting to retrieve the latest Bicep version: {err}.") +def _load_bicep_version_check_result_from_cache(): + try: + with open(_bicep_version_check_file_path, "r") as version_check_file: + version_check_data = json.load(version_check_file) + latest_release_tag = version_check_data["latestReleaseTag"] + last_check_time = datetime.fromisoformat(version_check_data["lastCheckTime"]) + cache_expired = datetime.now() - last_check_time > _bicep_version_check_cache_ttl + + return latest_release_tag, cache_expired + except (IOError, JSONDecodeError): + return None, True + + +def _refresh_bicep_version_check_cache(lastest_release_tag): + with open(_bicep_version_check_file_path, "w+") as version_check_file: + version_check_data = { + "lastCheckTime": datetime.now().isoformat(timespec="microseconds"), + "latestReleaseTag": lastest_release_tag, + } + json.dump(version_check_data, version_check_file) + + def _get_bicep_installed_version(bicep_executable_path): installed_version_output = _run_command(bicep_executable_path, ["--version"]) return _extract_semver(installed_version_output) @@ -150,12 +187,10 @@ def _get_bicep_download_url(system, release_tag): def _get_bicep_installation_path(system): - installation_folder = os.path.join(str(Path.home()), ".azure", "bin") - if system == "Windows": - return os.path.join(installation_folder, "bicep.exe") + return os.path.join(_bicep_installation_dir, "bicep.exe") if system in ("Linux", "Darwin"): - return os.path.join(installation_folder, "bicep") + return os.path.join(_bicep_installation_dir, "bicep") raise ValidationError(f'The platform "{format(system)}" is not supported.') @@ -180,9 +215,8 @@ def _run_command(bicep_installation_path, args): def _template_schema_to_target_scope(template_schema): template_schema_match = re.search(_template_schema_pattern, template_schema) - template_type = template_schema_match.group('templateType') if template_schema_match else None + template_type = template_schema_match.group("templateType") if template_schema_match else None template_type_lower = template_type.lower() if template_type else None - print(template_type) if template_type_lower == "deployment": return "resourceGroup" diff --git a/src/azure-cli/azure/cli/command_modules/resource/_params.py b/src/azure-cli/azure/cli/command_modules/resource/_params.py index 36bc44081b5..23e2b3efd69 100644 --- a/src/azure-cli/azure/cli/command_modules/resource/_params.py +++ b/src/azure-cli/azure/cli/command_modules/resource/_params.py @@ -573,17 +573,17 @@ def load_arguments(self, _): c.argument('resource_group', arg_type=resource_group_name_type) with self.argument_context('bicep build') as c: - c.argument('files', arg_type=CLIArgumentType(options_list=['--file', '-f'], completer=FilesCompleter(), + c.argument('file', arg_type=CLIArgumentType(options_list=['--file', '-f'], completer=FilesCompleter(), type=file_type, help="The path to the Bicep file to build in the file system.")) c.argument('outdir', arg_type=CLIArgumentType(options_list=['--outdir'], completer=DirectoriesCompleter(), help="When set, saves the output at the specified directory.")) c.argument('outfile', arg_type=CLIArgumentType(options_list=['--outfile'], completer=FilesCompleter(), - help="When set, saves the output as the specified file path.")) + help="When set, saves the output as the specified file path.")) c.argument('stdout', arg_type=CLIArgumentType(options_list=['--stdout'], action='store_true', help="When set, prints all output to stdout instead of corresponding files.")) with self.argument_context('bicep decompile') as c: - c.argument('files', arg_type=CLIArgumentType(options_list=['--file', '-f'], completer=FilesCompleter(), + c.argument('file', arg_type=CLIArgumentType(options_list=['--file', '-f'], completer=FilesCompleter(), type=file_type, help="The path to the ARM template to decompile in the file system.")) with self.argument_context('bicep install') as c: diff --git a/src/azure-cli/azure/cli/command_modules/resource/tests/latest/test_resource_bicep.py b/src/azure-cli/azure/cli/command_modules/resource/tests/latest/test_resource_bicep.py index f184aa62adc..e1fe7209793 100644 --- a/src/azure-cli/azure/cli/command_modules/resource/tests/latest/test_resource_bicep.py +++ b/src/azure-cli/azure/cli/command_modules/resource/tests/latest/test_resource_bicep.py @@ -29,7 +29,7 @@ def test_run_bicep_command_raise_error_if_not_installed_and_not_auto_install(sel @mock.patch("azure.cli.command_modules.resource._bicep.get_bicep_latest_release_tag") @mock.patch("azure.cli.command_modules.resource._bicep._get_bicep_installed_version") @mock.patch("os.path.isfile") - def test_run_bicep_command_check_upgrade( + def test_run_bicep_command_check_version( self, isfile_stub, _get_bicep_installed_version_stub, @@ -42,7 +42,7 @@ def test_run_bicep_command_check_upgrade( _get_bicep_installed_version_stub.return_value = "1.0.0" get_bicep_latest_release_tag_stub.return_value = "v2.0.0" - run_bicep_command(["--version"], check_upgrade=True) + run_bicep_command(["--version"], check_version=True) warning_mock.assert_called_once_with( 'A new Bicep release is available: %s. Upgrade now by running "az bicep upgrade".', From 31ed67bd7feeebdcef3d9184303b165c4b6486e0 Mon Sep 17 00:00:00 2001 From: Shenglong Li Date: Thu, 1 Apr 2021 18:03:47 -0700 Subject: [PATCH 5/5] Fix style issue --- src/azure-cli/azure/cli/command_modules/resource/_params.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/azure-cli/azure/cli/command_modules/resource/_params.py b/src/azure-cli/azure/cli/command_modules/resource/_params.py index 23e2b3efd69..f6e4e3198c4 100644 --- a/src/azure-cli/azure/cli/command_modules/resource/_params.py +++ b/src/azure-cli/azure/cli/command_modules/resource/_params.py @@ -574,7 +574,7 @@ def load_arguments(self, _): with self.argument_context('bicep build') as c: c.argument('file', arg_type=CLIArgumentType(options_list=['--file', '-f'], completer=FilesCompleter(), - type=file_type, help="The path to the Bicep file to build in the file system.")) + type=file_type, help="The path to the Bicep file to build in the file system.")) c.argument('outdir', arg_type=CLIArgumentType(options_list=['--outdir'], completer=DirectoriesCompleter(), help="When set, saves the output at the specified directory.")) c.argument('outfile', arg_type=CLIArgumentType(options_list=['--outfile'], completer=FilesCompleter(), @@ -584,7 +584,7 @@ def load_arguments(self, _): with self.argument_context('bicep decompile') as c: c.argument('file', arg_type=CLIArgumentType(options_list=['--file', '-f'], completer=FilesCompleter(), - type=file_type, help="The path to the ARM template to decompile in the file system.")) + type=file_type, help="The path to the ARM template to decompile in the file system.")) with self.argument_context('bicep install') as c: c.argument('version', options_list=['--version', '-v'], help='The version of Bicep CLI to be installed. Default to the latest if not specified.')