diff --git a/src/azure-cli/azure/cli/command_modules/storage/_help.py b/src/azure-cli/azure/cli/command_modules/storage/_help.py index d4d1c76fd99..35de0c48d00 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/_help.py +++ b/src/azure-cli/azure/cli/command_modules/storage/_help.py @@ -576,6 +576,24 @@ short-summary: Manage blob metadata. """ +helps['storage blob restore'] = """ +type: command +short-summary: Restore blobs in the specified blob ranges. +examples: + - name: Restore blobs in two specified blob ranges. For examples, (container1/blob1, container2/blob2) and (container2/blob3..container2/blob4). + text: az storage blob restore --account-name mystorageaccount -g MyResourceGroup -t 2020-02-27T03:59:59Z -r container1/blob1 container2/blob2 -r container2/blob3 container2/blob4 + - name: Restore blobs in the specified blob ranges from account start to account end. + text: az storage blob restore --account-name mystorageaccount -g MyResourceGroup -t 2020-02-27T03:59:59Z -r "" "" + - name: Restore blobs in the specified blob range. + text: | + time=`date -u -d "30 minutes" '+%Y-%m-%dT%H:%MZ'` + az storage blob restore --account-name mystorageaccount -g MyResourceGroup -t $time -r container0/blob1 container0/blob2 + - name: Restore blobs in the specified blob range without wait and query blob restore status with 'az storage account show'. + text: | + time=`date -u -d "30 minutes" '+%Y-%m-%dT%H:%MZ'` + az storage blob restore --account-name mystorageaccount -g MyResourceGroup -t $time -r container0/blob1 container0/blob2 --no-wait +""" + helps['storage blob service-properties'] = """ type: group short-summary: Manage storage blob service properties. diff --git a/src/azure-cli/azure/cli/command_modules/storage/_params.py b/src/azure-cli/azure/cli/command_modules/storage/_params.py index 94de59b35f7..9a5108b7671 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/_params.py +++ b/src/azure-cli/azure/cli/command_modules/storage/_params.py @@ -319,6 +319,11 @@ def load_arguments(self, _): # pylint: disable=too-many-locals, too-many-statem min_api='2018-07-01') c.argument('delete_retention_days', type=int, arg_group='Delete Retention Policy', validator=validator_delete_retention_days, min_api='2018-07-01') + c.argument('enable_restore_policy', arg_type=get_three_state_flag(), arg_group='Restore Policy', + min_api='2019-06-01', help="Enable blob restore policy when it set to true.") + c.argument('restore_days', type=int, arg_group='Restore Policy', + min_api='2019-06-01', help="The number of days for the blob can be restored. It should be greater " + "than zero and less than Delete Retention Days.") with self.argument_context('storage account generate-sas') as c: t_account_permissions = self.get_sdk('common.models#AccountPermissions') @@ -393,6 +398,17 @@ def load_arguments(self, _): # pylint: disable=too-many-locals, too-many-statem help=sas_help.format(get_permission_help_string(t_blob_permissions)), validator=get_permission_validator(t_blob_permissions)) + with self.argument_context('storage blob restore', resource_type=ResourceType.MGMT_STORAGE) as c: + from ._validators import BlobRangeAddAction + c.argument('blob_ranges', options_list=['--blob-range', '-r'], action=BlobRangeAddAction, nargs='+', + help='Blob ranges to restore. You need to two values to specify start_range and end_range for each ' + 'blob range, e.g. -r blob1 blob2. Note: Empty means account start as start range value, and ' + 'means account end for end range.') + c.argument('account_name', acct_name_type, id_part=None) + c.argument('resource_group_name', required=False, validator=process_resource_group) + c.argument('time_to_restore', type=get_datetime_type(True), options_list=['--time-to-restore', '-t'], + help='Restore blob to the specified time, which should be UTC datetime in (Y-m-d\'T\'H:M:S\'Z\').') + with self.argument_context('storage blob update') as c: t_blob_content_settings = self.get_sdk('blob.models#ContentSettings') c.register_content_settings_argument(t_blob_content_settings, update=True) diff --git a/src/azure-cli/azure/cli/command_modules/storage/_validators.py b/src/azure-cli/azure/cli/command_modules/storage/_validators.py index deac9150ebd..d650f3c0f52 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/_validators.py +++ b/src/azure-cli/azure/cli/command_modules/storage/_validators.py @@ -5,6 +5,8 @@ # pylint: disable=protected-access +import argparse + from knack.util import CLIError from knack.log import get_logger @@ -481,7 +483,6 @@ def validate_entity(namespace): missing_keys = '{}PartitionKey'.format(missing_keys) \ if 'PartitionKey' not in keys else missing_keys if missing_keys: - import argparse raise argparse.ArgumentError( None, 'incorrect usage: entity requires: {}'.format(missing_keys)) @@ -520,7 +521,6 @@ def validate_marker(namespace): del marker[key] marker[new_key] = val if expected_keys: - import argparse raise argparse.ArgumentError( None, 'incorrect usage: marker requires: {}'.format(' '.join(expected_keys))) @@ -908,7 +908,6 @@ def process_file_download_namespace(namespace): def process_metric_update_namespace(namespace): - import argparse namespace.hour = namespace.hour == 'true' namespace.minute = namespace.minute == 'true' namespace.api = namespace.api == 'true' if namespace.api else None @@ -1118,7 +1117,6 @@ def validate_azcopy_remove_arguments(cmd, namespace): def as_user_validator(namespace): if namespace.as_user: if namespace.expiry is None: - import argparse raise argparse.ArgumentError( None, 'incorrect usage: specify --expiry when as-user is enabled') @@ -1126,13 +1124,11 @@ def as_user_validator(namespace): from datetime import datetime, timedelta if expiry > datetime.utcnow() + timedelta(days=7): - import argparse raise argparse.ArgumentError( None, 'incorrect usage: --expiry should be within 7 days from now') if ((not hasattr(namespace, 'token_credential') or namespace.token_credential is None) and (not hasattr(namespace, 'auth_mode') or namespace.auth_mode != 'login')): - import argparse raise argparse.ArgumentError( None, "incorrect usage: specify '--auth-mode login' when as-user is enabled") @@ -1161,6 +1157,24 @@ def validator_delete_retention_days(namespace): "incorrect usage: '--delete-retention-days' must be less than or equal to 365") +# pylint: disable=too-few-public-methods +class BlobRangeAddAction(argparse._AppendAction): + def __call__(self, parser, namespace, values, option_string=None): + if not namespace.blob_ranges: + namespace.blob_ranges = [] + if isinstance(values, list): + values = ' '.join(values) + BlobRange = namespace._cmd.get_models('BlobRestoreRange', resource_type=ResourceType.MGMT_STORAGE) + try: + start_range, end_range = values.split(' ') + except (ValueError, TypeError): + raise CLIError('usage error: --blob-range VARIABLE OPERATOR VALUE') + namespace.blob_ranges.append(BlobRange( + start_range=start_range, + end_range=end_range + )) + + def validate_private_endpoint_connection_id(cmd, namespace): if namespace.connection_id: from azure.cli.core.util import parse_proxy_resource_id diff --git a/src/azure-cli/azure/cli/command_modules/storage/commands.py b/src/azure-cli/azure/cli/command_modules/storage/commands.py index 22da8dfdaba..9b45cdcf9bd 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/commands.py +++ b/src/azure-cli/azure/cli/command_modules/storage/commands.py @@ -130,6 +130,11 @@ def get_custom_sdk(custom_module, client_factory, resource_type=ResourceType.DAT operations_tmpl='azure.cli.command_modules.storage.operations.account#{}', client_factory=cf_mgmt_policy) + storage_blob_custom_type = CliCommandType( + operations_tmpl='azure.cli.command_modules.storage.operations.blob#{}', + client_factory=cf_sa, + resource_type=ResourceType.MGMT_STORAGE) + with self.command_group('storage account management-policy', management_policy_sdk, resource_type=ResourceType.MGMT_STORAGE, min_api='2018-11-01', custom_command_type=management_policy_custom_type) as g: @@ -246,6 +251,10 @@ def get_custom_sdk(custom_module, client_factory, resource_type=ResourceType.DAT g.storage_custom_command_oauth( 'copy start-batch', 'storage_blob_copy_batch') + with self.command_group('storage blob', storage_account_sdk, resource_type=ResourceType.MGMT_STORAGE, + custom_command_type=storage_blob_custom_type) as g: + g.custom_command('restore', 'restore_blob_ranges', min_api='2019-06-01', is_preview=True, supports_no_wait=True) + with self.command_group('storage blob incremental-copy', operations_tmpl='azure.multiapi.storage.blob.pageblobservice#PageBlobService.{}', client_factory=page_blob_service_factory, diff --git a/src/azure-cli/azure/cli/command_modules/storage/operations/account.py b/src/azure-cli/azure/cli/command_modules/storage/operations/account.py index 7254abf7e26..529e98292b0 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/operations/account.py +++ b/src/azure-cli/azure/cli/command_modules/storage/operations/account.py @@ -428,7 +428,7 @@ def update_management_policies(client, resource_group_name, account_name, parame # TODO: support updating other properties besides 'enable_change_feed,delete_retention_policy' def update_blob_service_properties(cmd, instance, enable_change_feed=None, enable_delete_retention=None, - delete_retention_days=None): + delete_retention_days=None, enable_restore_policy=None, restore_days=None): if enable_change_feed is not None: instance.change_feed = cmd.get_models('ChangeFeed')(enabled=enable_change_feed) @@ -437,5 +437,9 @@ def update_blob_service_properties(cmd, instance, enable_change_feed=None, enabl delete_retention_days = None instance.delete_retention_policy = cmd.get_models('DeleteRetentionPolicy')( enabled=enable_delete_retention, days=delete_retention_days) - + if enable_restore_policy is not None: + if enable_restore_policy is False: + restore_days = None + instance.restore_policy = cmd.get_models('RestorePolicyProperties')( + enabled=enable_restore_policy, days=restore_days) return instance diff --git a/src/azure-cli/azure/cli/command_modules/storage/operations/blob.py b/src/azure-cli/azure/cli/command_modules/storage/operations/blob.py index c811f7f7076..455b2c79494 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/operations/blob.py +++ b/src/azure-cli/azure/cli/command_modules/storage/operations/blob.py @@ -6,8 +6,9 @@ from __future__ import print_function import os -from datetime import datetime -from datetime import timezone +from datetime import datetime, timezone + +from azure.cli.core.util import sdk_no_wait from azure.cli.command_modules.storage.url_quote_util import encode_for_url, make_encoded_file_url_and_params from azure.cli.command_modules.storage.util import (create_blob_service_from_storage_client, create_file_share_from_storage_client, @@ -30,6 +31,17 @@ def delete_container(client, container_name, fail_not_exist=False, lease_id=None if_unmodified_since=if_unmodified_since, timeout=timeout) +def restore_blob_ranges(cmd, client, resource_group_name, account_name, time_to_restore, blob_ranges=None, + no_wait=False): + + if blob_ranges is None: + BlobRestoreRange = cmd.get_models("BlobRestoreRange") + blob_ranges = [BlobRestoreRange(start_range="", end_range="")] + + return sdk_no_wait(no_wait, client.restore_blob_ranges, resource_group_name=resource_group_name, + account_name=account_name, time_to_restore=time_to_restore, blob_ranges=blob_ranges) + + def set_blob_tier(client, container_name, blob_name, tier, blob_type='block', timeout=None): if blob_type == 'block': return client.set_standard_blob_tier(container_name=container_name, blob_name=blob_name, diff --git a/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_blob_live_scenarios.py b/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_blob_live_scenarios.py index 60908c2df19..f3ae351c19b 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_blob_live_scenarios.py +++ b/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_blob_live_scenarios.py @@ -4,6 +4,7 @@ # -------------------------------------------------------------------------------------------- import os +from datetime import datetime, timedelta from azure.cli.testsdk import (LiveScenarioTest, ResourceGroupPreparer, StorageAccountPreparer, JMESPathCheck, JMESPathCheckExists, NoneCheck, api_version_constraint) from azure.cli.core.profiles import ResourceType @@ -84,3 +85,79 @@ def verify_blob_upload_and_download(self, group, account, file_size_kb, blob_typ self.assertTrue(os.path.isfile(downloaded), 'The file is not downloaded.') self.assertEqual(file_size_kb * 1024, os.stat(downloaded).st_size, 'The download file size is not right.') + + @ResourceGroupPreparer(name_prefix="storage_blob_restore") + @StorageAccountPreparer(name_prefix="storageblobrestore", kind="StorageV2", location="eastus2euap", + sku='Standard_LRS') + def test_storage_blob_restore(self, resource_group, storage_account): + import time + self.cmd('storage account blob-service-properties update --enable-change-feed --enable-delete-retention --delete-retention-days 2 -n {sa}')\ + .assert_with_checks(JMESPathCheck('changeFeed.enabled', True), + JMESPathCheck('deleteRetentionPolicy.enabled', True), + JMESPathCheck('deleteRetentionPolicy.days', 2)) + time.sleep(60) + # Enable Restore Policy + self.cmd('storage account blob-service-properties update --enable-restore-policy --restore-days 1 -n {sa}')\ + .assert_with_checks(JMESPathCheck('restorePolicy.enabled', True), + JMESPathCheck('restorePolicy.days', 1)) + + c1 = self.create_random_name(prefix='containera', length=24) + c2 = self.create_random_name(prefix='containerb', length=24) + b1 = self.create_random_name(prefix='blob1', length=24) + b2 = self.create_random_name(prefix='blob2', length=24) + b3 = self.create_random_name(prefix='blob3', length=24) + b4 = self.create_random_name(prefix='blob4', length=24) + + local_file = self.create_temp_file(256) + + account_key = self.cmd('storage account keys list -n {} -g {} --query "[0].value" -otsv' + .format(storage_account, resource_group)).output + + # Prepare containers and blobs + for container in [c1, c2]: + self.cmd('storage container create -n {} --account-name {} --account-key {}'.format( + container, storage_account, account_key)) \ + .assert_with_checks(JMESPathCheck('created', True)) + for blob in [b1, b2, b3, b4]: + self.cmd('storage blob upload -c {} -f "{}" -n {} --account-name {} --account-key {}'.format( + container, local_file, blob, storage_account, account_key)) + self.cmd('storage blob list -c {} --account-name {} --account-key {}'.format( + container, storage_account, account_key)) \ + .assert_with_checks(JMESPathCheck('length(@)', 4)) + + self.cmd('storage container delete -n {} --account-name {} --account-key {}'.format( + container, storage_account, account_key)) \ + .assert_with_checks(JMESPathCheck('deleted', True)) + + time.sleep(10) + + # Restore blobs, with specific ranges + time_to_restore = (datetime.utcnow() + timedelta(seconds=-5)).strftime('%Y-%m-%dT%H:%MZ') + + # c1/b1 -> c1/b2 + start_range = '/'.join([c1, b1]) + end_range = '/'.join([c1, b2]) + self.cmd('storage blob restore -t {} -r {} {} --account-name {} -g {}'.format( + time_to_restore, start_range, end_range, storage_account, resource_group), checks=[ + JMESPathCheck('status', 'Complete'), + JMESPathCheck('parameters.blobRanges[0].startRange', start_range), + JMESPathCheck('parameters.blobRanges[0].endRange', end_range)]) + + self.cmd('storage blob restore -t {} -r {} {} --account-name {} -g {} --no-wait'.format( + time_to_restore, start_range, end_range, storage_account, resource_group)) + + time.sleep(90) + + time_to_restore = (datetime.utcnow() + timedelta(seconds=-5)).strftime('%Y-%m-%dT%H:%MZ') + # c1/b2 -> c2/b3 + start_range = '/'.join([c1, b2]) + end_range = '/'.join([c2, b3]) + self.cmd('storage blob restore -t {} -r {} {} --account-name {} -g {}'.format( + time_to_restore, start_range, end_range, storage_account, resource_group), checks=[ + JMESPathCheck('status', 'Complete'), + JMESPathCheck('parameters.blobRanges[0].startRange', start_range), + JMESPathCheck('parameters.blobRanges[0].endRange', end_range)]) + + time.sleep(90) + self.cmd('storage blob restore -t {} --account-name {} -g {} --no-wait'.format( + time_to_restore, storage_account, resource_group))