[Storage] Add PITR support

src/azure-cli/azure/cli/command_modules/storage/_help.py

@@ -576,6 +576,24 @@
 short-summary: Manage blob metadata.
 """
 
+helps['storage blob restore'] = """
+type: command
+short-summary: Restore blobs in the specified blob ranges.
+examples:
+  - name: Restore blobs in two specified blob ranges. For examples, (container1/blob1, container2/blob2) and (container2/blob3..container2/blob4).
+    text: az storage blob restore --account-name mystorageaccount -g MyResourceGroup -t 2020-02-27T03:59:59Z -r container1/blob1 container2/blob2 -r container2/blob3 container2/blob4
+  - name: Restore blobs in the specified blob ranges from account start to account end.
+    text: az storage blob restore --account-name mystorageaccount -g MyResourceGroup -t 2020-02-27T03:59:59Z -r "" ""
+  - name: Restore blobs in the specified blob range.
+    text: |
+        time=`date -u -d "30 minutes" '+%Y-%m-%dT%H:%MZ'`
+        az storage blob restore --account-name mystorageaccount -g MyResourceGroup -t $time -r container0/blob1 container0/blob2
+  - name: Restore blobs in the specified blob range without wait and query blob restore status with 'az storage account show'.
+    text: |
+        time=`date -u -d "30 minutes" '+%Y-%m-%dT%H:%MZ'`
+        az storage blob restore --account-name mystorageaccount -g MyResourceGroup -t $time -r container0/blob1 container0/blob2 --no-wait
+"""
+
 helps['storage blob service-properties'] = """
 type: group
 short-summary: Manage storage blob service properties.

src/azure-cli/azure/cli/command_modules/storage/_params.py

@@ -319,6 +319,11 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
                    min_api='2018-07-01')
         c.argument('delete_retention_days', type=int, arg_group='Delete Retention Policy',
                    validator=validator_delete_retention_days, min_api='2018-07-01')
+        c.argument('enable_restore_policy', arg_type=get_three_state_flag(), arg_group='Restore Policy',
+                   min_api='2019-06-01', help="Enable blob restore policy when it set to true.")
+        c.argument('restore_days', type=int, arg_group='Restore Policy',
+                   min_api='2019-06-01', help="The number of days for the blob can be restored. It should be greater "
+                   "than zero and less than Delete Retention Days.")
 
     with self.argument_context('storage account generate-sas') as c:
         t_account_permissions = self.get_sdk('common.models#AccountPermissions')
@@ -393,6 +398,17 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
                    help=sas_help.format(get_permission_help_string(t_blob_permissions)),
                    validator=get_permission_validator(t_blob_permissions))
 
+    with self.argument_context('storage blob restore', resource_type=ResourceType.MGMT_STORAGE) as c:
+        from ._validators import BlobRangeAddAction
+        c.argument('blob_ranges', options_list=['--blob-range', '-r'], action=BlobRangeAddAction, nargs='+',
+                   help='Blob ranges to restore. You need to two values to specify start_range and end_range for each '
+                        'blob range, e.g. -r blob1 blob2. Note: Empty means account start as start range value, and '
+                        'means account end for end range.')
+        c.argument('account_name', acct_name_type, id_part=None)
+        c.argument('resource_group_name', required=False, validator=process_resource_group)
+        c.argument('time_to_restore', type=get_datetime_type(True), options_list=['--time-to-restore', '-t'],
+                   help='Restore blob to the specified time, which should be UTC datetime in (Y-m-d\'T\'H:M:S\'Z\').')
+
     with self.argument_context('storage blob update') as c:
         t_blob_content_settings = self.get_sdk('blob.models#ContentSettings')
         c.register_content_settings_argument(t_blob_content_settings, update=True)

src/azure-cli/azure/cli/command_modules/storage/_validators.py

@@ -5,6 +5,8 @@
 
 # pylint: disable=protected-access
 
+import argparse
+
 from knack.util import CLIError
 from knack.log import get_logger
 
@@ -481,7 +483,6 @@ def validate_entity(namespace):
     missing_keys = '{}PartitionKey'.format(missing_keys) \
         if 'PartitionKey' not in keys else missing_keys
     if missing_keys:
-        import argparse
         raise argparse.ArgumentError(
             None, 'incorrect usage: entity requires: {}'.format(missing_keys))
 
@@ -520,7 +521,6 @@ def validate_marker(namespace):
             del marker[key]
             marker[new_key] = val
     if expected_keys:
-        import argparse
         raise argparse.ArgumentError(
             None, 'incorrect usage: marker requires: {}'.format(' '.join(expected_keys)))
 
@@ -908,7 +908,6 @@ def process_file_download_namespace(namespace):
 
 
 def process_metric_update_namespace(namespace):
-    import argparse
     namespace.hour = namespace.hour == 'true'
     namespace.minute = namespace.minute == 'true'
     namespace.api = namespace.api == 'true' if namespace.api else None
@@ -1118,21 +1117,18 @@ def validate_azcopy_remove_arguments(cmd, namespace):
 def as_user_validator(namespace):
     if namespace.as_user:
         if namespace.expiry is None:
-            import argparse
             raise argparse.ArgumentError(
                 None, 'incorrect usage: specify --expiry when as-user is enabled')
 
         expiry = get_datetime_type(False)(namespace.expiry)
 
         from datetime import datetime, timedelta
         if expiry > datetime.utcnow() + timedelta(days=7):
-            import argparse
             raise argparse.ArgumentError(
                 None, 'incorrect usage: --expiry should be within 7 days from now')
 
         if ((not hasattr(namespace, 'token_credential') or namespace.token_credential is None) and
                 (not hasattr(namespace, 'auth_mode') or namespace.auth_mode != 'login')):
-            import argparse
             raise argparse.ArgumentError(
                 None, "incorrect usage: specify '--auth-mode login' when as-user is enabled")
 
@@ -1161,6 +1157,24 @@ def validator_delete_retention_days(namespace):
                 "incorrect usage: '--delete-retention-days' must be less than or equal to 365")
 
 
+# pylint: disable=too-few-public-methods
+class BlobRangeAddAction(argparse._AppendAction):
+    def __call__(self, parser, namespace, values, option_string=None):
+        if not namespace.blob_ranges:
+            namespace.blob_ranges = []
+        if isinstance(values, list):
+            values = ' '.join(values)
+        BlobRange = namespace._cmd.get_models('BlobRestoreRange', resource_type=ResourceType.MGMT_STORAGE)
+        try:
+            start_range, end_range = values.split(' ')
+        except (ValueError, TypeError):
+            raise CLIError('usage error: --blob-range VARIABLE OPERATOR VALUE')
+        namespace.blob_ranges.append(BlobRange(
+            start_range=start_range,
+            end_range=end_range
+        ))
+
+
 def validate_private_endpoint_connection_id(cmd, namespace):
     if namespace.connection_id:
         from azure.cli.core.util import parse_proxy_resource_id

src/azure-cli/azure/cli/command_modules/storage/commands.py

@@ -130,6 +130,11 @@ def get_custom_sdk(custom_module, client_factory, resource_type=ResourceType.DAT
         operations_tmpl='azure.cli.command_modules.storage.operations.account#{}',
         client_factory=cf_mgmt_policy)
 
+    storage_blob_custom_type = CliCommandType(
+        operations_tmpl='azure.cli.command_modules.storage.operations.blob#{}',
+        client_factory=cf_sa,
+        resource_type=ResourceType.MGMT_STORAGE)
+
     with self.command_group('storage account management-policy', management_policy_sdk,
                             resource_type=ResourceType.MGMT_STORAGE, min_api='2018-11-01',
                             custom_command_type=management_policy_custom_type) as g:
@@ -246,6 +251,10 @@ def get_custom_sdk(custom_module, client_factory, resource_type=ResourceType.DAT
         g.storage_custom_command_oauth(
             'copy start-batch', 'storage_blob_copy_batch')
 
+    with self.command_group('storage blob', storage_account_sdk, resource_type=ResourceType.MGMT_STORAGE,
+                            custom_command_type=storage_blob_custom_type) as g:
+        g.custom_command('restore', 'restore_blob_ranges', min_api='2019-06-01', is_preview=True, supports_no_wait=True)
+
     with self.command_group('storage blob incremental-copy',
                             operations_tmpl='azure.multiapi.storage.blob.pageblobservice#PageBlobService.{}',
                             client_factory=page_blob_service_factory,

src/azure-cli/azure/cli/command_modules/storage/operations/account.py

@@ -428,7 +428,7 @@ def update_management_policies(client, resource_group_name, account_name, parame
 
 # TODO: support updating other properties besides 'enable_change_feed,delete_retention_policy'
 def update_blob_service_properties(cmd, instance, enable_change_feed=None, enable_delete_retention=None,
-                                   delete_retention_days=None):
+                                   delete_retention_days=None, enable_restore_policy=None, restore_days=None):
     if enable_change_feed is not None:
         instance.change_feed = cmd.get_models('ChangeFeed')(enabled=enable_change_feed)
 
@@ -437,5 +437,9 @@ def update_blob_service_properties(cmd, instance, enable_change_feed=None, enabl
             delete_retention_days = None
         instance.delete_retention_policy = cmd.get_models('DeleteRetentionPolicy')(
             enabled=enable_delete_retention, days=delete_retention_days)
-
+    if enable_restore_policy is not None:
+        if enable_restore_policy is False:
+            restore_days = None
+        instance.restore_policy = cmd.get_models('RestorePolicyProperties')(
+            enabled=enable_restore_policy, days=restore_days)
     return instance

src/azure-cli/azure/cli/command_modules/storage/operations/blob.py

@@ -6,8 +6,9 @@
 from __future__ import print_function
 
 import os
-from datetime import datetime
-from datetime import timezone
+from datetime import datetime, timezone
+
+from azure.cli.core.util import sdk_no_wait
 from azure.cli.command_modules.storage.url_quote_util import encode_for_url, make_encoded_file_url_and_params
 from azure.cli.command_modules.storage.util import (create_blob_service_from_storage_client,
                                                     create_file_share_from_storage_client,
@@ -30,6 +31,17 @@ def delete_container(client, container_name, fail_not_exist=False, lease_id=None
         if_unmodified_since=if_unmodified_since, timeout=timeout)
 
 
+def restore_blob_ranges(cmd, client, resource_group_name, account_name, time_to_restore, blob_ranges=None,
+                        no_wait=False):
+
+    if blob_ranges is None:
+        BlobRestoreRange = cmd.get_models("BlobRestoreRange")
+        blob_ranges = [BlobRestoreRange(start_range="", end_range="")]
+
+    return sdk_no_wait(no_wait, client.restore_blob_ranges, resource_group_name=resource_group_name,
+                       account_name=account_name, time_to_restore=time_to_restore, blob_ranges=blob_ranges)
+
+
 def set_blob_tier(client, container_name, blob_name, tier, blob_type='block', timeout=None):
     if blob_type == 'block':
         return client.set_standard_blob_tier(container_name=container_name, blob_name=blob_name,

src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_blob_live_scenarios.py

@@ -4,6 +4,7 @@
 # --------------------------------------------------------------------------------------------
 
 import os
+from datetime import datetime, timedelta
 from azure.cli.testsdk import (LiveScenarioTest, ResourceGroupPreparer, StorageAccountPreparer,
                                JMESPathCheck, JMESPathCheckExists, NoneCheck, api_version_constraint)
 from azure.cli.core.profiles import ResourceType
@@ -84,3 +85,79 @@ def verify_blob_upload_and_download(self, group, account, file_size_kb, blob_typ
             self.assertTrue(os.path.isfile(downloaded), 'The file is not downloaded.')
             self.assertEqual(file_size_kb * 1024, os.stat(downloaded).st_size,
                              'The download file size is not right.')
+
+    @ResourceGroupPreparer(name_prefix="storage_blob_restore")
+    @StorageAccountPreparer(name_prefix="storageblobrestore", kind="StorageV2", location="eastus2euap",
+                            sku='Standard_LRS')
+    def test_storage_blob_restore(self, resource_group, storage_account):
+        import time
+        self.cmd('storage account blob-service-properties update --enable-change-feed --enable-delete-retention --delete-retention-days 2 -n {sa}')\
+            .assert_with_checks(JMESPathCheck('changeFeed.enabled', True),
+                                JMESPathCheck('deleteRetentionPolicy.enabled', True),
+                                JMESPathCheck('deleteRetentionPolicy.days', 2))
+        time.sleep(60)
+        # Enable Restore Policy
+        self.cmd('storage account blob-service-properties update --enable-restore-policy --restore-days 1 -n {sa}')\
+            .assert_with_checks(JMESPathCheck('restorePolicy.enabled', True),
+                                JMESPathCheck('restorePolicy.days', 1))
+
+        c1 = self.create_random_name(prefix='containera', length=24)
+        c2 = self.create_random_name(prefix='containerb', length=24)
+        b1 = self.create_random_name(prefix='blob1', length=24)
+        b2 = self.create_random_name(prefix='blob2', length=24)
+        b3 = self.create_random_name(prefix='blob3', length=24)
+        b4 = self.create_random_name(prefix='blob4', length=24)
+
+        local_file = self.create_temp_file(256)
+
+        account_key = self.cmd('storage account keys list -n {} -g {} --query "[0].value" -otsv'
+                               .format(storage_account, resource_group)).output
+
+        # Prepare containers and blobs
+        for container in [c1, c2]:
+            self.cmd('storage container create -n {} --account-name {} --account-key {}'.format(
+                container, storage_account, account_key)) \
+                .assert_with_checks(JMESPathCheck('created', True))
+            for blob in [b1, b2, b3, b4]:
+                self.cmd('storage blob upload -c {} -f "{}" -n {} --account-name {} --account-key {}'.format(
+                    container, local_file, blob, storage_account, account_key))
+            self.cmd('storage blob list -c {} --account-name {} --account-key {}'.format(
+                container, storage_account, account_key)) \
+                .assert_with_checks(JMESPathCheck('length(@)', 4))
+
+            self.cmd('storage container delete -n {} --account-name {} --account-key {}'.format(
+                container, storage_account, account_key)) \
+                .assert_with_checks(JMESPathCheck('deleted', True))
+
+        time.sleep(10)
+
+        # Restore blobs, with specific ranges
+        time_to_restore = (datetime.utcnow() + timedelta(seconds=-5)).strftime('%Y-%m-%dT%H:%MZ')
+
+        # c1/b1 -> c1/b2
+        start_range = '/'.join([c1, b1])
+        end_range = '/'.join([c1, b2])
+        self.cmd('storage blob restore -t {} -r {} {} --account-name {} -g {}'.format(
+            time_to_restore, start_range, end_range, storage_account, resource_group), checks=[
+            JMESPathCheck('status', 'Complete'),
+            JMESPathCheck('parameters.blobRanges[0].startRange', start_range),
+            JMESPathCheck('parameters.blobRanges[0].endRange', end_range)])
+
+        self.cmd('storage blob restore -t {} -r {} {} --account-name {} -g {} --no-wait'.format(
+            time_to_restore, start_range, end_range, storage_account, resource_group))
+
+        time.sleep(90)
+
+        time_to_restore = (datetime.utcnow() + timedelta(seconds=-5)).strftime('%Y-%m-%dT%H:%MZ')
+        # c1/b2 -> c2/b3
+        start_range = '/'.join([c1, b2])
+        end_range = '/'.join([c2, b3])
+        self.cmd('storage blob restore -t {} -r {} {} --account-name {} -g {}'.format(
+            time_to_restore, start_range, end_range, storage_account, resource_group), checks=[
+            JMESPathCheck('status', 'Complete'),
+            JMESPathCheck('parameters.blobRanges[0].startRange', start_range),
+            JMESPathCheck('parameters.blobRanges[0].endRange', end_range)])
+
+        time.sleep(90)
+        self.cmd('storage blob restore -t {} --account-name {} -g {} --no-wait'.format(
+            time_to_restore, storage_account, resource_group))