[Storage] Fix #10620 in azure cli to support ADLS #1038

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged

Juliehzl merged 46 commits into Azure:master from Juliehzl:ADLS

Oct 31, 2019

src/storage-preview/HISTORY.rst

-Original file line number
+Diff line change
@@ Expand Up / @@ -3,6 +3,11 @@ @@
      Release History
     ===============
+.2.9 (2019-10-31)
+    ++++++++++++++++
+    * Integrate Azcopy v10.3.1
+    * Add `az storage blob directory` command group
 .2.8 (2019-7-5)
     ++++++++++++++++
     * Remove file-add command argument
@@ Expand Down @@

src/storage-preview/azext_storage_preview/__init__.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -8,14 +8,15 @@ @@
     from azure.cli.core.commands import AzCommandGroup, AzArgumentContext
     import azext_storage_preview._help  # pylint: disable=unused-import
-    from .profiles import CUSTOM_DATA_STORAGE, CUSTOM_MGMT_STORAGE, CUSTOM_MGMT_PREVIEW_STORAGE
+    from .profiles import CUSTOM_DATA_STORAGE, CUSTOM_MGMT_STORAGE, CUSTOM_MGMT_PREVIEW_STORAGE, CUSTOM_DATA_STORAGE_ADLS
     class StorageCommandsLoader(AzCommandsLoader):
         def __init__(self, cli_ctx=None):
             from azure.cli.core.commands import CliCommandType
             register_resource_type('latest', CUSTOM_DATA_STORAGE, '2018-03-28')
+            register_resource_type('latest', CUSTOM_DATA_STORAGE_ADLS, '2019-02-02-preview')
             register_resource_type('latest', CUSTOM_MGMT_STORAGE, '2018-07-01')
             register_resource_type('latest', CUSTOM_MGMT_PREVIEW_STORAGE, '2018-03-01-preview')
             storage_custom = CliCommandType(operations_tmpl='azext_storage_preview.custom#{}')
@@ Expand Down @@

src/storage-preview/azext_storage_preview/_client_factory.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -97,6 +97,19 @@ def blob_data_service_factory(cli_ctx, kwargs): @@
                                             token_credential=kwargs.pop('token_credential', None))
+    def adls_blob_data_service_factory(cli_ctx, kwargs):
+        from .sdkutil import get_adls_blob_service_by_type
+        blob_type = kwargs.get('blob_type')
+        blob_service = get_adls_blob_service_by_type(cli_ctx, blob_type) or get_adls_blob_service_by_type(cli_ctx, 'block')
+        return generic_data_service_factory(cli_ctx, blob_service, kwargs.pop('account_name', None),
+                                            kwargs.pop('account_key', None),
+                                            connection_string=kwargs.pop('connection_string', None),
+                                            sas_token=kwargs.pop('sas_token', None),
+                                            socket_timeout=kwargs.pop('socket_timeout', None),
+                                            token_credential=kwargs.pop('token_credential', None))
     def cloud_storage_account_service_factory(cli_ctx, kwargs):
         t_cloud_storage_account = get_sdk(cli_ctx, CUSTOM_DATA_STORAGE, 'common#CloudStorageAccount')
         account_name = kwargs.pop('account_name', None)
@@ Expand Down @@

src/storage-preview/azext_storage_preview/_format.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -52,6 +52,7 @@ def transform_container_show(result): @@
     def transform_blob_output(result):
         return build_table_output(result, [
             ('Name', 'name'),
+            ('IsDirectory', 'metadata.hdi_isfolder'),
             ('Blob Type', 'properties.blobType'),
             ('Blob Tier', 'properties.blobTier'),
             ('Length', 'properties.contentLength'),
@@ Expand Down @@

src/storage-preview/azext_storage_preview/_help.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -111,3 +111,141 @@ @@
         type: command
         short-summary: Run a command directly using the AzCopy CLI. Please use SAS tokens for authentication.
     """
+    helps['storage blob directory'] = """
+        type: group
+        short-summary: Manage blob directories in storage account container.
+        long-summary: To use the directory commands, please make sure your storage account type is StorageV2.
+    """
+    helps['storage blob directory access'] = """
+        type: group
+        short-summary: Manage the access control properties of a directory when Hierarchical Namespace is enabled
+    """
+    helps['storage blob directory access set'] = """
+        type: command
+        short-summary: Set the access control properties of a directory.
+        examples:
+            - name: Set the access control properties of a directory.
+              text: az storage blob directory access set -a "user::rwx,group::r--,other::---" -d MyDirectoryPath -c MyContainer --account-name MyStorageAccount
+    """
+    helps['storage blob directory access show'] = """
+        type: command
+        short-summary: Show the access control properties of a directory.
+        examples:
+            - name: Show the access control properties of a directory.
+              text: az storage blob directory access show -d MyDirectoryPath -c MyContainer --account-name MyStorageAccount
+    """
+    helps['storage blob directory access update'] = """
+        type: command
+        short-summary: Update the access control properties of a directory.
+        examples:
+            - name: Update the access control properties of a directory.
+              text: az storage blob directory access update --permissions "rwxrwxrwx" -d MyDirectoryPath -c MyContainer --account-name MyStorageAccount
+    """
+    helps['storage blob directory create'] = """
+        type: command
+        short-summary: Create a storage blob directory in a storage container.
+        long-summary: Create a storage blob directory which can contain other directories or blobs in a storage container.
+        examples:
+            - name: Create a storage blob directory in a storage container.
+              text: az storage blob directory create -c MyContainer -d MyDirectoryPath --account-name MyStorageAccount
+    """
+    helps['storage blob directory delete'] = """
+        type: command
+        short-summary: Delete a storage blob directory in a storage container.
+        long-summary: >
+            This operation's behavior is different depending on whether Hierarchical Namespace
+            is enabled; if yes, then the delete operation can be atomic and instantaneous;
+            if not, the operation is performed in batches and a continuation token could be returned.
+        examples:
+            - name: Delete a storage blob directory in a storage container.
+              text: az storage blob directory delete -c MyContainer -d MyDirectoryPath --account-name MyStorageAccount
+    """
+    helps['storage blob directory download'] = """
+        type: command
+        short-summary: Download blobs to a local file path.
+        examples:
+            - name: Download a single blob in a storage blob directory.
+              text: az storage blob directory download -c MyContainer --account-name MyStorageAccount -s "path/to/blob" -d "<local-path>"
+            - name: Download the entire directory in a storage container.
+              text: az storage blob directory download -c MyContainer --account-name MyStorageAccount -s SourceDirectoryPath -d "<local-path>" --recursive
+            - name: Download an entire subdirectory of a storage blob directory.
+              text: az storage blob directory download -c MyContainer --account-name MyStorageAccount -s "path/to/subdirectory" -d "<local-path>" --recursive
+    """
+    helps['storage blob directory exists'] = """
+        type: command
+        short-summary: Check for the existence of a blob directory in a storage container.
+        examples:
+            - name: Check for the existence of a blob directory in a storage container.
+              text: az storage blob directory exists -c MyContainer -d MyDirectoryPath --account-name MyStorageAccount
+    """
+    helps['storage blob directory list'] = """
+        type: command
+        short-summary: List blobs and blob subdirectories in a storage directory.
+        examples:
+            - name: List blobs and blob subdirectories in a storage directory.
+              text: az storage blob directory list -c MyContainer -d DestinationDirectoryPath --account-name MyStorageAccount
+    """
+    helps['storage blob directory metadata'] = """
+        type: group
+        short-summary: Manage directory metadata.
+    """
+    helps['storage blob directory metadata show'] = """
+        type: command
+        short-summary: Show all user-defined metadata for the specified blob directory.
+        examples:
+            - name: Show all user-defined metadata for the specified blob directory.
+              text: az storage blob directory metadata show -c MyContainer -d MyDirectoryPath --account-name MyStorageAccount
+    """
+    helps['storage blob directory metadata update'] = """
+        type: command
+        short-summary: Set user-defined metadata for the specified blob directory as one or more name-value pairs.
+        examples:
+            - name: Set user-defined metadata for the specified blob directory as one or more name-value pairs.
+              text: az storage blob directory metadata update --metadata tag1=value1 -c MyContainer -d MyDirectoryPath --account-name MyStorageAccount
+    """
+    helps['storage blob directory move'] = """
+        type: command
+        short-summary: Move a storage directory to another storage blob directory in a storage container.
+        long-summary: >
+            Move a storage directory and all its content (which can contain other directories or blobs) to another storage
+            blob directory in a storage container. This operation's behavior is different depending on whether Hierarchical
+            Namespace is enabled; if yes, the move operation is atomic and no marker is returned; if not, the operation is
+            performed in batches and a continuation token could be returned.
+        examples:
+            - name: Move a storage directory to another storage blob directory in a storage container.
+              text: az storage blob directory move -c MyContainer -d DestinationDirectoryPath -s SourceDirectoryPath --account-name MyStorageAccount
+    """
+    helps['storage blob directory show'] = """
+        type: command
+        short-summary: Show a storage blob directory properties in a storage container.
+        examples:
+            - name: Show a storage blob directory properties in a storage container.
+              text: az storage blob show -c MyContainer -d MyDirectoryPath --account-name MyStorageAccount
+    """
+    helps['storage blob directory upload'] = """
+        type: command
+        short-summary: Upload blobs to a storage blob directory.
+        examples:
+            - name: Upload a single blob to a storage blob directory.
+              text: az storage blob directory upload -c MyContainer --account-name MyStorageAccount -s "path/to/file" -d directory
+            - name: Upload a directory to a container.
+              text: az storage blob directory upload -c MyContainer --account-name MyStorageAccount -s "path/to/directory" -d directory --recursive
+            - name: Upload the contents of a directory to a container.
+              text: az storage blob directory upload -c MyContainer --account-name MyStorageAccount -s "path/to/directory/*" -d directory --recursive
+    """

src/storage-preview/azext_storage_preview/_params.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -10,7 +10,8 @@ @@
     from ._validators import (get_datetime_type, validate_metadata, validate_custom_domain, process_resource_group,
                               validate_bypass, validate_encryption_source, storage_account_key_options, validate_key,
                               validate_azcopy_upload_destination_url, validate_azcopy_download_source_url,
-                              validate_azcopy_target_url)
+                              validate_azcopy_target_url, validate_included_datasets,
+                              validate_blob_directory_download_source_url, validate_blob_directory_upload_destination_url)
     from .profiles import CUSTOM_MGMT_STORAGE
@@ Expand All @@
                                          completer=get_resource_name_completion_list('Microsoft.Storage/storageAccounts'))
         container_name_type = CLIArgumentType(options_list=['--container-name', '-c'], help='The container name.',
                                               completer=get_container_name_completions)
-        directory_type = CLIArgumentType(options_list=['--directory-name', '-d'], help='The directory name.',
-                                         completer=get_storage_name_completion_list(t_file_service,
-                                                                                    'list_directories_and_files',
-                                                                                    parent='share_name'))
+        directory_path_type = CLIArgumentType(options_list=['--directory-path', '-d'], help='The directory path name.',
+                                              parent='container_name')
         share_name_type = CLIArgumentType(options_list=['--share-name', '-s'], help='The file share name.',
                                           completer=get_storage_name_completion_list(t_file_service, 'list_shares'))
         table_name_type = CLIArgumentType(options_list=['--table-name', '-t'],
                                           completer=get_storage_name_completion_list(t_table_service, 'list_tables'))
         with self.argument_context('storage') as c:
             c.argument('container_name', container_name_type)
-            c.argument('directory_name', directory_type)
             c.argument('share_name', share_name_type)
             c.argument('table_name', table_name_type)
             c.argument('retry_wait', options_list=('--retry-interval',))
@@ Expand Down Expand Up @@
         with self.argument_context('storage azcopy run-command') as c:
             c.positional('command_args', help='Command to run using azcopy. Please start commands with "azcopy ".')
+        # New commands parameters for ADLS Gen2
+        with self.argument_context('storage blob directory') as c:
+            c.argument('directory_path', directory_path_type)
+            c.argument('container_name', container_name_type)
+        with self.argument_context('storage blob directory access') as c:
+            c.argument('path', directory_path_type)
+        with self.argument_context('storage blob directory access set') as c:
+            c.argument('acl', options_list=['--acl-spec', '-a'], required=True,
+                       help='The ACL specification to set on the path in the format '
+                       '"[default:]user|group|other:[entity id or UPN]:r|-w|-x|-,'
+                       '[default:]user|group|other:[entity id or UPN]:r|-w|-x|-,...".')
+            c.ignore('owner', 'group', 'permissions')
+        with self.argument_context('storage blob directory access update') as c:
+            c.argument('acl', options_list=['--acl-spec', '-a'],
+                       help='The ACL specification to set on the path in the format '
+                       '"[default:]user|group|other:[entity id or UPN]:r|-w|-x|-,'
+                       '[default:]user|group|other:[entity id or UPN]:r|-w|-x|-,...".')
+            c.argument('owner', help='The owning user for the directory.')
+            c.argument('group', help='The owning group for the directory.')
+            c.argument('permissions', help='The POSIX access permissions for the file owner,'
+                       'the file owning group, and others. Both symbolic (rwxrw-rw-) and 4-digit '
+                       'octal notation (e.g. 0766) are supported.')
+        with self.argument_context('storage blob directory download') as c:
+            c.extra('source_container', options_list=['--container', '-c'], required=True,
+                    help='The download source container.')
+            c.extra('source_path', options_list=['--source-directory', '-s'], required=True,
+                    validator=validate_blob_directory_download_source_url,
+                    help='The download source directory path.')
+            c.argument('destination', options_list=['--destination-path', '-d'],
+                       help='The destination directory path to download.')
+            c.argument('recursive', options_list=['--recursive', '-r'], action='store_true',
+                       help='Recursively download blobs. If enabled, all the blobs including the blobs in subdirectories '
+                            'will be downloaded.')
+            c.ignore('source')
+        with self.argument_context('storage blob directory exists') as c:
+            c.argument('blob_name', directory_path_type, required=True)
+        with self.argument_context('storage blob directory list') as c:
+            c.argument('include', validator=validate_included_datasets, default='mc')
+        with self.argument_context('storage blob directory metadata') as c:
+            c.argument('blob_name', directory_path_type)
+        with self.argument_context('storage blob directory move') as c:
+            c.argument('destination_path', options_list=['--destination-path', '-d'],
+                       help='The destination blob directory path.')
+            c.argument('source_path', options_list=['--source-path', '-s'],
+                       help='The source blob directory path.')
+        with self.argument_context('storage blob directory show') as c:
+            c.argument('directory_name', directory_path_type)
+            c.argument('container_name', container_name_type)
+            # c.argument('snapshot', help='The snapshot parameter is an opaque DateTime value that, '
+            #                            'when present, specifies the directory snapshot to retrieve.')
+            c.ignore('snapshot')
+            c.argument('lease-id', help='Required if the blob has an active lease.')
+        with self.argument_context('storage blob directory upload') as c:
+            c.extra('destination_container', options_list=['--container', '-c'], required=True,
+                    help='The upload destination container.')
+            c.extra('destination_path', options_list=['--destination-directory', '-d'], required=True,
+                    validator=validate_blob_directory_upload_destination_url,
+                    help='The upload destination directory path.')
+            c.argument('source', options_list=['--source', '-s'],
+                       help='The source file path to upload from.')
+            c.argument('recursive', options_list=['--recursive', '-r'], action='store_true',
+                       help='Recursively upload blobs. If enabled, all the blobs including the blobs in subdirectories will'
+                            ' be uploaded.')
+            c.ignore('destination')

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[Storage] Fix #10620 in azure cli to support ADLS #1038

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!