diff --git a/src/azure-cli/azure/cli/command_modules/storage/_help.py b/src/azure-cli/azure/cli/command_modules/storage/_help.py index 36f2db650bf..16c87066156 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/_help.py +++ b/src/azure-cli/azure/cli/command_modules/storage/_help.py @@ -1928,6 +1928,34 @@ crafted: true """ +helps['storage fs directory upload'] = """ + type: command + short-summary: Upload files or subdirectories to a directory in ADLS Gen2 file system. + examples: + - name: Upload a single file to a storage blob directory. + text: az storage fs directory upload -f myfilesystem --account-name mystorageaccount -s "path/to/file" -d directory + - name: Upload a local directory to root directory in ADLS Gen2 file system. + text: az storage fs directory upload -f myfilesystem --account-name mystorageaccount -s "path/to/directory" --recursive + - name: Upload a local directory to a directory in ADLS Gen2 file system. + text: az storage fs directory upload -f myfilesystem --account-name mystorageaccount -s "path/to/directory" -d directory --recursive + - name: Upload a set of files in a local directory to a directory in ADLS Gen2 file system. + text: az storage fs directory upload -f myfilesystem --account-name mystorageaccount -s "path/to/file*" -d directory --recursive +""" + +helps['storage fs directory download'] = """ + type: command + short-summary: Download files from the directory in ADLS Gen2 file system to a local file path. + examples: + - name: Download a single file in a directory in ADLS Gen2 file system. + text: az storage fs directory download -f myfilesystem --account-name mystorageaccount -s "path/to/file" -d "" + - name: Download whole ADLS Gen2 file system. + text: az storage fs directory download -f myfilesystem --account-name mystorageaccount -d "" --recursive + - name: Download the entire directory in ADLS Gen2 file system. + text: az storage fs directory download -f myfilesystem --account-name mystorageaccount -s SourceDirectoryPath -d "" --recursive + - name: Download an entire subdirectory in ADLS Gen2 file system. + text: az storage fs directory download -f myfilesystem --account-name mystorageaccount -s "path/to/subdirectory" -d "" --recursive +""" + helps['storage fs file'] = """ type: group short-summary: Manage files in Azure Data Lake Storage Gen2 account. diff --git a/src/azure-cli/azure/cli/command_modules/storage/_params.py b/src/azure-cli/azure/cli/command_modules/storage/_params.py index a6cf1e88b1f..410890e7102 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/_params.py +++ b/src/azure-cli/azure/cli/command_modules/storage/_params.py @@ -1561,6 +1561,33 @@ def load_arguments(self, _): # pylint: disable=too-many-locals, too-many-statem help='The new directory name the users want to move to. The value must have the following format: ' '"{filesystem}/{directory}/{subdirectory}".') + with self.argument_context('storage fs directory upload') as c: + from ._validators import validate_fs_directory_upload_destination_url + c.extra('destination_fs', options_list=['--file-system', '-f'], required=True, + help='The upload destination file system.') + c.extra('destination_path', options_list=['--destination-path', '-d'], + validator=validate_fs_directory_upload_destination_url, + help='The upload destination directory path. It should be an absolute path to file system. ' + 'If the specified destination path does not exist, a new directory path will be created.') + c.argument('source', options_list=['--source', '-s'], + help='The source file path to upload from.') + c.argument('recursive', recursive_type, help='Recursively upload files. If enabled, all the files ' + 'including the files in subdirectories will be uploaded.') + c.ignore('destination') + + with self.argument_context('storage fs directory download') as c: + from ._validators import validate_fs_directory_download_source_url + c.extra('source_fs', options_list=['--file-system', '-f'], required=True, + help='The download source file system.') + c.extra('source_path', options_list=['--source-path', '-s'], + validator=validate_fs_directory_download_source_url, + help='The download source directory path. It should be an absolute path to file system.') + c.argument('destination', options_list=['--destination-path', '-d'], + help='The destination local directory path to download.') + c.argument('recursive', recursive_type, help='Recursively download files. If enabled, all the files ' + 'including the files in subdirectories will be downloaded.') + c.ignore('source') + with self.argument_context('storage fs file list') as c: c.extra('file_system_name', options_list=['-f', '--file-system'], help="File system name.", required=True) c.argument('recursive', arg_type=get_three_state_flag(), default=True, diff --git a/src/azure-cli/azure/cli/command_modules/storage/_validators.py b/src/azure-cli/azure/cli/command_modules/storage/_validators.py index 77ee838646f..13acafcfabb 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/_validators.py +++ b/src/azure-cli/azure/cli/command_modules/storage/_validators.py @@ -14,7 +14,8 @@ from azure.cli.command_modules.storage._client_factory import (get_storage_data_service_client, blob_data_service_factory, file_data_service_factory, - storage_client_factory) + storage_client_factory, + cf_adls_file_system) from azure.cli.command_modules.storage.util import glob_files_locally, guess_content_type from azure.cli.command_modules.storage.sdkutil import get_table_data_type from azure.cli.command_modules.storage.url_quote_util import encode_for_url @@ -1577,6 +1578,61 @@ def validate_azcopy_credential(cmd, namespace): service=service, resource_types='sco', permissions='rl') +def is_directory(props): + return 'hdi_isfolder' in props.metadata.keys() and props.metadata['hdi_isfolder'] == 'true' + + +def validate_fs_directory_upload_destination_url(cmd, namespace): + kwargs = {'account_name': namespace.account_name, + 'account_key': namespace.account_key, + 'connection_string': namespace.connection_string, + 'sas_token': namespace.sas_token, + 'file_system_name': namespace.destination_fs} + client = cf_adls_file_system(cmd.cli_ctx, kwargs) + url = client.url + if namespace.destination_path: + from azure.core.exceptions import AzureError + from azure.cli.core.azclierror import InvalidArgumentValueError + file_client = client.get_file_client(file_path=namespace.destination_path) + try: + props = file_client.get_file_properties() + if not is_directory(props): + raise InvalidArgumentValueError('usage error: You are specifying --destination-path with a file name, ' + 'not directory name. Please change to a valid directory name. ' + 'If you want to upload to a file, please use ' + '`az storage fs file upload` command.') + except AzureError: + pass + url = file_client.url + + if _is_valid_uri(url): + namespace.destination = url + else: + namespace.destination = _add_sas_for_url(cmd, url=url, account_name=namespace.account_name, + account_key=namespace.account_key, sas_token=namespace.sas_token, + service='blob', resource_types='co', permissions='rwdlac') + del namespace.destination_fs + del namespace.destination_path + + +def validate_fs_directory_download_source_url(cmd, namespace): + kwargs = {'account_name': namespace.account_name, + 'account_key': namespace.account_key, + 'connection_string': namespace.connection_string, + 'sas_token': namespace.sas_token, + 'file_system_name': namespace.source_fs} + client = cf_adls_file_system(cmd.cli_ctx, kwargs) + url = client.url + if namespace.source_path: + file_client = client.get_file_client(file_path=namespace.source_path) + url = file_client.url + namespace.source = _add_sas_for_url(cmd, url=url, account_name=namespace.account_name, + account_key=namespace.account_key, sas_token=namespace.sas_token, + service='blob', resource_types='co', permissions='rl') + del namespace.source_fs + del namespace.source_path + + def validate_text_configuration(cmd, ns): DelimitedTextDialect = cmd.get_models('_models#DelimitedTextDialect', resource_type=ResourceType.DATA_STORAGE_BLOB) DelimitedJSON = cmd.get_models('_models#DelimitedJSON', resource_type=ResourceType.DATA_STORAGE_BLOB) diff --git a/src/azure-cli/azure/cli/command_modules/storage/commands.py b/src/azure-cli/azure/cli/command_modules/storage/commands.py index ff717094f16..2b749f33dcb 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/commands.py +++ b/src/azure-cli/azure/cli/command_modules/storage/commands.py @@ -779,6 +779,10 @@ def get_custom_sdk(custom_module, client_factory, resource_type=ResourceType.DAT g.storage_command_oauth('metadata show', 'get_directory_properties', exception_handler=show_exception_handler, transform=transform_metadata) + with self.command_group('storage fs directory', custom_command_type=get_custom_sdk('azcopy', None))as g: + g.storage_custom_command_oauth('upload', 'storage_fs_directory_copy', is_preview=True) + g.storage_custom_command_oauth('download', 'storage_fs_directory_copy', is_preview=True) + with self.command_group('storage fs file', adls_file_sdk, resource_type=ResourceType.DATA_STORAGE_FILEDATALAKE, custom_command_type=get_custom_sdk('fs_file', cf_adls_file), min_api='2018-11-09') as g: from ._transformers import transform_storage_list_output, create_boolean_result_output_transformer diff --git a/src/azure-cli/azure/cli/command_modules/storage/operations/azcopy.py b/src/azure-cli/azure/cli/command_modules/storage/operations/azcopy.py index 40463c97be6..685f1030b69 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/operations/azcopy.py +++ b/src/azure-cli/azure/cli/command_modules/storage/operations/azcopy.py @@ -65,6 +65,18 @@ def storage_remove(cmd, client, service, target, recursive=None, exclude_pattern azcopy.remove(_add_url_sas(target, sas_token), flags=flags) +# pylint: disable=unused-argument +def storage_fs_directory_copy(cmd, source, destination, recursive=None, **kwargs): + azcopy = AzCopy() + if kwargs.get('token_credential'): + azcopy = _azcopy_login_client(cmd) + + flags = [] + if recursive: + flags.append('--recursive') + azcopy.copy(source, destination, flags=flags) + + def storage_blob_sync(cmd, client, source, destination, exclude_pattern=None, include_pattern=None, exclude_path=None): azcopy = _azcopy_blob_client(cmd, client) diff --git a/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_adls_gen2_scenarios.py b/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_adls_gen2_scenarios.py index 1f2f06738a3..abcd43e1514 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_adls_gen2_scenarios.py +++ b/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_adls_gen2_scenarios.py @@ -3,11 +3,12 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- import unittest +import os -from azure.cli.testsdk import (ScenarioTest, ResourceGroupPreparer, StorageAccountPreparer, JMESPathCheck, NoneCheck, +from azure.cli.testsdk import (ScenarioTest, LiveScenarioTest, ResourceGroupPreparer, StorageAccountPreparer, JMESPathCheck, NoneCheck, api_version_constraint, RoleBasedServicePrincipalPreparer) from azure.cli.core.profiles import ResourceType -from ..storage_test_util import StorageScenarioMixin +from ..storage_test_util import StorageScenarioMixin, StorageTestFilesPreparer from knack.util import CLIError @@ -448,3 +449,110 @@ def test_adls_metadata_scenarios(self, resource_group, storage_account): account_info, file, filesystem) self.storage_cmd('storage fs file metadata show -p {} -f {}', account_info, file, filesystem) \ .assert_with_checks(JMESPathCheck('test', 'beta'), JMESPathCheck('cat', 'file')) + + +class StorageADLSGen2LiveTests(StorageScenarioMixin, LiveScenarioTest): + @ResourceGroupPreparer() + @StorageAccountPreparer(kind="StorageV2", hns=True) + @StorageTestFilesPreparer() + def test_adls_directory_upload(self, resource_group, storage_account, test_dir): + account_info = self.get_account_info(resource_group, storage_account) + connection_string = self.get_connection_string(resource_group, storage_account) + + filesystem = 'testfilesystem' + directory = 'testdir' + self.storage_cmd('storage fs create -n {}', account_info, filesystem) + self.storage_cmd('storage fs directory create -n {} -f {}', account_info, directory, filesystem) + + from datetime import datetime, timedelta + expiry = (datetime.utcnow() + timedelta(hours=1)).strftime('%Y-%m-%dT%H:%MZ') + sas = self.storage_cmd('storage container generate-sas -n {} --https-only ' + '--permissions acdlrw --expiry {} -otsv', + account_info, filesystem, expiry).output.strip() + + # Upload a single file to the fs directory + self.storage_cmd('storage fs directory upload -f {} -d {} -s "{}"', account_info, filesystem, directory, + os.path.join(test_dir, 'readme')) + self.storage_cmd('storage fs file list -f {} --path {}', account_info, filesystem, directory) \ + .assert_with_checks(JMESPathCheck('length(@)', 1)) + + # Upload a local directory to the fs directory + self.oauth_cmd('storage fs directory upload -f {} -d {} -s "{}" --recursive --account-name {}', filesystem, + directory, os.path.join(test_dir, 'apple'), storage_account) + self.oauth_cmd('storage fs file list -f {} --path {} --account-name {}', + filesystem, directory, storage_account)\ + .assert_with_checks(JMESPathCheck('length(@)', 12)) + + # Upload files in a local directory to the fs directory + self.cmd('storage fs directory upload -f {} -d {} -s "{}" --recursive --connection-string {}' + .format(filesystem, directory, os.path.join(test_dir, 'butter/file_*'), connection_string)) + self.cmd('storage fs file list -f {} --path {} --connection-string {}' + .format(filesystem, directory, connection_string), checks=[JMESPathCheck('length(@)', 22)]) + + # Upload files in a local directory to the fs subdirectory + self.cmd('storage fs directory upload -f {} -d {} -s "{}" --recursive --account-name {} --sas-token {}' + .format(filesystem, '/'.join([directory, 'subdir']), + os.path.join(test_dir, 'butter/file_*'), storage_account, sas)) + self.cmd('storage fs file list -f {} --path {} --account-name {} --sas-token {}' + .format(filesystem, '/'.join([directory, 'subdir']), storage_account, sas), + checks=[JMESPathCheck('length(@)', 10)]) + + # Upload single file to the fs root directory + self.storage_cmd('storage fs directory upload -f {} -s "{}"', account_info, filesystem, + os.path.join(test_dir, 'readme')) + self.storage_cmd('storage fs file exists -f {} --path {}', account_info, filesystem, 'readme') \ + .assert_with_checks(JMESPathCheck('exists', True)) + + # Upload files to the fs root directory + self.storage_cmd('storage fs directory upload -f {} -s "{}" -r', account_info, filesystem, + os.path.join(test_dir, 'duff')) + self.storage_cmd('storage fs file exists -f {} --path {}', account_info, filesystem, 'duff/edward') \ + .assert_with_checks(JMESPathCheck('exists', True)) + + # Argument validation: Fail when destination path is file name + self.cmd('storage fs directory upload -f {} -d {} -s {} --account-name {}'.format( + filesystem, '/'.join([directory, 'readme']), test_dir, storage_account), expect_failure=True) + + @ResourceGroupPreparer() + @StorageAccountPreparer(kind="StorageV2", hns=True) + @StorageTestFilesPreparer() + def test_adls_directory_download(self, resource_group, storage_account, test_dir): + account_info = self.get_account_info(resource_group, storage_account) + connection_string = self.get_connection_string(resource_group, storage_account) + + filesystem = 'testfilesystem' + directory = 'testdir' + self.storage_cmd('storage fs create -n {}', account_info, filesystem) + self.storage_cmd('storage fs directory create -n {} -f {}', account_info, directory, filesystem) + self.storage_cmd('storage fs directory upload -f {} -d {} -s "{}" --recursive', account_info, filesystem, + directory, os.path.join(test_dir, 'readme')) + self.storage_cmd('storage fs directory upload -f {} -d {} -s "{}" --recursive', account_info, filesystem, + directory, os.path.join(test_dir, 'apple')) + + from datetime import datetime, timedelta + expiry = (datetime.utcnow() + timedelta(hours=1)).strftime('%Y-%m-%dT%H:%MZ') + sas = self.storage_cmd('storage container generate-sas -n {} --https-only ' + '--permissions lr --expiry {} -otsv', + account_info, filesystem, expiry).output.strip() + + local_folder = self.create_temp_dir() + # Download a single file + self.storage_cmd('storage fs directory download -f {} -s "{}" -d "{}" --recursive', account_info, filesystem, + '/'.join([directory, 'readme']), local_folder) + self.assertEqual(1, sum(len(f) for r, d, f in os.walk(local_folder))) + + # Download entire directory + self.oauth_cmd('storage fs directory download -f {} -s {} -d "{}" --recursive --account-name {}', + filesystem, directory, local_folder, storage_account) + self.assertEqual(2, sum(len(d) for r, d, f in os.walk(local_folder))) + self.assertEqual(12, sum(len(f) for r, d, f in os.walk(local_folder))) + + # Download an entire subdirectory of a storage blob directory. + self.cmd('storage fs directory download -f {} -s {} -d "{}" --recursive --connection-string {}' + .format(filesystem, '/'.join([directory, 'apple']), local_folder, connection_string)) + self.assertEqual(3, sum(len(d) for r, d, f in os.walk(local_folder))) + + # Download from root directory + self.cmd('storage fs directory download -f {} -d "{}" --recursive --account-name {} --sas-token {}' + .format(filesystem, local_folder, storage_account, sas)) + self.assertEqual(6, sum(len(d) for r, d, f in os.walk(local_folder))) diff --git a/src/azure-cli/azure/cli/command_modules/storage/tests/storage_test_util.py b/src/azure-cli/azure/cli/command_modules/storage/tests/storage_test_util.py index 4d4e14a3de3..083ea40a2a4 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/tests/storage_test_util.py +++ b/src/azure-cli/azure/cli/command_modules/storage/tests/storage_test_util.py @@ -26,6 +26,10 @@ def get_account_key(self, group, name): return self.cmd(template.format(name, group)).output + def get_connection_string(self, group, name): + return self.cmd('storage account show-connection-string -n {} -g {} ' + '--query connectionString -otsv'.format(name, group)).output.strip() + def get_account_info(self, group, name): """Returns the storage account name and key in a tuple""" return name, self.get_account_key(group, name)