diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/azure_storage_checkpoint_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/azure_storage_checkpoint_manager.py index 05440824f23b..18acb52db82a 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/azure_storage_checkpoint_manager.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/azure_storage_checkpoint_manager.py @@ -12,7 +12,7 @@ import asyncio import requests -from azure.storage.blob import BlockBlobService +from .vendor.storage.blob import BlockBlobService from azure.eventprocessorhost.azure_blob_lease import AzureBlobLease from azure.eventprocessorhost.checkpoint import Checkpoint from azure.eventprocessorhost.abstract_lease_manager import AbstractLeaseManager diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/__init__.py new file mode 100644 index 000000000000..de40ea7ca058 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/__init__.py @@ -0,0 +1 @@ +__import__('pkg_resources').declare_namespace(__name__) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/__init__.py new file mode 100644 index 000000000000..de40ea7ca058 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/__init__.py @@ -0,0 +1 @@ +__import__('pkg_resources').declare_namespace(__name__) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/__init__.py new file mode 100644 index 000000000000..eb3e5d0fde33 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/__init__.py @@ -0,0 +1,31 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from .appendblobservice import AppendBlobService +from .blockblobservice import BlockBlobService +from .models import ( + Container, + ContainerProperties, + Blob, + BlobProperties, + BlobBlock, + BlobBlockList, + PageRange, + ContentSettings, + CopyProperties, + ContainerPermissions, + BlobPermissions, + _LeaseActions, + AppendBlockProperties, + PageBlobProperties, + ResourceProperties, + Include, + SequenceNumberAction, + BlockListType, + PublicAccess, + BlobPrefix, + DeleteSnapshot, +) +from .pageblobservice import PageBlobService diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_constants.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_constants.py new file mode 100644 index 000000000000..b450d83e430d --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_constants.py @@ -0,0 +1,14 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +__author__ = 'Microsoft Corp. ' +__version__ = '1.3.1' + +# x-ms-version for storage service. +X_MS_VERSION = '2018-03-28' + +# internal configurations, should not be changed +_LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE = 4 * 1024 * 1024 diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_deserialization.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_deserialization.py new file mode 100644 index 000000000000..3365ebfa726c --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_deserialization.py @@ -0,0 +1,452 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from azure.common import AzureException +from dateutil import parser + +try: + from xml.etree import cElementTree as ETree +except ImportError: + from xml.etree import ElementTree as ETree + +from ..common._common_conversion import ( + _decode_base64_to_text, + _to_str, + _get_content_md5 +) +from ..common._deserialization import ( + _parse_properties, + _to_int, + _parse_metadata, + _convert_xml_to_signed_identifiers, + _bool, +) +from .models import ( + Container, + Blob, + BlobBlock, + BlobBlockList, + BlobBlockState, + BlobProperties, + PageRange, + ContainerProperties, + AppendBlockProperties, + PageBlobProperties, + ResourceProperties, + BlobPrefix, + AccountInformation, +) +from ._encryption import _decrypt_blob +from ..common.models import _list +from ..common._error import ( + _validate_content_match, + _ERROR_DECRYPTION_FAILURE, +) + + +def _parse_base_properties(response): + ''' + Extracts basic response headers. + ''' + resource_properties = ResourceProperties() + resource_properties.last_modified = parser.parse(response.headers.get('last-modified')) + resource_properties.etag = response.headers.get('etag') + + return resource_properties + + +def _parse_page_properties(response): + ''' + Extracts page response headers. + ''' + put_page = PageBlobProperties() + put_page.last_modified = parser.parse(response.headers.get('last-modified')) + put_page.etag = response.headers.get('etag') + put_page.sequence_number = _to_int(response.headers.get('x-ms-blob-sequence-number')) + + return put_page + + +def _parse_append_block(response): + ''' + Extracts append block response headers. + ''' + append_block = AppendBlockProperties() + append_block.last_modified = parser.parse(response.headers.get('last-modified')) + append_block.etag = response.headers.get('etag') + append_block.append_offset = _to_int(response.headers.get('x-ms-blob-append-offset')) + append_block.committed_block_count = _to_int(response.headers.get('x-ms-blob-committed-block-count')) + + return append_block + + +def _parse_snapshot_blob(response, name): + ''' + Extracts snapshot return header. + ''' + snapshot = response.headers.get('x-ms-snapshot') + + return _parse_blob(response, name, snapshot) + + +def _parse_lease(response): + ''' + Extracts lease time and ID return headers. + ''' + lease = {'time': response.headers.get('x-ms-lease-time')} + if lease['time']: + lease['time'] = _to_int(lease['time']) + + lease['id'] = response.headers.get('x-ms-lease-id') + + return lease + + +def _parse_blob(response, name, snapshot, validate_content=False, require_encryption=False, + key_encryption_key=None, key_resolver_function=None, start_offset=None, end_offset=None): + if response is None: + return None + + metadata = _parse_metadata(response) + props = _parse_properties(response, BlobProperties) + + # For range gets, only look at 'x-ms-blob-content-md5' for overall MD5 + content_settings = getattr(props, 'content_settings') + if 'content-range' in response.headers: + if 'x-ms-blob-content-md5' in response.headers: + setattr(content_settings, 'content_md5', _to_str(response.headers['x-ms-blob-content-md5'])) + else: + delattr(content_settings, 'content_md5') + + if validate_content: + computed_md5 = _get_content_md5(response.body) + _validate_content_match(response.headers['content-md5'], computed_md5) + + if key_encryption_key is not None or key_resolver_function is not None: + try: + response.body = _decrypt_blob(require_encryption, key_encryption_key, key_resolver_function, + response, start_offset, end_offset) + except: + raise AzureException(_ERROR_DECRYPTION_FAILURE) + + return Blob(name, snapshot, response.body, props, metadata) + + +def _parse_container(response, name): + if response is None: + return None + + metadata = _parse_metadata(response) + props = _parse_properties(response, ContainerProperties) + return Container(name, props, metadata) + + +def _convert_xml_to_signed_identifiers_and_access(response): + acl = _convert_xml_to_signed_identifiers(response) + acl.public_access = response.headers.get('x-ms-blob-public-access') + + return acl + + +def _convert_xml_to_containers(response): + ''' + + + string-value + string-value + int-value + + + container-name + + date/time-value + etag + locked | unlocked + available | leased | expired | breaking | broken + infinite | fixed + blob | container + true | false + true | false + + + value + + + + marker-value + + ''' + if response is None or response.body is None: + return None + + containers = _list() + list_element = ETree.fromstring(response.body) + + # Set next marker + setattr(containers, 'next_marker', list_element.findtext('NextMarker')) + + containers_element = list_element.find('Containers') + + for container_element in containers_element.findall('Container'): + # Name element + container = Container() + container.name = container_element.findtext('Name') + + # Metadata + metadata_root_element = container_element.find('Metadata') + if metadata_root_element is not None: + container.metadata = dict() + for metadata_element in metadata_root_element: + container.metadata[metadata_element.tag] = metadata_element.text + + # Properties + properties_element = container_element.find('Properties') + container.properties.etag = properties_element.findtext('Etag') + container.properties.last_modified = parser.parse(properties_element.findtext('Last-Modified')) + container.properties.lease_status = properties_element.findtext('LeaseStatus') + container.properties.lease_state = properties_element.findtext('LeaseState') + container.properties.lease_duration = properties_element.findtext('LeaseDuration') + container.properties.public_access = properties_element.findtext('PublicAccess') + container.properties.has_immutability_policy = properties_element.findtext('HasImmutabilityPolicy') + container.properties.has_legal_hold = properties_element.findtext('HasLegalHold') + + # Add container to list + containers.append(container) + + return containers + + +LIST_BLOBS_ATTRIBUTE_MAP = { + 'Last-Modified': (None, 'last_modified', parser.parse), + 'Etag': (None, 'etag', _to_str), + 'x-ms-blob-sequence-number': (None, 'sequence_number', _to_int), + 'BlobType': (None, 'blob_type', _to_str), + 'Content-Length': (None, 'content_length', _to_int), + 'ServerEncrypted': (None, 'server_encrypted', _bool), + 'Content-Type': ('content_settings', 'content_type', _to_str), + 'Content-Encoding': ('content_settings', 'content_encoding', _to_str), + 'Content-Disposition': ('content_settings', 'content_disposition', _to_str), + 'Content-Language': ('content_settings', 'content_language', _to_str), + 'Content-MD5': ('content_settings', 'content_md5', _to_str), + 'Cache-Control': ('content_settings', 'cache_control', _to_str), + 'LeaseStatus': ('lease', 'status', _to_str), + 'LeaseState': ('lease', 'state', _to_str), + 'LeaseDuration': ('lease', 'duration', _to_str), + 'CopyId': ('copy', 'id', _to_str), + 'CopySource': ('copy', 'source', _to_str), + 'CopyStatus': ('copy', 'status', _to_str), + 'CopyProgress': ('copy', 'progress', _to_str), + 'CopyCompletionTime': ('copy', 'completion_time', _to_str), + 'CopyStatusDescription': ('copy', 'status_description', _to_str), + 'AccessTier': (None, 'blob_tier', _to_str), + 'AccessTierChangeTime': (None, 'blob_tier_change_time', parser.parse), + 'AccessTierInferred': (None, 'blob_tier_inferred', _bool), + 'ArchiveStatus': (None, 'rehydration_status', _to_str), + 'DeletedTime': (None, 'deleted_time', parser.parse), + 'RemainingRetentionDays': (None, 'remaining_retention_days', _to_int), + 'Creation-Time': (None, 'creation_time', parser.parse), +} + + +def _convert_xml_to_blob_list(response): + ''' + + + string-value + string-value + int-value + string-value + + + blob-name + true + date-time-value + + date-time-value + etag + size-in-bytes + blob-content-type + + + + + sequence-number + BlockBlob|PageBlob|AppendBlob + locked|unlocked + available | leased | expired | breaking | broken + infinite | fixed + id + pending | success | aborted | failed + source url + bytes copied/bytes total + datetime + error string + P4 | P6 | P10 | P20 | P30 | P40 | P50 | P60 | Archive | Cool | Hot + date-time-value + true + datetime + int + date-time-value + + + value + + + + blob-prefix + + + + + ''' + if response is None or response.body is None: + return None + + blob_list = _list() + list_element = ETree.fromstring(response.body) + + setattr(blob_list, 'next_marker', list_element.findtext('NextMarker')) + + blobs_element = list_element.find('Blobs') + blob_prefix_elements = blobs_element.findall('BlobPrefix') + if blob_prefix_elements is not None: + for blob_prefix_element in blob_prefix_elements: + prefix = BlobPrefix() + prefix.name = blob_prefix_element.findtext('Name') + blob_list.append(prefix) + + for blob_element in blobs_element.findall('Blob'): + blob = Blob() + blob.name = blob_element.findtext('Name') + blob.snapshot = blob_element.findtext('Snapshot') + + deleted = blob_element.findtext('Deleted') + if deleted: + blob.deleted = _bool(deleted) + + # Properties + properties_element = blob_element.find('Properties') + if properties_element is not None: + for property_element in properties_element: + info = LIST_BLOBS_ATTRIBUTE_MAP.get(property_element.tag) + if info is None: + setattr(blob.properties, property_element.tag, _to_str(property_element.text)) + elif info[0] is None: + setattr(blob.properties, info[1], info[2](property_element.text)) + else: + attr = getattr(blob.properties, info[0]) + setattr(attr, info[1], info[2](property_element.text)) + + # Metadata + metadata_root_element = blob_element.find('Metadata') + if metadata_root_element is not None: + blob.metadata = dict() + for metadata_element in metadata_root_element: + blob.metadata[metadata_element.tag] = metadata_element.text + + # Add blob to list + blob_list.append(blob) + + return blob_list + + +def _convert_xml_to_block_list(response): + ''' + + + + + base64-encoded-block-id + size-in-bytes + + + + + base64-encoded-block-id + size-in-bytes + + + + + Converts xml response to block list class. + ''' + if response is None or response.body is None: + return None + + block_list = BlobBlockList() + + list_element = ETree.fromstring(response.body) + + committed_blocks_element = list_element.find('CommittedBlocks') + if committed_blocks_element is not None: + for block_element in committed_blocks_element.findall('Block'): + block_id = _decode_base64_to_text(block_element.findtext('Name', '')) + block_size = int(block_element.findtext('Size')) + block = BlobBlock(id=block_id, state=BlobBlockState.Committed) + block._set_size(block_size) + block_list.committed_blocks.append(block) + + uncommitted_blocks_element = list_element.find('UncommittedBlocks') + if uncommitted_blocks_element is not None: + for block_element in uncommitted_blocks_element.findall('Block'): + block_id = _decode_base64_to_text(block_element.findtext('Name', '')) + block_size = int(block_element.findtext('Size')) + block = BlobBlock(id=block_id, state=BlobBlockState.Uncommitted) + block._set_size(block_size) + block_list.uncommitted_blocks.append(block) + + return block_list + + +def _convert_xml_to_page_ranges(response): + ''' + + + + Start Byte + End Byte + + + Start Byte + End Byte + + + Start Byte + End Byte + + + ''' + if response is None or response.body is None: + return None + + page_list = list() + + list_element = ETree.fromstring(response.body) + + for page_range_element in list_element: + if page_range_element.tag == 'PageRange': + is_cleared = False + elif page_range_element.tag == 'ClearRange': + is_cleared = True + else: + pass # ignore any unrecognized Page Range types + + page_list.append( + PageRange( + int(page_range_element.findtext('Start')), + int(page_range_element.findtext('End')), + is_cleared + ) + ) + + return page_list + + +def _parse_account_information(response): + account_info = AccountInformation() + account_info.sku_name = response.headers['x-ms-sku-name'] + account_info.account_kind = response.headers['x-ms-account-kind'] + + return account_info diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_download_chunking.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_download_chunking.py new file mode 100644 index 000000000000..e68a0e5dee42 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_download_chunking.py @@ -0,0 +1,178 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import threading + + +def _download_blob_chunks(blob_service, container_name, blob_name, snapshot, + download_size, block_size, progress, start_range, end_range, + stream, max_connections, progress_callback, validate_content, + lease_id, if_modified_since, if_unmodified_since, if_match, + if_none_match, timeout, operation_context): + + downloader_class = _ParallelBlobChunkDownloader if max_connections > 1 else _SequentialBlobChunkDownloader + + downloader = downloader_class( + blob_service, + container_name, + blob_name, + snapshot, + download_size, + block_size, + progress, + start_range, + end_range, + stream, + progress_callback, + validate_content, + lease_id, + if_modified_since, + if_unmodified_since, + if_match, + if_none_match, + timeout, + operation_context, + ) + + if max_connections > 1: + import concurrent.futures + executor = concurrent.futures.ThreadPoolExecutor(max_connections) + list(executor.map(downloader.process_chunk, downloader.get_chunk_offsets())) + else: + for chunk in downloader.get_chunk_offsets(): + downloader.process_chunk(chunk) + + +class _BlobChunkDownloader(object): + def __init__(self, blob_service, container_name, blob_name, snapshot, download_size, + chunk_size, progress, start_range, end_range, stream, + progress_callback, validate_content, lease_id, if_modified_since, + if_unmodified_since, if_match, if_none_match, timeout, operation_context): + # identifiers for the blob + self.blob_service = blob_service + self.container_name = container_name + self.blob_name = blob_name + self.snapshot = snapshot + + # information on the download range/chunk size + self.chunk_size = chunk_size + self.download_size = download_size + self.start_index = start_range + self.blob_end = end_range + + # the destination that we will write to + self.stream = stream + + # progress related + self.progress_callback = progress_callback + self.progress_total = progress + + # parameters for each get blob operation + self.timeout = timeout + self.operation_context = operation_context + self.validate_content = validate_content + self.lease_id = lease_id + self.if_modified_since = if_modified_since + self.if_unmodified_since = if_unmodified_since + self.if_match = if_match + self.if_none_match = if_none_match + + def get_chunk_offsets(self): + index = self.start_index + while index < self.blob_end: + yield index + index += self.chunk_size + + def process_chunk(self, chunk_start): + if chunk_start + self.chunk_size > self.blob_end: + chunk_end = self.blob_end + else: + chunk_end = chunk_start + self.chunk_size + + chunk_data = self._download_chunk(chunk_start, chunk_end).content + length = chunk_end - chunk_start + if length > 0: + self._write_to_stream(chunk_data, chunk_start) + self._update_progress(length) + + # should be provided by the subclass + def _update_progress(self, length): + pass + + # should be provided by the subclass + def _write_to_stream(self, chunk_data, chunk_start): + pass + + def _download_chunk(self, chunk_start, chunk_end): + response = self.blob_service._get_blob( + self.container_name, + self.blob_name, + snapshot=self.snapshot, + start_range=chunk_start, + end_range=chunk_end - 1, + validate_content=self.validate_content, + lease_id=self.lease_id, + if_modified_since=self.if_modified_since, + if_unmodified_since=self.if_unmodified_since, + if_match=self.if_match, + if_none_match=self.if_none_match, + timeout=self.timeout, + _context=self.operation_context + ) + + # This makes sure that if_match is set so that we can validate + # that subsequent downloads are to an unmodified blob + self.if_match = response.properties.etag + return response + + +class _ParallelBlobChunkDownloader(_BlobChunkDownloader): + def __init__(self, blob_service, container_name, blob_name, snapshot, download_size, + chunk_size, progress, start_range, end_range, stream, + progress_callback, validate_content, lease_id, if_modified_since, + if_unmodified_since, if_match, if_none_match, timeout, operation_context): + + super(_ParallelBlobChunkDownloader, self).__init__(blob_service, container_name, blob_name, snapshot, + download_size, + chunk_size, progress, start_range, end_range, stream, + progress_callback, validate_content, lease_id, + if_modified_since, + if_unmodified_since, if_match, if_none_match, timeout, + operation_context) + + # for a parallel download, the stream is always seekable, so we note down the current position + # in order to seek to the right place when out-of-order chunks come in + self.stream_start = stream.tell() + + # since parallel operations are going on + # it is essential to protect the writing and progress reporting operations + self.stream_lock = threading.Lock() + self.progress_lock = threading.Lock() + + def _update_progress(self, length): + if self.progress_callback is not None: + with self.progress_lock: + self.progress_total += length + total_so_far = self.progress_total + self.progress_callback(total_so_far, self.download_size) + + def _write_to_stream(self, chunk_data, chunk_start): + with self.stream_lock: + self.stream.seek(self.stream_start + (chunk_start - self.start_index)) + self.stream.write(chunk_data) + + +class _SequentialBlobChunkDownloader(_BlobChunkDownloader): + def __init__(self, *args): + super(_SequentialBlobChunkDownloader, self).__init__(*args) + + def _update_progress(self, length): + if self.progress_callback is not None: + self.progress_total += length + self.progress_callback(self.progress_total, self.download_size) + + def _write_to_stream(self, chunk_data, chunk_start): + # chunk_start is ignored in the case of sequential download since we cannot seek the destination stream + self.stream.write(chunk_data) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_encryption.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_encryption.py new file mode 100644 index 000000000000..f1e9b540b0bf --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_encryption.py @@ -0,0 +1,187 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +from json import ( + dumps, + loads, +) +from os import urandom + +from cryptography.hazmat.primitives.padding import PKCS7 + +from ..common._encryption import ( + _generate_encryption_data_dict, + _generate_AES_CBC_cipher, + _dict_to_encryption_data, + _validate_and_unwrap_cek, + _EncryptionAlgorithm, +) +from ..common._error import ( + _validate_not_none, + _validate_key_encryption_key_wrap, + _ERROR_DATA_NOT_ENCRYPTED, + _ERROR_UNSUPPORTED_ENCRYPTION_ALGORITHM, +) + + +def _encrypt_blob(blob, key_encryption_key): + ''' + Encrypts the given blob using AES256 in CBC mode with 128 bit padding. + Wraps the generated content-encryption-key using the user-provided key-encryption-key (kek). + Returns a json-formatted string containing the encryption metadata. This method should + only be used when a blob is small enough for single shot upload. Encrypting larger blobs + is done as a part of the _upload_blob_chunks method. + + :param bytes blob: + The blob to be encrypted. + :param object key_encryption_key: + The user-provided key-encryption-key. Must implement the following methods: + wrap_key(key)--wraps the specified key using an algorithm of the user's choice. + get_key_wrap_algorithm()--returns the algorithm used to wrap the specified symmetric key. + get_kid()--returns a string key id for this key-encryption-key. + :return: A tuple of json-formatted string containing the encryption metadata and the encrypted blob data. + :rtype: (str, bytes) + ''' + + _validate_not_none('blob', blob) + _validate_not_none('key_encryption_key', key_encryption_key) + _validate_key_encryption_key_wrap(key_encryption_key) + + # AES256 uses 256 bit (32 byte) keys and always with 16 byte blocks + content_encryption_key = urandom(32) + initialization_vector = urandom(16) + + cipher = _generate_AES_CBC_cipher(content_encryption_key, initialization_vector) + + # PKCS7 with 16 byte blocks ensures compatibility with AES. + padder = PKCS7(128).padder() + padded_data = padder.update(blob) + padder.finalize() + + # Encrypt the data. + encryptor = cipher.encryptor() + encrypted_data = encryptor.update(padded_data) + encryptor.finalize() + encryption_data = _generate_encryption_data_dict(key_encryption_key, content_encryption_key, + initialization_vector) + encryption_data['EncryptionMode'] = 'FullBlob' + + return dumps(encryption_data), encrypted_data + + +def _generate_blob_encryption_data(key_encryption_key): + ''' + Generates the encryption_metadata for the blob. + + :param bytes key_encryption_key: + The key-encryption-key used to wrap the cek associate with this blob. + :return: A tuple containing the cek and iv for this blob as well as the + serialized encryption metadata for the blob. + :rtype: (bytes, bytes, str) + ''' + encryption_data = None + content_encryption_key = None + initialization_vector = None + if key_encryption_key: + _validate_key_encryption_key_wrap(key_encryption_key) + content_encryption_key = urandom(32) + initialization_vector = urandom(16) + encryption_data = _generate_encryption_data_dict(key_encryption_key, + content_encryption_key, + initialization_vector) + encryption_data['EncryptionMode'] = 'FullBlob' + encryption_data = dumps(encryption_data) + + return content_encryption_key, initialization_vector, encryption_data + + +def _decrypt_blob(require_encryption, key_encryption_key, key_resolver, + response, start_offset, end_offset): + ''' + Decrypts the given blob contents and returns only the requested range. + + :param bool require_encryption: + Whether or not the calling blob service requires objects to be decrypted. + :param object key_encryption_key: + The user-provided key-encryption-key. Must implement the following methods: + wrap_key(key)--wraps the specified key using an algorithm of the user's choice. + get_key_wrap_algorithm()--returns the algorithm used to wrap the specified symmetric key. + get_kid()--returns a string key id for this key-encryption-key. + :param key_resolver(kid): + The user-provided key resolver. Uses the kid string to return a key-encryption-key + implementing the interface defined above. + :return: The decrypted blob content. + :rtype: bytes + ''' + _validate_not_none('response', response) + content = response.body + _validate_not_none('content', content) + + try: + encryption_data = _dict_to_encryption_data(loads(response.headers['x-ms-meta-encryptiondata'])) + except: + if require_encryption: + raise ValueError(_ERROR_DATA_NOT_ENCRYPTED) + else: + return content + + if not (encryption_data.encryption_agent.encryption_algorithm == _EncryptionAlgorithm.AES_CBC_256): + raise ValueError(_ERROR_UNSUPPORTED_ENCRYPTION_ALGORITHM) + + blob_type = response.headers['x-ms-blob-type'] + + iv = None + unpad = False + start_range, end_range = 0, len(content) + if 'content-range' in response.headers: + content_range = response.headers['content-range'] + # Format: 'bytes x-y/size' + + # Ignore the word 'bytes' + content_range = content_range.split(' ') + + content_range = content_range[1].split('-') + start_range = int(content_range[0]) + content_range = content_range[1].split('/') + end_range = int(content_range[0]) + blob_size = int(content_range[1]) + + if start_offset >= 16: + iv = content[:16] + content = content[16:] + start_offset -= 16 + else: + iv = encryption_data.content_encryption_IV + + if end_range == blob_size - 1: + unpad = True + else: + unpad = True + iv = encryption_data.content_encryption_IV + + if blob_type == 'PageBlob': + unpad = False + + content_encryption_key = _validate_and_unwrap_cek(encryption_data, key_encryption_key, key_resolver) + cipher = _generate_AES_CBC_cipher(content_encryption_key, iv) + decryptor = cipher.decryptor() + + content = decryptor.update(content) + decryptor.finalize() + if unpad: + unpadder = PKCS7(128).unpadder() + content = unpadder.update(content) + unpadder.finalize() + + return content[start_offset: len(content) - end_offset] + + +def _get_blob_encryptor_and_padder(cek, iv, should_pad): + encryptor = None + padder = None + + if cek is not None and iv is not None: + cipher = _generate_AES_CBC_cipher(cek, iv) + encryptor = cipher.encryptor() + padder = PKCS7(128).padder() if should_pad else None + + return encryptor, padder diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_error.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_error.py new file mode 100644 index 000000000000..f24edc81377e --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_error.py @@ -0,0 +1,29 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +_ERROR_PAGE_BLOB_SIZE_ALIGNMENT = \ + 'Invalid page blob size: {0}. ' + \ + 'The size must be aligned to a 512-byte boundary.' + +_ERROR_PAGE_BLOB_START_ALIGNMENT = \ + 'start_range must align with 512 page size' + +_ERROR_PAGE_BLOB_END_ALIGNMENT = \ + 'end_range must align with 512 page size' + +_ERROR_INVALID_BLOCK_ID = \ + 'All blocks in block list need to have valid block ids.' + +_ERROR_INVALID_LEASE_DURATION = \ + "lease_duration param needs to be between 15 and 60 or -1." + +_ERROR_INVALID_LEASE_BREAK_PERIOD = \ + "lease_break_period param needs to be between 0 and 60." + +_ERROR_NO_SINGLE_THREAD_CHUNKING = \ + 'To use blob chunk downloader more than 1 thread must be ' + \ + 'used since get_blob_to_bytes should be called for single threaded ' + \ + 'blob downloads.' diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_serialization.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_serialization.py new file mode 100644 index 000000000000..100b40898561 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_serialization.py @@ -0,0 +1,118 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from xml.sax.saxutils import escape as xml_escape + +try: + from xml.etree import cElementTree as ETree +except ImportError: + from xml.etree import ElementTree as ETree +from ..common._common_conversion import ( + _encode_base64, + _str, +) +from ..common._error import ( + _validate_not_none, + _ERROR_START_END_NEEDED_FOR_MD5, + _ERROR_RANGE_TOO_LARGE_FOR_MD5, +) +from ._error import ( + _ERROR_PAGE_BLOB_START_ALIGNMENT, + _ERROR_PAGE_BLOB_END_ALIGNMENT, + _ERROR_INVALID_BLOCK_ID, +) +from io import BytesIO + + +def _get_path(container_name=None, blob_name=None): + ''' + Creates the path to access a blob resource. + + container_name: + Name of container. + blob_name: + The path to the blob. + ''' + if container_name and blob_name: + return '/{0}/{1}'.format( + _str(container_name), + _str(blob_name)) + elif container_name: + return '/{0}'.format(_str(container_name)) + else: + return '/' + + +def _validate_and_format_range_headers(request, start_range, end_range, start_range_required=True, + end_range_required=True, check_content_md5=False, align_to_page=False): + # If end range is provided, start range must be provided + if start_range_required or end_range is not None: + _validate_not_none('start_range', start_range) + if end_range_required: + _validate_not_none('end_range', end_range) + + # Page ranges must be 512 aligned + if align_to_page: + if start_range is not None and start_range % 512 != 0: + raise ValueError(_ERROR_PAGE_BLOB_START_ALIGNMENT) + if end_range is not None and end_range % 512 != 511: + raise ValueError(_ERROR_PAGE_BLOB_END_ALIGNMENT) + + # Format based on whether end_range is present + request.headers = request.headers or {} + if end_range is not None: + request.headers['x-ms-range'] = 'bytes={0}-{1}'.format(start_range, end_range) + elif start_range is not None: + request.headers['x-ms-range'] = "bytes={0}-".format(start_range) + + # Content MD5 can only be provided for a complete range less than 4MB in size + if check_content_md5: + if start_range is None or end_range is None: + raise ValueError(_ERROR_START_END_NEEDED_FOR_MD5) + if end_range - start_range > 4 * 1024 * 1024: + raise ValueError(_ERROR_RANGE_TOO_LARGE_FOR_MD5) + + request.headers['x-ms-range-get-content-md5'] = 'true' + + +def _convert_block_list_to_xml(block_id_list): + ''' + + + first-base64-encoded-block-id + second-base64-encoded-block-id + third-base64-encoded-block-id + + + Convert a block list to xml to send. + + block_id_list: + A list of BlobBlock containing the block ids and block state that are used in put_block_list. + Only get block from latest blocks. + ''' + if block_id_list is None: + return '' + + block_list_element = ETree.Element('BlockList') + + # Enabled + for block in block_id_list: + if block.id is None: + raise ValueError(_ERROR_INVALID_BLOCK_ID) + id = xml_escape(_str(format(_encode_base64(block.id)))) + ETree.SubElement(block_list_element, block.state).text = id + + # Add xml declaration and serialize + try: + stream = BytesIO() + ETree.ElementTree(block_list_element).write(stream, xml_declaration=True, encoding='utf-8', method='xml') + except: + raise + finally: + output = stream.getvalue() + stream.close() + + # return xml value + return output diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_upload_chunking.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_upload_chunking.py new file mode 100644 index 000000000000..b94f05811be7 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_upload_chunking.py @@ -0,0 +1,496 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from io import (BytesIO, IOBase, SEEK_CUR, SEEK_END, SEEK_SET, UnsupportedOperation) +from threading import Lock + +from math import ceil + +from ..common._common_conversion import _encode_base64 +from ..common._error import _ERROR_VALUE_SHOULD_BE_SEEKABLE_STREAM +from ..common._serialization import ( + url_quote, + _get_data_bytes_only, + _len_plus +) +from ._constants import ( + _LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE +) +from ._encryption import ( + _get_blob_encryptor_and_padder, +) +from .models import BlobBlock + + +def _upload_blob_chunks(blob_service, container_name, blob_name, + blob_size, block_size, stream, max_connections, + progress_callback, validate_content, lease_id, uploader_class, + maxsize_condition=None, if_modified_since=None, if_unmodified_since=None, if_match=None, + if_none_match=None, timeout=None, + content_encryption_key=None, initialization_vector=None, resource_properties=None): + encryptor, padder = _get_blob_encryptor_and_padder(content_encryption_key, initialization_vector, + uploader_class is not _PageBlobChunkUploader) + + uploader = uploader_class( + blob_service, + container_name, + blob_name, + blob_size, + block_size, + stream, + max_connections > 1, + progress_callback, + validate_content, + lease_id, + timeout, + encryptor, + padder + ) + + uploader.maxsize_condition = maxsize_condition + + # Access conditions do not work with parallelism + if max_connections > 1: + uploader.if_match = uploader.if_none_match = uploader.if_modified_since = uploader.if_unmodified_since = None + else: + uploader.if_match = if_match + uploader.if_none_match = if_none_match + uploader.if_modified_since = if_modified_since + uploader.if_unmodified_since = if_unmodified_since + + if progress_callback is not None: + progress_callback(0, blob_size) + + if max_connections > 1: + import concurrent.futures + from threading import BoundedSemaphore + + ''' + Ensures we bound the chunking so we only buffer and submit 'max_connections' amount of work items to the executor. + This is necessary as the executor queue will keep accepting submitted work items, which results in buffering all the blocks if + the max_connections + 1 ensures the next chunk is already buffered and ready for when the worker thread is available. + ''' + chunk_throttler = BoundedSemaphore(max_connections + 1) + + executor = concurrent.futures.ThreadPoolExecutor(max_connections) + futures = [] + running_futures = [] + + # Check for exceptions and fail fast. + for chunk in uploader.get_chunk_streams(): + for f in running_futures: + if f.done(): + if f.exception(): + raise f.exception() + else: + running_futures.remove(f) + + chunk_throttler.acquire() + future = executor.submit(uploader.process_chunk, chunk) + + # Calls callback upon completion (even if the callback was added after the Future task is done). + future.add_done_callback(lambda x: chunk_throttler.release()) + futures.append(future) + running_futures.append(future) + + # result() will wait until completion and also raise any exceptions that may have been set. + range_ids = [f.result() for f in futures] + else: + range_ids = [uploader.process_chunk(result) for result in uploader.get_chunk_streams()] + + if resource_properties: + resource_properties.last_modified = uploader.last_modified + resource_properties.etag = uploader.etag + + return range_ids + + +def _upload_blob_substream_blocks(blob_service, container_name, blob_name, + blob_size, block_size, stream, max_connections, + progress_callback, validate_content, lease_id, uploader_class, + maxsize_condition=None, if_match=None, timeout=None): + uploader = uploader_class( + blob_service, + container_name, + blob_name, + blob_size, + block_size, + stream, + max_connections > 1, + progress_callback, + validate_content, + lease_id, + timeout, + None, + None + ) + + uploader.maxsize_condition = maxsize_condition + + # ETag matching does not work with parallelism as a ranged upload may start + # before the previous finishes and provides an etag + uploader.if_match = if_match if not max_connections > 1 else None + + if progress_callback is not None: + progress_callback(0, blob_size) + + if max_connections > 1: + import concurrent.futures + executor = concurrent.futures.ThreadPoolExecutor(max_connections) + range_ids = list(executor.map(uploader.process_substream_block, uploader.get_substream_blocks())) + else: + range_ids = [uploader.process_substream_block(result) for result in uploader.get_substream_blocks()] + + return range_ids + + +class _BlobChunkUploader(object): + def __init__(self, blob_service, container_name, blob_name, blob_size, + chunk_size, stream, parallel, progress_callback, + validate_content, lease_id, timeout, encryptor, padder): + self.blob_service = blob_service + self.container_name = container_name + self.blob_name = blob_name + self.blob_size = blob_size + self.chunk_size = chunk_size + self.stream = stream + self.parallel = parallel + self.stream_start = stream.tell() if parallel else None + self.stream_lock = Lock() if parallel else None + self.progress_callback = progress_callback + self.progress_total = 0 + self.progress_lock = Lock() if parallel else None + self.validate_content = validate_content + self.lease_id = lease_id + self.timeout = timeout + self.encryptor = encryptor + self.padder = padder + self.last_modified = None + self.etag = None + + def get_chunk_streams(self): + index = 0 + while True: + data = b'' + read_size = self.chunk_size + + # Buffer until we either reach the end of the stream or get a whole chunk. + while True: + if self.blob_size: + read_size = min(self.chunk_size - len(data), self.blob_size - (index + len(data))) + temp = self.stream.read(read_size) + temp = _get_data_bytes_only('temp', temp) + data += temp + + # We have read an empty string and so are at the end + # of the buffer or we have read a full chunk. + if temp == b'' or len(data) == self.chunk_size: + break + + if len(data) == self.chunk_size: + if self.padder: + data = self.padder.update(data) + if self.encryptor: + data = self.encryptor.update(data) + yield index, data + else: + if self.padder: + data = self.padder.update(data) + self.padder.finalize() + if self.encryptor: + data = self.encryptor.update(data) + self.encryptor.finalize() + if len(data) > 0: + yield index, data + break + index += len(data) + + def process_chunk(self, chunk_data): + chunk_bytes = chunk_data[1] + chunk_offset = chunk_data[0] + return self._upload_chunk_with_progress(chunk_offset, chunk_bytes) + + def _update_progress(self, length): + if self.progress_callback is not None: + if self.progress_lock is not None: + with self.progress_lock: + self.progress_total += length + total = self.progress_total + else: + self.progress_total += length + total = self.progress_total + self.progress_callback(total, self.blob_size) + + def _upload_chunk_with_progress(self, chunk_offset, chunk_data): + range_id = self._upload_chunk(chunk_offset, chunk_data) + self._update_progress(len(chunk_data)) + return range_id + + def get_substream_blocks(self): + assert self.chunk_size is not None + lock = self.stream_lock + blob_length = self.blob_size + + if blob_length is None: + blob_length = _len_plus(self.stream) + if blob_length is None: + raise ValueError(_ERROR_VALUE_SHOULD_BE_SEEKABLE_STREAM.format('stream')) + + blocks = int(ceil(blob_length / (self.chunk_size * 1.0))) + last_block_size = self.chunk_size if blob_length % self.chunk_size == 0 else blob_length % self.chunk_size + + for i in range(blocks): + yield ('BlockId{}'.format("%05d" % i), + _SubStream(self.stream, i * self.chunk_size, last_block_size if i == blocks - 1 else self.chunk_size, + lock)) + + def process_substream_block(self, block_data): + return self._upload_substream_block_with_progress(block_data[0], block_data[1]) + + def _upload_substream_block_with_progress(self, block_id, block_stream): + range_id = self._upload_substream_block(block_id, block_stream) + self._update_progress(len(block_stream)) + return range_id + + def set_response_properties(self, resp): + self.etag = resp.etag + self.last_modified = resp.last_modified + + +class _BlockBlobChunkUploader(_BlobChunkUploader): + def _upload_chunk(self, chunk_offset, chunk_data): + block_id = url_quote(_encode_base64('{0:032d}'.format(chunk_offset))) + self.blob_service._put_block( + self.container_name, + self.blob_name, + chunk_data, + block_id, + validate_content=self.validate_content, + lease_id=self.lease_id, + timeout=self.timeout, + ) + return BlobBlock(block_id) + + def _upload_substream_block(self, block_id, block_stream): + try: + self.blob_service._put_block( + self.container_name, + self.blob_name, + block_stream, + block_id, + validate_content=self.validate_content, + lease_id=self.lease_id, + timeout=self.timeout, + ) + finally: + block_stream.close() + return BlobBlock(block_id) + + +class _PageBlobChunkUploader(_BlobChunkUploader): + def _is_chunk_empty(self, chunk_data): + # read until non-zero byte is encountered + # if reached the end without returning, then chunk_data is all 0's + for each_byte in chunk_data: + if each_byte != 0 and each_byte != b'\x00': + return False + return True + + def _upload_chunk(self, chunk_start, chunk_data): + # avoid uploading the empty pages + if not self._is_chunk_empty(chunk_data): + chunk_end = chunk_start + len(chunk_data) - 1 + resp = self.blob_service._update_page( + self.container_name, + self.blob_name, + chunk_data, + chunk_start, + chunk_end, + validate_content=self.validate_content, + lease_id=self.lease_id, + if_match=self.if_match, + timeout=self.timeout, + ) + + if not self.parallel: + self.if_match = resp.etag + + self.set_response_properties(resp) + + +class _AppendBlobChunkUploader(_BlobChunkUploader): + def _upload_chunk(self, chunk_offset, chunk_data): + if not hasattr(self, 'current_length'): + resp = self.blob_service.append_block( + self.container_name, + self.blob_name, + chunk_data, + validate_content=self.validate_content, + lease_id=self.lease_id, + maxsize_condition=self.maxsize_condition, + timeout=self.timeout, + if_modified_since=self.if_modified_since, + if_unmodified_since=self.if_unmodified_since, + if_match=self.if_match, + if_none_match=self.if_none_match + ) + + self.current_length = resp.append_offset + else: + resp = self.blob_service.append_block( + self.container_name, + self.blob_name, + chunk_data, + validate_content=self.validate_content, + lease_id=self.lease_id, + maxsize_condition=self.maxsize_condition, + appendpos_condition=self.current_length + chunk_offset, + timeout=self.timeout, + ) + + self.set_response_properties(resp) + + +class _SubStream(IOBase): + def __init__(self, wrapped_stream, stream_begin_index, length, lockObj): + # Python 2.7: file-like objects created with open() typically support seek(), but are not + # derivations of io.IOBase and thus do not implement seekable(). + # Python > 3.0: file-like objects created with open() are derived from io.IOBase. + try: + # only the main thread runs this, so there's no need grabbing the lock + wrapped_stream.seek(0, SEEK_CUR) + except: + raise ValueError("Wrapped stream must support seek().") + + self._lock = lockObj + self._wrapped_stream = wrapped_stream + self._position = 0 + self._stream_begin_index = stream_begin_index + self._length = length + self._buffer = BytesIO() + + # we must avoid buffering more than necessary, and also not use up too much memory + # so the max buffer size is capped at 4MB + self._max_buffer_size = length if length < _LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE \ + else _LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE + self._current_buffer_start = 0 + self._current_buffer_size = 0 + + def __len__(self): + return self._length + + def close(self): + if self._buffer: + self._buffer.close() + self._wrapped_stream = None + IOBase.close(self) + + def fileno(self): + return self._wrapped_stream.fileno() + + def flush(self): + pass + + def read(self, n): + if self.closed: + raise ValueError("Stream is closed.") + + # adjust if out of bounds + if n + self._position >= self._length: + n = self._length - self._position + + # return fast + if n is 0 or self._buffer.closed: + return b'' + + # attempt first read from the read buffer and update position + read_buffer = self._buffer.read(n) + bytes_read = len(read_buffer) + bytes_remaining = n - bytes_read + self._position += bytes_read + + # repopulate the read buffer from the underlying stream to fulfill the request + # ensure the seek and read operations are done atomically (only if a lock is provided) + if bytes_remaining > 0: + with self._buffer: + # either read in the max buffer size specified on the class + # or read in just enough data for the current block/sub stream + current_max_buffer_size = min(self._max_buffer_size, self._length - self._position) + + # lock is only defined if max_connections > 1 (parallel uploads) + if self._lock: + with self._lock: + # reposition the underlying stream to match the start of the data to read + absolute_position = self._stream_begin_index + self._position + self._wrapped_stream.seek(absolute_position, SEEK_SET) + # If we can't seek to the right location, our read will be corrupted so fail fast. + if self._wrapped_stream.tell() != absolute_position: + raise IOError("Stream failed to seek to the desired location.") + buffer_from_stream = self._wrapped_stream.read(current_max_buffer_size) + else: + buffer_from_stream = self._wrapped_stream.read(current_max_buffer_size) + + if buffer_from_stream: + # update the buffer with new data from the wrapped stream + # we need to note down the start position and size of the buffer, in case seek is performed later + self._buffer = BytesIO(buffer_from_stream) + self._current_buffer_start = self._position + self._current_buffer_size = len(buffer_from_stream) + + # read the remaining bytes from the new buffer and update position + second_read_buffer = self._buffer.read(bytes_remaining) + read_buffer += second_read_buffer + self._position += len(second_read_buffer) + + return read_buffer + + def readable(self): + return True + + def readinto(self, b): + raise UnsupportedOperation + + def seek(self, offset, whence=0): + if whence is SEEK_SET: + start_index = 0 + elif whence is SEEK_CUR: + start_index = self._position + elif whence is SEEK_END: + start_index = self._length + offset = - offset + else: + raise ValueError("Invalid argument for the 'whence' parameter.") + + pos = start_index + offset + + if pos > self._length: + pos = self._length + elif pos < 0: + pos = 0 + + # check if buffer is still valid + # if not, drop buffer + if pos < self._current_buffer_start or pos >= self._current_buffer_start + self._current_buffer_size: + self._buffer.close() + self._buffer = BytesIO() + else: # if yes seek to correct position + delta = pos - self._current_buffer_start + self._buffer.seek(delta, SEEK_SET) + + self._position = pos + return pos + + def seekable(self): + return True + + def tell(self): + return self._position + + def write(self): + raise UnsupportedOperation + + def writelines(self): + raise UnsupportedOperation + + def writeable(self): + return False diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/appendblobservice.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/appendblobservice.py new file mode 100644 index 000000000000..8369cb3727e9 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/appendblobservice.py @@ -0,0 +1,661 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import sys +from os import path + +from ..common._common_conversion import ( + _to_str, + _int_to_str, + _datetime_to_utc_string, + _get_content_md5, +) +from ..common._constants import ( + SERVICE_HOST_BASE, + DEFAULT_PROTOCOL, +) +from ..common._error import ( + _validate_not_none, + _validate_type_bytes, + _validate_encryption_unsupported, + _ERROR_VALUE_NEGATIVE, +) +from ..common._http import HTTPRequest +from ..common._serialization import ( + _get_data_bytes_only, + _add_metadata_headers, +) +from ._deserialization import ( + _parse_append_block, + _parse_base_properties, +) +from ._serialization import ( + _get_path, +) +from ._upload_chunking import ( + _AppendBlobChunkUploader, + _upload_blob_chunks, +) +from .baseblobservice import BaseBlobService +from .models import ( + _BlobTypes, + ResourceProperties +) + +if sys.version_info >= (3,): + from io import BytesIO +else: + from cStringIO import StringIO as BytesIO + + +class AppendBlobService(BaseBlobService): + ''' + An append blob is comprised of blocks and is optimized for append operations. + When you modify an append blob, blocks are added to the end of the blob only, + via the append_block operation. Updating or deleting of existing blocks is not + supported. Unlike a block blob, an append blob does not expose its block IDs. + + Each block in an append blob can be a different size, up to a maximum of 4 MB, + and an append blob can include up to 50,000 blocks. The maximum size of an + append blob is therefore slightly more than 195 GB (4 MB X 50,000 blocks). + + :ivar int MAX_BLOCK_SIZE: + The size of the blocks put by append_blob_from_* methods. Smaller blocks + may be put if there is less data provided. The maximum block size the service + supports is 4MB. + ''' + MAX_BLOCK_SIZE = 4 * 1024 * 1024 + + def __init__(self, account_name=None, account_key=None, sas_token=None, is_emulated=False, + protocol=DEFAULT_PROTOCOL, endpoint_suffix=SERVICE_HOST_BASE, custom_domain=None, request_session=None, + connection_string=None, socket_timeout=None, token_credential=None): + ''' + :param str account_name: + The storage account name. This is used to authenticate requests + signed with an account key and to construct the storage endpoint. It + is required unless a connection string is given, or if a custom + domain is used with anonymous authentication. + :param str account_key: + The storage account key. This is used for shared key authentication. + If neither account key or sas token is specified, anonymous access + will be used. + :param str sas_token: + A shared access signature token to use to authenticate requests + instead of the account key. If account key and sas token are both + specified, account key will be used to sign. If neither are + specified, anonymous access will be used. + :param bool is_emulated: + Whether to use the emulator. Defaults to False. If specified, will + override all other parameters besides connection string and request + session. + :param str protocol: + The protocol to use for requests. Defaults to https. + :param str endpoint_suffix: + The host base component of the url, minus the account name. Defaults + to Azure (core.windows.net). Override this to use the China cloud + (core.chinacloudapi.cn). + :param str custom_domain: + The custom domain to use. This can be set in the Azure Portal. For + example, 'www.mydomain.com'. + :param requests.Session request_session: + The session object to use for http requests. + :param str connection_string: + If specified, this will override all other parameters besides + request session. See + http://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/ + for the connection string format. + :param int socket_timeout: + If specified, this will override the default socket timeout. The timeout specified is in seconds. + See DEFAULT_SOCKET_TIMEOUT in _constants.py for the default value. + :param token_credential: + A token credential used to authenticate HTTPS requests. The token value + should be updated before its expiration. + :type `~..common.TokenCredential` + ''' + self.blob_type = _BlobTypes.AppendBlob + super(AppendBlobService, self).__init__( + account_name, account_key, sas_token, is_emulated, protocol, endpoint_suffix, + custom_domain, request_session, connection_string, socket_timeout, token_credential) + + def create_blob(self, container_name, blob_name, content_settings=None, + metadata=None, lease_id=None, + if_modified_since=None, if_unmodified_since=None, + if_match=None, if_none_match=None, timeout=None): + ''' + Creates a blob or overrides an existing blob. Use if_none_match=* to + prevent overriding an existing blob. + + See create_blob_from_* for high level + functions that handle the creation and upload of large blobs with + automatic chunking and progress notifications. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of blob to create or update. + :param ~azure.storage.blob.models.ContentSettings content_settings: + ContentSettings object used to set blob properties. + :param metadata: + Name-value pairs associated with the blob as metadata. + :type metadata: dict(str, str) + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to + perform the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: ETag and last modified properties for the updated Append Blob + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_encryption_unsupported(self.require_encryption, self.key_encryption_key) + + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.query = {'timeout': _int_to_str(timeout)} + request.headers = { + 'x-ms-blob-type': _to_str(self.blob_type), + 'x-ms-lease-id': _to_str(lease_id), + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'If-Match': _to_str(if_match), + 'If-None-Match': _to_str(if_none_match) + } + _add_metadata_headers(metadata, request) + if content_settings is not None: + request.headers.update(content_settings._to_headers()) + + return self._perform_request(request, _parse_base_properties) + + def append_block(self, container_name, blob_name, block, + validate_content=False, maxsize_condition=None, + appendpos_condition=None, + lease_id=None, if_modified_since=None, + if_unmodified_since=None, if_match=None, + if_none_match=None, timeout=None): + ''' + Commits a new block of data to the end of an existing append blob. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param bytes block: + Content of the block in bytes. + :param bool validate_content: + If true, calculates an MD5 hash of the block content. The storage + service checks the hash of the content that has arrived + with the hash that was sent. This is primarily valuable for detecting + bitflips on the wire if using http instead of https as https (the default) + will already validate. Note that this MD5 hash is not stored with the + blob. + :param int maxsize_condition: + Optional conditional header. The max length in bytes permitted for + the append blob. If the Append Block operation would cause the blob + to exceed that limit or if the blob size is already greater than the + value specified in this header, the request will fail with + MaxBlobSizeConditionNotMet error (HTTP status code 412 - Precondition Failed). + :param int appendpos_condition: + Optional conditional header, used only for the Append Block operation. + A number indicating the byte offset to compare. Append Block will + succeed only if the append position is equal to this number. If it + is not, the request will fail with the + AppendPositionConditionNotMet error + (HTTP status code 412 - Precondition Failed). + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: + ETag, last modified, append offset, and committed block count + properties for the updated Append Blob + :rtype: :class:`~azure.storage.blob.models.AppendBlockProperties` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('block', block) + _validate_encryption_unsupported(self.require_encryption, self.key_encryption_key) + + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.query = { + 'comp': 'appendblock', + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-blob-condition-maxsize': _to_str(maxsize_condition), + 'x-ms-blob-condition-appendpos': _to_str(appendpos_condition), + 'x-ms-lease-id': _to_str(lease_id), + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'If-Match': _to_str(if_match), + 'If-None-Match': _to_str(if_none_match) + } + request.body = _get_data_bytes_only('block', block) + + if validate_content: + computed_md5 = _get_content_md5(request.body) + request.headers['Content-MD5'] = _to_str(computed_md5) + + return self._perform_request(request, _parse_append_block) + + # ----Convenience APIs---------------------------------------------- + + def append_blob_from_path( + self, container_name, blob_name, file_path, validate_content=False, + maxsize_condition=None, progress_callback=None, lease_id=None, timeout=None, + if_modified_since=None, if_unmodified_since=None, if_match=None, + if_none_match=None): + ''' + Appends to the content of an existing blob from a file path, with automatic + chunking and progress notifications. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of blob to create or update. + :param str file_path: + Path of the file to upload as the blob content. + :param bool validate_content: + If true, calculates an MD5 hash for each chunk of the blob. The storage + service checks the hash of the content that has arrived with the hash + that was sent. This is primarily valuable for detecting bitflips on + the wire if using http instead of https as https (the default) will + already validate. Note that this MD5 hash is not stored with the + blob. + :param int maxsize_condition: + Optional conditional header. The max length in bytes permitted for + the append blob. If the Append Block operation would cause the blob + to exceed that limit or if the blob size is already greater than the + value specified in this header, the request will fail with + MaxBlobSizeConditionNotMet error (HTTP status code 412 - Precondition Failed). + :param progress_callback: + Callback for progress with signature function(current, total) where + current is the number of bytes transfered so far, and total is the + size of the blob, or None if the total size is unknown. + :type progress_callback: func(current, total) + :param str lease_id: + Required if the blob has an active lease. + :param int timeout: + The timeout parameter is expressed in seconds. This method may make + multiple calls to the Azure service and the timeout will apply to + each call individually. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetime will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetime will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :return: ETag and last modified properties for the Append Blob + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('file_path', file_path) + _validate_encryption_unsupported(self.require_encryption, self.key_encryption_key) + + count = path.getsize(file_path) + with open(file_path, 'rb') as stream: + return self.append_blob_from_stream( + container_name, + blob_name, + stream, + count=count, + validate_content=validate_content, + maxsize_condition=maxsize_condition, + progress_callback=progress_callback, + lease_id=lease_id, + timeout=timeout, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + if_match=if_match, + if_none_match=if_none_match) + + def append_blob_from_bytes( + self, container_name, blob_name, blob, index=0, count=None, + validate_content=False, maxsize_condition=None, progress_callback=None, + lease_id=None, timeout=None, if_modified_since=None, if_unmodified_since=None, if_match=None, + if_none_match=None): + ''' + Appends to the content of an existing blob from an array of bytes, with + automatic chunking and progress notifications. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of blob to create or update. + :param bytes blob: + Content of blob as an array of bytes. + :param int index: + Start index in the array of bytes. + :param int count: + Number of bytes to upload. Set to None or negative value to upload + all bytes starting from index. + :param bool validate_content: + If true, calculates an MD5 hash for each chunk of the blob. The storage + service checks the hash of the content that has arrived with the hash + that was sent. This is primarily valuable for detecting bitflips on + the wire if using http instead of https as https (the default) will + already validate. Note that this MD5 hash is not stored with the + blob. + :param int maxsize_condition: + Optional conditional header. The max length in bytes permitted for + the append blob. If the Append Block operation would cause the blob + to exceed that limit or if the blob size is already greater than the + value specified in this header, the request will fail with + MaxBlobSizeConditionNotMet error (HTTP status code 412 - Precondition Failed). + :param progress_callback: + Callback for progress with signature function(current, total) where + current is the number of bytes transfered so far, and total is the + size of the blob, or None if the total size is unknown. + :type progress_callback: func(current, total) + :param str lease_id: + Required if the blob has an active lease. + :param int timeout: + The timeout parameter is expressed in seconds. This method may make + multiple calls to the Azure service and the timeout will apply to + each call individually. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetime will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetime will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :return: ETag and last modified properties for the Append Blob + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('blob', blob) + _validate_not_none('index', index) + _validate_type_bytes('blob', blob) + _validate_encryption_unsupported(self.require_encryption, self.key_encryption_key) + + if index < 0: + raise IndexError(_ERROR_VALUE_NEGATIVE.format('index')) + + if count is None or count < 0: + count = len(blob) - index + + stream = BytesIO(blob) + stream.seek(index) + + return self.append_blob_from_stream( + container_name, + blob_name, + stream, + count=count, + validate_content=validate_content, + maxsize_condition=maxsize_condition, + lease_id=lease_id, + progress_callback=progress_callback, + timeout=timeout, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + if_match=if_match, + if_none_match=if_none_match) + + def append_blob_from_text( + self, container_name, blob_name, text, encoding='utf-8', + validate_content=False, maxsize_condition=None, progress_callback=None, + lease_id=None, timeout=None, if_modified_since=None, if_unmodified_since=None, if_match=None, + if_none_match=None): + ''' + Appends to the content of an existing blob from str/unicode, with + automatic chunking and progress notifications. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of blob to create or update. + :param str text: + Text to upload to the blob. + :param str encoding: + Python encoding to use to convert the text to bytes. + :param bool validate_content: + If true, calculates an MD5 hash for each chunk of the blob. The storage + service checks the hash of the content that has arrived with the hash + that was sent. This is primarily valuable for detecting bitflips on + the wire if using http instead of https as https (the default) will + already validate. Note that this MD5 hash is not stored with the + blob. + :param int maxsize_condition: + Optional conditional header. The max length in bytes permitted for + the append blob. If the Append Block operation would cause the blob + to exceed that limit or if the blob size is already greater than the + value specified in this header, the request will fail with + MaxBlobSizeConditionNotMet error (HTTP status code 412 - Precondition Failed). + :param progress_callback: + Callback for progress with signature function(current, total) where + current is the number of bytes transfered so far, and total is the + size of the blob, or None if the total size is unknown. + :type progress_callback: func(current, total) + :param str lease_id: + Required if the blob has an active lease. + :param int timeout: + The timeout parameter is expressed in seconds. This method may make + multiple calls to the Azure service and the timeout will apply to + each call individually. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetime will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetime will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :return: ETag and last modified properties for the Append Blob + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('text', text) + _validate_encryption_unsupported(self.require_encryption, self.key_encryption_key) + + if not isinstance(text, bytes): + _validate_not_none('encoding', encoding) + text = text.encode(encoding) + + return self.append_blob_from_bytes( + container_name, + blob_name, + text, + index=0, + count=len(text), + validate_content=validate_content, + maxsize_condition=maxsize_condition, + lease_id=lease_id, + progress_callback=progress_callback, + timeout=timeout, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + if_match=if_match, + if_none_match=if_none_match) + + def append_blob_from_stream( + self, container_name, blob_name, stream, count=None, + validate_content=False, maxsize_condition=None, progress_callback=None, + lease_id=None, timeout=None, if_modified_since=None, if_unmodified_since=None, if_match=None, + if_none_match=None): + ''' + Appends to the content of an existing blob from a file/stream, with + automatic chunking and progress notifications. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of blob to create or update. + :param io.IOBase stream: + Opened stream to upload as the blob content. + :param int count: + Number of bytes to read from the stream. This is optional, but + should be supplied for optimal performance. + :param bool validate_content: + If true, calculates an MD5 hash for each chunk of the blob. The storage + service checks the hash of the content that has arrived with the hash + that was sent. This is primarily valuable for detecting bitflips on + the wire if using http instead of https as https (the default) will + already validate. Note that this MD5 hash is not stored with the + blob. + :param int maxsize_condition: + Conditional header. The max length in bytes permitted for + the append blob. If the Append Block operation would cause the blob + to exceed that limit or if the blob size is already greater than the + value specified in this header, the request will fail with + MaxBlobSizeConditionNotMet error (HTTP status code 412 - Precondition Failed). + :param progress_callback: + Callback for progress with signature function(current, total) where + current is the number of bytes transfered so far, and total is the + size of the blob, or None if the total size is unknown. + :type progress_callback: func(current, total) + :param str lease_id: + Required if the blob has an active lease. + :param int timeout: + The timeout parameter is expressed in seconds. This method may make + multiple calls to the Azure service and the timeout will apply to + each call individually. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetime will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetime will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :return: ETag and last modified properties for the Append Blob + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('stream', stream) + _validate_encryption_unsupported(self.require_encryption, self.key_encryption_key) + + # _upload_blob_chunks returns the block ids for block blobs so resource_properties + # is passed as a parameter to get the last_modified and etag for page and append blobs. + # this info is not needed for block_blobs since _put_block_list is called after which gets this info + resource_properties = ResourceProperties() + _upload_blob_chunks( + blob_service=self, + container_name=container_name, + blob_name=blob_name, + blob_size=count, + block_size=self.MAX_BLOCK_SIZE, + stream=stream, + max_connections=1, # upload not easily parallelizable + progress_callback=progress_callback, + validate_content=validate_content, + lease_id=lease_id, + uploader_class=_AppendBlobChunkUploader, + maxsize_condition=maxsize_condition, + timeout=timeout, + resource_properties=resource_properties, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + if_match=if_match, + if_none_match=if_none_match + ) + + return resource_properties diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/baseblobservice.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/baseblobservice.py new file mode 100644 index 000000000000..adb9127ca5f5 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/baseblobservice.py @@ -0,0 +1,3280 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import sys +from abc import ABCMeta + +from azure.common import AzureHttpError + +from ..common._auth import ( + _StorageSASAuthentication, + _StorageSharedKeyAuthentication, + _StorageNoAuthentication, +) +from ..common._common_conversion import ( + _int_to_str, + _to_str, + _datetime_to_utc_string, +) +from ..common._connection import _ServiceParameters +from ..common._constants import ( + SERVICE_HOST_BASE, + DEFAULT_PROTOCOL, +) +from ..common._deserialization import ( + _convert_xml_to_service_properties, + _parse_metadata, + _parse_properties, + _convert_xml_to_service_stats, + _parse_length_from_content_range, +) +from ..common._error import ( + _dont_fail_not_exist, + _dont_fail_on_exist, + _validate_not_none, + _validate_decryption_required, + _validate_access_policies, + _ERROR_PARALLEL_NOT_SEEKABLE, +) +from ..common._http import HTTPRequest +from ..common._serialization import ( + _get_request_body, + _convert_signed_identifiers_to_xml, + _convert_service_properties_to_xml, + _add_metadata_headers, +) +from ..common.models import ( + Services, + ListGenerator, + _OperationContext, +) + +from .sharedaccesssignature import ( + BlobSharedAccessSignature, +) +from ..common.storageclient import StorageClient +from ._deserialization import ( + _convert_xml_to_containers, + _parse_blob, + _convert_xml_to_blob_list, + _parse_container, + _parse_snapshot_blob, + _parse_lease, + _convert_xml_to_signed_identifiers_and_access, + _parse_base_properties, + _parse_account_information, +) +from ._download_chunking import _download_blob_chunks +from ._error import ( + _ERROR_INVALID_LEASE_DURATION, + _ERROR_INVALID_LEASE_BREAK_PERIOD, +) +from ._serialization import ( + _get_path, + _validate_and_format_range_headers, +) +from .models import ( + BlobProperties, + _LeaseActions, + ContainerPermissions, + BlobPermissions, +) + +from ._constants import ( + X_MS_VERSION, + __version__ as package_version, +) + +_CONTAINER_ALREADY_EXISTS_ERROR_CODE = 'ContainerAlreadyExists' +_BLOB_NOT_FOUND_ERROR_CODE = 'BlobNotFound' +_CONTAINER_NOT_FOUND_ERROR_CODE = 'ContainerNotFound' + +if sys.version_info >= (3,): + from io import BytesIO +else: + from cStringIO import StringIO as BytesIO + + +class BaseBlobService(StorageClient): + ''' + This is the main class managing Blob resources. + + The Blob service stores text and binary data as blobs in the cloud. + The Blob service offers the following three resources: the storage account, + containers, and blobs. Within your storage account, containers provide a + way to organize sets of blobs. For more information please see: + https://msdn.microsoft.com/en-us/library/azure/ee691964.aspx + + :ivar int MAX_SINGLE_GET_SIZE: + The size of the first range get performed by get_blob_to_* methods if + max_connections is greater than 1. Less data will be returned if the + blob is smaller than this. + :ivar int MAX_CHUNK_GET_SIZE: + The size of subsequent range gets performed by get_blob_to_* methods if + max_connections is greater than 1 and the blob is larger than MAX_SINGLE_GET_SIZE. + Less data will be returned if the remainder of the blob is smaller than + this. If this is set to larger than 4MB, content_validation will throw an + error if enabled. However, if content_validation is not desired a size + greater than 4MB may be optimal. Setting this below 4MB is not recommended. + :ivar object key_encryption_key: + The key-encryption-key optionally provided by the user. If provided, will be used to + encrypt/decrypt in supported methods. + For methods requiring decryption, either the key_encryption_key OR the resolver must be provided. + If both are provided, the resolver will take precedence. + Must implement the following methods for APIs requiring encryption: + wrap_key(key)--wraps the specified key (bytes) using an algorithm of the user's choice. Returns the encrypted key as bytes. + get_key_wrap_algorithm()--returns the algorithm used to wrap the specified symmetric key. + get_kid()--returns a string key id for this key-encryption-key. + Must implement the following methods for APIs requiring decryption: + unwrap_key(key, algorithm)--returns the unwrapped form of the specified symmetric key using the string-specified algorithm. + get_kid()--returns a string key id for this key-encryption-key. + :ivar function key_resolver_function(kid): + A function to resolve keys optionally provided by the user. If provided, will be used to decrypt in supported methods. + For methods requiring decryption, either the key_encryption_key OR + the resolver must be provided. If both are provided, the resolver will take precedence. + It uses the kid string to return a key-encryption-key implementing the interface defined above. + :ivar bool require_encryption: + A flag that may be set to ensure that all messages successfully uploaded to the queue and all those downloaded and + successfully read from the queue are/were encrypted while on the server. If this flag is set, all required + parameters for encryption/decryption must be provided. See the above comments on the key_encryption_key and resolver. + ''' + + __metaclass__ = ABCMeta + MAX_SINGLE_GET_SIZE = 32 * 1024 * 1024 + MAX_CHUNK_GET_SIZE = 4 * 1024 * 1024 + + def __init__(self, account_name=None, account_key=None, sas_token=None, is_emulated=False, + protocol=DEFAULT_PROTOCOL, endpoint_suffix=SERVICE_HOST_BASE, custom_domain=None, request_session=None, + connection_string=None, socket_timeout=None, token_credential=None): + ''' + :param str account_name: + The storage account name. This is used to authenticate requests + signed with an account key and to construct the storage endpoint. It + is required unless a connection string is given, or if a custom + domain is used with anonymous authentication. + :param str account_key: + The storage account key. This is used for shared key authentication. + If neither account key or sas token is specified, anonymous access + will be used. + :param str sas_token: + A shared access signature token to use to authenticate requests + instead of the account key. If account key and sas token are both + specified, account key will be used to sign. If neither are + specified, anonymous access will be used. + :param bool is_emulated: + Whether to use the emulator. Defaults to False. If specified, will + override all other parameters besides connection string and request + session. + :param str protocol: + The protocol to use for requests. Defaults to https. + :param str endpoint_suffix: + The host base component of the url, minus the account name. Defaults + to Azure (core.windows.net). Override this to use the China cloud + (core.chinacloudapi.cn). + :param str custom_domain: + The custom domain to use. This can be set in the Azure Portal. For + example, 'www.mydomain.com'. + :param requests.Session request_session: + The session object to use for http requests. + :param str connection_string: + If specified, this will override all other parameters besides + request session. See + http://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/ + for the connection string format + :param int socket_timeout: + If specified, this will override the default socket timeout. The timeout specified is in seconds. + See DEFAULT_SOCKET_TIMEOUT in _constants.py for the default value. + :param token_credential: + A token credential used to authenticate HTTPS requests. The token value + should be updated before its expiration. + :type `~..common.TokenCredential` + ''' + service_params = _ServiceParameters.get_service_parameters( + 'blob', + account_name=account_name, + account_key=account_key, + sas_token=sas_token, + token_credential=token_credential, + is_emulated=is_emulated, + protocol=protocol, + endpoint_suffix=endpoint_suffix, + custom_domain=custom_domain, + request_session=request_session, + connection_string=connection_string, + socket_timeout=socket_timeout) + + super(BaseBlobService, self).__init__(service_params) + + if self.account_key: + self.authentication = _StorageSharedKeyAuthentication( + self.account_name, + self.account_key, + self.is_emulated + ) + elif self.sas_token: + self.authentication = _StorageSASAuthentication(self.sas_token) + elif self.token_credential: + self.authentication = self.token_credential + else: + self.authentication = _StorageNoAuthentication() + + self.require_encryption = False + self.key_encryption_key = None + self.key_resolver_function = None + self._X_MS_VERSION = X_MS_VERSION + self._update_user_agent_string(package_version) + + def make_blob_url(self, container_name, blob_name, protocol=None, sas_token=None, snapshot=None): + ''' + Creates the url to access a blob. + + :param str container_name: + Name of container. + :param str blob_name: + Name of blob. + :param str protocol: + Protocol to use: 'http' or 'https'. If not specified, uses the + protocol specified when BaseBlobService was initialized. + :param str sas_token: + Shared access signature token created with + generate_shared_access_signature. + :param str snapshot: + An string value that uniquely identifies the snapshot. The value of + this query parameter indicates the snapshot version. + :return: blob access URL. + :rtype: str + ''' + + url = '{}://{}/{}/{}'.format( + protocol or self.protocol, + self.primary_endpoint, + container_name, + blob_name, + ) + + if snapshot and sas_token: + url = '{}?snapshot={}&{}'.format(url, snapshot, sas_token) + elif snapshot: + url = '{}?snapshot={}'.format(url, snapshot) + elif sas_token: + url = '{}?{}'.format(url, sas_token) + + return url + + def make_container_url(self, container_name, protocol=None, sas_token=None): + ''' + Creates the url to access a container. + + :param str container_name: + Name of container. + :param str protocol: + Protocol to use: 'http' or 'https'. If not specified, uses the + protocol specified when BaseBlobService was initialized. + :param str sas_token: + Shared access signature token created with + generate_shared_access_signature. + :return: container access URL. + :rtype: str + ''' + + url = '{}://{}/{}?restype=container'.format( + protocol or self.protocol, + self.primary_endpoint, + container_name, + ) + + if sas_token: + url = '{}&{}'.format(url, sas_token) + + return url + + def generate_account_shared_access_signature(self, resource_types, permission, + expiry, start=None, ip=None, protocol=None): + ''' + Generates a shared access signature for the blob service. + Use the returned signature with the sas_token parameter of any BlobService. + + :param ResourceTypes resource_types: + Specifies the resource types that are accessible with the account SAS. + :param AccountPermissions permission: + The permissions associated with the shared access signature. The + user is restricted to operations allowed by the permissions. + Required unless an id is given referencing a stored access policy + which contains this field. This field must be omitted if it has been + specified in an associated stored access policy. + :param expiry: + The time at which the shared access signature becomes invalid. + Required unless an id is given referencing a stored access policy + which contains this field. This field must be omitted if it has + been specified in an associated stored access policy. Azure will always + convert values to UTC. If a date is passed in without timezone info, it + is assumed to be UTC. + :type expiry: datetime or str + :param start: + The time at which the shared access signature becomes valid. If + omitted, start time for this call is assumed to be the time when the + storage service receives the request. Azure will always convert values + to UTC. If a date is passed in without timezone info, it is assumed to + be UTC. + :type start: datetime or str + :param str ip: + Specifies an IP address or a range of IP addresses from which to accept requests. + If the IP address from which the request originates does not match the IP address + or address range specified on the SAS token, the request is not authenticated. + For example, specifying sip=168.1.5.65 or sip=168.1.5.60-168.1.5.70 on the SAS + restricts the request to those IP addresses. + :param str protocol: + Specifies the protocol permitted for a request made. The default value + is https,http. See :class:`~..common.models.Protocol` for possible values. + :return: A Shared Access Signature (sas) token. + :rtype: str + ''' + _validate_not_none('self.account_name', self.account_name) + _validate_not_none('self.account_key', self.account_key) + + sas = BlobSharedAccessSignature(self.account_name, self.account_key) + return sas.generate_account(Services.BLOB, resource_types, permission, + expiry, start=start, ip=ip, protocol=protocol) + + def generate_container_shared_access_signature(self, container_name, + permission=None, expiry=None, + start=None, id=None, ip=None, protocol=None, + cache_control=None, content_disposition=None, + content_encoding=None, content_language=None, + content_type=None): + ''' + Generates a shared access signature for the container. + Use the returned signature with the sas_token parameter of any BlobService. + + :param str container_name: + Name of container. + :param ContainerPermissions permission: + The permissions associated with the shared access signature. The + user is restricted to operations allowed by the permissions. + Permissions must be ordered read, write, delete, list. + Required unless an id is given referencing a stored access policy + which contains this field. This field must be omitted if it has been + specified in an associated stored access policy. + :param expiry: + The time at which the shared access signature becomes invalid. + Required unless an id is given referencing a stored access policy + which contains this field. This field must be omitted if it has + been specified in an associated stored access policy. Azure will always + convert values to UTC. If a date is passed in without timezone info, it + is assumed to be UTC. + :type expiry: datetime or str + :param start: + The time at which the shared access signature becomes valid. If + omitted, start time for this call is assumed to be the time when the + storage service receives the request. Azure will always convert values + to UTC. If a date is passed in without timezone info, it is assumed to + be UTC. + :type start: datetime or str + :param str id: + A unique value up to 64 characters in length that correlates to a + stored access policy. To create a stored access policy, use + set_blob_service_properties. + :param str ip: + Specifies an IP address or a range of IP addresses from which to accept requests. + If the IP address from which the request originates does not match the IP address + or address range specified on the SAS token, the request is not authenticated. + For example, specifying sip=168.1.5.65 or sip=168.1.5.60-168.1.5.70 on the SAS + restricts the request to those IP addresses. + :param str protocol: + Specifies the protocol permitted for a request made. The default value + is https,http. See :class:`~..common.models.Protocol` for possible values. + :param str cache_control: + Response header value for Cache-Control when resource is accessed + using this shared access signature. + :param str content_disposition: + Response header value for Content-Disposition when resource is accessed + using this shared access signature. + :param str content_encoding: + Response header value for Content-Encoding when resource is accessed + using this shared access signature. + :param str content_language: + Response header value for Content-Language when resource is accessed + using this shared access signature. + :param str content_type: + Response header value for Content-Type when resource is accessed + using this shared access signature. + :return: A Shared Access Signature (sas) token. + :rtype: str + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('self.account_name', self.account_name) + _validate_not_none('self.account_key', self.account_key) + + sas = BlobSharedAccessSignature(self.account_name, self.account_key) + return sas.generate_container( + container_name, + permission, + expiry, + start=start, + id=id, + ip=ip, + protocol=protocol, + cache_control=cache_control, + content_disposition=content_disposition, + content_encoding=content_encoding, + content_language=content_language, + content_type=content_type, + ) + + def generate_blob_shared_access_signature( + self, container_name, blob_name, permission=None, + expiry=None, start=None, id=None, ip=None, protocol=None, + cache_control=None, content_disposition=None, + content_encoding=None, content_language=None, + content_type=None): + ''' + Generates a shared access signature for the blob. + Use the returned signature with the sas_token parameter of any BlobService. + + :param str container_name: + Name of container. + :param str blob_name: + Name of blob. + :param BlobPermissions permission: + The permissions associated with the shared access signature. The + user is restricted to operations allowed by the permissions. + Permissions must be ordered read, write, delete, list. + Required unless an id is given referencing a stored access policy + which contains this field. This field must be omitted if it has been + specified in an associated stored access policy. + :param expiry: + The time at which the shared access signature becomes invalid. + Required unless an id is given referencing a stored access policy + which contains this field. This field must be omitted if it has + been specified in an associated stored access policy. Azure will always + convert values to UTC. If a date is passed in without timezone info, it + is assumed to be UTC. + :type expiry: datetime or str + :param start: + The time at which the shared access signature becomes valid. If + omitted, start time for this call is assumed to be the time when the + storage service receives the request. Azure will always convert values + to UTC. If a date is passed in without timezone info, it is assumed to + be UTC. + :type start: datetime or str + :param str id: + A unique value up to 64 characters in length that correlates to a + stored access policy. To create a stored access policy, use :func:`~set_container_acl`. + :param str ip: + Specifies an IP address or a range of IP addresses from which to accept requests. + If the IP address from which the request originates does not match the IP address + or address range specified on the SAS token, the request is not authenticated. + For example, specifying sip=168.1.5.65 or sip=168.1.5.60-168.1.5.70 on the SAS + restricts the request to those IP addresses. + :param str protocol: + Specifies the protocol permitted for a request made. The default value + is https,http. See :class:`~..common.models.Protocol` for possible values. + :param str cache_control: + Response header value for Cache-Control when resource is accessed + using this shared access signature. + :param str content_disposition: + Response header value for Content-Disposition when resource is accessed + using this shared access signature. + :param str content_encoding: + Response header value for Content-Encoding when resource is accessed + using this shared access signature. + :param str content_language: + Response header value for Content-Language when resource is accessed + using this shared access signature. + :param str content_type: + Response header value for Content-Type when resource is accessed + using this shared access signature. + :return: A Shared Access Signature (sas) token. + :rtype: str + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('self.account_name', self.account_name) + _validate_not_none('self.account_key', self.account_key) + + sas = BlobSharedAccessSignature(self.account_name, self.account_key) + return sas.generate_blob( + container_name, + blob_name, + permission, + expiry, + start=start, + id=id, + ip=ip, + protocol=protocol, + cache_control=cache_control, + content_disposition=content_disposition, + content_encoding=content_encoding, + content_language=content_language, + content_type=content_type, + ) + + def list_containers(self, prefix=None, num_results=None, include_metadata=False, + marker=None, timeout=None): + ''' + Returns a generator to list the containers under the specified account. + The generator will lazily follow the continuation tokens returned by + the service and stop when all containers have been returned or num_results is reached. + + If num_results is specified and the account has more than that number of + containers, the generator will have a populated next_marker field once it + finishes. This marker can be used to create a new generator if more + results are desired. + + :param str prefix: + Filters the results to return only containers whose names + begin with the specified prefix. + :param int num_results: + Specifies the maximum number of containers to return. A single list + request may return up to 1000 contianers and potentially a continuation + token which should be followed to get additional resutls. + :param bool include_metadata: + Specifies that container metadata be returned in the response. + :param str marker: + An opaque continuation token. This value can be retrieved from the + next_marker field of a previous generator object if num_results was + specified and that generator has finished enumerating results. If + specified, this generator will begin returning results from the point + where the previous generator stopped. + :param int timeout: + The timeout parameter is expressed in seconds. + ''' + include = 'metadata' if include_metadata else None + operation_context = _OperationContext(location_lock=True) + kwargs = {'prefix': prefix, 'marker': marker, 'max_results': num_results, + 'include': include, 'timeout': timeout, '_context': operation_context} + resp = self._list_containers(**kwargs) + + return ListGenerator(resp, self._list_containers, (), kwargs) + + def _list_containers(self, prefix=None, marker=None, max_results=None, + include=None, timeout=None, _context=None): + ''' + Returns a list of the containers under the specified account. + + :param str prefix: + Filters the results to return only containers whose names + begin with the specified prefix. + :param str marker: + A string value that identifies the portion of the list + to be returned with the next list operation. The operation returns + a next_marker value within the response body if the list returned was + not complete. The marker value may then be used in a subsequent + call to request the next set of list items. The marker value is + opaque to the client. + :param int max_results: + Specifies the maximum number of containers to return. A single list + request may return up to 1000 contianers and potentially a continuation + token which should be followed to get additional resutls. + :param str include: + Include this parameter to specify that the container's + metadata be returned as part of the response body. set this + parameter to string 'metadata' to get container's metadata. + :param int timeout: + The timeout parameter is expressed in seconds. + ''' + request = HTTPRequest() + request.method = 'GET' + request.host_locations = self._get_host_locations(secondary=True) + request.path = _get_path() + request.query = { + 'comp': 'list', + 'prefix': _to_str(prefix), + 'marker': _to_str(marker), + 'maxresults': _int_to_str(max_results), + 'include': _to_str(include), + 'timeout': _int_to_str(timeout) + } + + return self._perform_request(request, _convert_xml_to_containers, operation_context=_context) + + def create_container(self, container_name, metadata=None, + public_access=None, fail_on_exist=False, timeout=None): + ''' + Creates a new container under the specified account. If the container + with the same name already exists, the operation fails if + fail_on_exist is True. + + :param str container_name: + Name of container to create. + :param metadata: + A dict with name_value pairs to associate with the + container as metadata. Example:{'Category':'test'} + :type metadata: dict(str, str) + :param ~azure.storage.blob.models.PublicAccess public_access: + Possible values include: container, blob. + :param bool fail_on_exist: + Specify whether to throw an exception when the container exists. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: True if container is created, False if container already exists. + :rtype: bool + ''' + _validate_not_none('container_name', container_name) + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name) + request.query = { + 'restype': 'container', + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-blob-public-access': _to_str(public_access) + } + _add_metadata_headers(metadata, request) + + if not fail_on_exist: + try: + self._perform_request(request, expected_errors=[_CONTAINER_ALREADY_EXISTS_ERROR_CODE]) + return True + except AzureHttpError as ex: + _dont_fail_on_exist(ex) + return False + else: + self._perform_request(request) + return True + + def get_container_properties(self, container_name, lease_id=None, timeout=None): + ''' + Returns all user-defined metadata and system properties for the specified + container. The data returned does not include the container's list of blobs. + + :param str container_name: + Name of existing container. + :param str lease_id: + If specified, get_container_properties only succeeds if the + container's lease is active and matches this ID. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: properties for the specified container within a container object. + :rtype: :class:`~azure.storage.blob.models.Container` + ''' + _validate_not_none('container_name', container_name) + request = HTTPRequest() + request.method = 'GET' + request.host_locations = self._get_host_locations(secondary=True) + request.path = _get_path(container_name) + request.query = { + 'restype': 'container', + 'timeout': _int_to_str(timeout), + } + request.headers = {'x-ms-lease-id': _to_str(lease_id)} + + return self._perform_request(request, _parse_container, [container_name]) + + def get_container_metadata(self, container_name, lease_id=None, timeout=None): + ''' + Returns all user-defined metadata for the specified container. + + :param str container_name: + Name of existing container. + :param str lease_id: + If specified, get_container_metadata only succeeds if the + container's lease is active and matches this ID. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: + A dictionary representing the container metadata name, value pairs. + :rtype: dict(str, str) + ''' + _validate_not_none('container_name', container_name) + request = HTTPRequest() + request.method = 'GET' + request.host_locations = self._get_host_locations(secondary=True) + request.path = _get_path(container_name) + request.query = { + 'restype': 'container', + 'comp': 'metadata', + 'timeout': _int_to_str(timeout), + } + request.headers = {'x-ms-lease-id': _to_str(lease_id)} + + return self._perform_request(request, _parse_metadata) + + def set_container_metadata(self, container_name, metadata=None, + lease_id=None, if_modified_since=None, timeout=None): + ''' + Sets one or more user-defined name-value pairs for the specified + container. Each call to this operation replaces all existing metadata + attached to the container. To remove all metadata from the container, + call this operation with no metadata dict. + + :param str container_name: + Name of existing container. + :param metadata: + A dict containing name-value pairs to associate with the container as + metadata. Example: {'category':'test'} + :type metadata: dict(str, str) + :param str lease_id: + If specified, set_container_metadata only succeeds if the + container's lease is active and matches this ID. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: ETag and last modified properties for the updated Container + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + _validate_not_none('container_name', container_name) + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name) + request.query = { + 'restype': 'container', + 'comp': 'metadata', + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'x-ms-lease-id': _to_str(lease_id), + } + _add_metadata_headers(metadata, request) + + return self._perform_request(request, _parse_base_properties) + + def get_container_acl(self, container_name, lease_id=None, timeout=None): + ''' + Gets the permissions for the specified container. + The permissions indicate whether container data may be accessed publicly. + + :param str container_name: + Name of existing container. + :param lease_id: + If specified, get_container_acl only succeeds if the + container's lease is active and matches this ID. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: A dictionary of access policies associated with the container. dict of str to + :class:`..common.models.AccessPolicy` and a public_access property + if public access is turned on + ''' + _validate_not_none('container_name', container_name) + request = HTTPRequest() + request.method = 'GET' + request.host_locations = self._get_host_locations(secondary=True) + request.path = _get_path(container_name) + request.query = { + 'restype': 'container', + 'comp': 'acl', + 'timeout': _int_to_str(timeout), + } + request.headers = {'x-ms-lease-id': _to_str(lease_id)} + + return self._perform_request(request, _convert_xml_to_signed_identifiers_and_access) + + def set_container_acl(self, container_name, signed_identifiers=None, + public_access=None, lease_id=None, + if_modified_since=None, if_unmodified_since=None, timeout=None): + ''' + Sets the permissions for the specified container or stored access + policies that may be used with Shared Access Signatures. The permissions + indicate whether blobs in a container may be accessed publicly. + + :param str container_name: + Name of existing container. + :param signed_identifiers: + A dictionary of access policies to associate with the container. The + dictionary may contain up to 5 elements. An empty dictionary + will clear the access policies set on the service. + :type signed_identifiers: dict(str, :class:`~..common.models.AccessPolicy`) + :param ~azure.storage.blob.models.PublicAccess public_access: + Possible values include: container, blob. + :param str lease_id: + If specified, set_container_acl only succeeds if the + container's lease is active and matches this ID. + :param datetime if_modified_since: + A datetime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified date/time. + :param datetime if_unmodified_since: + A datetime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: ETag and last modified properties for the updated Container + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + _validate_not_none('container_name', container_name) + _validate_access_policies(signed_identifiers) + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name) + request.query = { + 'restype': 'container', + 'comp': 'acl', + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-blob-public-access': _to_str(public_access), + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'x-ms-lease-id': _to_str(lease_id), + } + request.body = _get_request_body( + _convert_signed_identifiers_to_xml(signed_identifiers)) + + return self._perform_request(request, _parse_base_properties) + + def delete_container(self, container_name, fail_not_exist=False, + lease_id=None, if_modified_since=None, + if_unmodified_since=None, timeout=None): + ''' + Marks the specified container for deletion. The container and any blobs + contained within it are later deleted during garbage collection. + + :param str container_name: + Name of container to delete. + :param bool fail_not_exist: + Specify whether to throw an exception when the container doesn't + exist. + :param str lease_id: + If specified, delete_container only succeeds if the + container's lease is active and matches this ID. + Required if the container has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: True if container is deleted, False container doesn't exist. + :rtype: bool + ''' + _validate_not_none('container_name', container_name) + request = HTTPRequest() + request.method = 'DELETE' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name) + request.query = { + 'restype': 'container', + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-lease-id': _to_str(lease_id), + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + } + + if not fail_not_exist: + try: + self._perform_request(request, expected_errors=[_CONTAINER_NOT_FOUND_ERROR_CODE]) + return True + except AzureHttpError as ex: + _dont_fail_not_exist(ex) + return False + else: + self._perform_request(request) + return True + + def _lease_container_impl( + self, container_name, lease_action, lease_id, lease_duration, + lease_break_period, proposed_lease_id, if_modified_since, + if_unmodified_since, timeout): + ''' + Establishes and manages a lease on a container. + The Lease Container operation can be called in one of five modes + Acquire, to request a new lease + Renew, to renew an existing lease + Change, to change the ID of an existing lease + Release, to free the lease if it is no longer needed so that another + client may immediately acquire a lease against the container + Break, to end the lease but ensure that another client cannot acquire + a new lease until the current lease period has expired + + :param str container_name: + Name of existing container. + :param str lease_action: + Possible _LeaseActions values: acquire|renew|release|break|change + :param str lease_id: + Required if the container has an active lease. + :param int lease_duration: + Specifies the duration of the lease, in seconds, or negative one + (-1) for a lease that never expires. A non-infinite lease can be + between 15 and 60 seconds. A lease duration cannot be changed + using renew or change. For backwards compatibility, the default is + 60, and the value is only used on an acquire operation. + :param int lease_break_period: + For a break operation, this is the proposed duration of + seconds that the lease should continue before it is broken, between + 0 and 60 seconds. This break period is only used if it is shorter + than the time remaining on the lease. If longer, the time remaining + on the lease is used. A new lease will not be available before the + break period has expired, but the lease may be held for longer than + the break period. If this header does not appear with a break + operation, a fixed-duration lease breaks after the remaining lease + period elapses, and an infinite lease breaks immediately. + :param str proposed_lease_id: + Optional for Acquire, required for Change. Proposed lease ID, in a + GUID string format. The Blob service returns 400 (Invalid request) + if the proposed lease ID is not in the correct format. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: + Response headers returned from the service call. + :rtype: dict(str, str) + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('lease_action', lease_action) + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name) + request.query = { + 'restype': 'container', + 'comp': 'lease', + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-lease-id': _to_str(lease_id), + 'x-ms-lease-action': _to_str(lease_action), + 'x-ms-lease-duration': _to_str(lease_duration), + 'x-ms-lease-break-period': _to_str(lease_break_period), + 'x-ms-proposed-lease-id': _to_str(proposed_lease_id), + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + } + + return self._perform_request(request, _parse_lease) + + def acquire_container_lease( + self, container_name, lease_duration=-1, proposed_lease_id=None, + if_modified_since=None, if_unmodified_since=None, timeout=None): + ''' + Requests a new lease. If the container does not have an active lease, + the Blob service creates a lease on the container and returns a new + lease ID. + + :param str container_name: + Name of existing container. + :param int lease_duration: + Specifies the duration of the lease, in seconds, or negative one + (-1) for a lease that never expires. A non-infinite lease can be + between 15 and 60 seconds. A lease duration cannot be changed + using renew or change. Default is -1 (infinite lease). + :param str proposed_lease_id: + Proposed lease ID, in a GUID string format. The Blob service returns + 400 (Invalid request) if the proposed lease ID is not in the correct format. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: the lease ID of the newly created lease. + :return: str + ''' + _validate_not_none('lease_duration', lease_duration) + if lease_duration is not -1 and \ + (lease_duration < 15 or lease_duration > 60): + raise ValueError(_ERROR_INVALID_LEASE_DURATION) + + lease = self._lease_container_impl(container_name, + _LeaseActions.Acquire, + None, # lease_id + lease_duration, + None, # lease_break_period + proposed_lease_id, + if_modified_since, + if_unmodified_since, + timeout) + return lease['id'] + + def renew_container_lease( + self, container_name, lease_id, if_modified_since=None, + if_unmodified_since=None, timeout=None): + ''' + Renews the lease. The lease can be renewed if the lease ID specified + matches that associated with the container. Note that + the lease may be renewed even if it has expired as long as the container + has not been leased again since the expiration of that lease. When you + renew a lease, the lease duration clock resets. + + :param str container_name: + Name of existing container. + :param str lease_id: + Lease ID for active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: the lease ID of the renewed lease. + :return: str + ''' + _validate_not_none('lease_id', lease_id) + + lease = self._lease_container_impl(container_name, + _LeaseActions.Renew, + lease_id, + None, # lease_duration + None, # lease_break_period + None, # proposed_lease_id + if_modified_since, + if_unmodified_since, + timeout) + return lease['id'] + + def release_container_lease( + self, container_name, lease_id, if_modified_since=None, + if_unmodified_since=None, timeout=None): + ''' + Release the lease. The lease may be released if the lease_id specified matches + that associated with the container. Releasing the lease allows another client + to immediately acquire the lease for the container as soon as the release is complete. + + :param str container_name: + Name of existing container. + :param str lease_id: + Lease ID for active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param int timeout: + The timeout parameter is expressed in seconds. + ''' + _validate_not_none('lease_id', lease_id) + + self._lease_container_impl(container_name, + _LeaseActions.Release, + lease_id, + None, # lease_duration + None, # lease_break_period + None, # proposed_lease_id + if_modified_since, + if_unmodified_since, + timeout) + + def break_container_lease( + self, container_name, lease_break_period=None, + if_modified_since=None, if_unmodified_since=None, timeout=None): + ''' + Break the lease, if the container has an active lease. Once a lease is + broken, it cannot be renewed. Any authorized request can break the lease; + the request is not required to specify a matching lease ID. When a lease + is broken, the lease break period is allowed to elapse, during which time + no lease operation except break and release can be performed on the container. + When a lease is successfully broken, the response indicates the interval + in seconds until a new lease can be acquired. + + :param str container_name: + Name of existing container. + :param int lease_break_period: + This is the proposed duration of seconds that the lease + should continue before it is broken, between 0 and 60 seconds. This + break period is only used if it is shorter than the time remaining + on the lease. If longer, the time remaining on the lease is used. + A new lease will not be available before the break period has + expired, but the lease may be held for longer than the break + period. If this header does not appear with a break + operation, a fixed-duration lease breaks after the remaining lease + period elapses, and an infinite lease breaks immediately. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: Approximate time remaining in the lease period, in seconds. + :return: int + ''' + if (lease_break_period is not None) and (lease_break_period < 0 or lease_break_period > 60): + raise ValueError(_ERROR_INVALID_LEASE_BREAK_PERIOD) + + lease = self._lease_container_impl(container_name, + _LeaseActions.Break, + None, # lease_id + None, # lease_duration + lease_break_period, + None, # proposed_lease_id + if_modified_since, + if_unmodified_since, + timeout) + return lease['time'] + + def change_container_lease( + self, container_name, lease_id, proposed_lease_id, + if_modified_since=None, if_unmodified_since=None, timeout=None): + ''' + Change the lease ID of an active lease. A change must include the current + lease ID and a new lease ID. + + :param str container_name: + Name of existing container. + :param str lease_id: + Lease ID for active lease. + :param str proposed_lease_id: + Proposed lease ID, in a GUID string format. The Blob service returns 400 + (Invalid request) if the proposed lease ID is not in the correct format. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param int timeout: + The timeout parameter is expressed in seconds. + ''' + _validate_not_none('lease_id', lease_id) + + self._lease_container_impl(container_name, + _LeaseActions.Change, + lease_id, + None, # lease_duration + None, # lease_break_period + proposed_lease_id, + if_modified_since, + if_unmodified_since, + timeout) + + def list_blobs(self, container_name, prefix=None, num_results=None, include=None, + delimiter=None, marker=None, timeout=None): + ''' + Returns a generator to list the blobs under the specified container. + The generator will lazily follow the continuation tokens returned by + the service and stop when all blobs have been returned or num_results is reached. + + If num_results is specified and the account has more than that number of + blobs, the generator will have a populated next_marker field once it + finishes. This marker can be used to create a new generator if more + results are desired. + + :param str container_name: + Name of existing container. + :param str prefix: + Filters the results to return only blobs whose names + begin with the specified prefix. + :param int num_results: + Specifies the maximum number of blobs to return, + including all :class:`BlobPrefix` elements. If the request does not specify + num_results or specifies a value greater than 5,000, the server will + return up to 5,000 items. Setting num_results to a value less than + or equal to zero results in error response code 400 (Bad Request). + :param ~azure.storage.blob.models.Include include: + Specifies one or more additional datasets to include in the response. + :param str delimiter: + When the request includes this parameter, the operation + returns a :class:`~azure.storage.blob.models.BlobPrefix` element in the + result list that acts as a placeholder for all blobs whose names begin + with the same substring up to the appearance of the delimiter character. + The delimiter may be a single character or a string. + :param str marker: + An opaque continuation token. This value can be retrieved from the + next_marker field of a previous generator object if num_results was + specified and that generator has finished enumerating results. If + specified, this generator will begin returning results from the point + where the previous generator stopped. + :param int timeout: + The timeout parameter is expressed in seconds. + ''' + operation_context = _OperationContext(location_lock=True) + args = (container_name,) + kwargs = {'prefix': prefix, 'marker': marker, 'max_results': num_results, + 'include': include, 'delimiter': delimiter, 'timeout': timeout, + '_context': operation_context} + resp = self._list_blobs(*args, **kwargs) + + return ListGenerator(resp, self._list_blobs, args, kwargs) + + def _list_blobs(self, container_name, prefix=None, marker=None, + max_results=None, include=None, delimiter=None, timeout=None, + _context=None): + ''' + Returns the list of blobs under the specified container. + + :param str container_name: + Name of existing container. + :parm str prefix: + Filters the results to return only blobs whose names + begin with the specified prefix. + :param str marker: + A string value that identifies the portion of the list + to be returned with the next list operation. The operation returns + a next_marker value within the response body if the list returned was + not complete. The marker value may then be used in a subsequent + call to request the next set of list items. The marker value is + opaque to the client. + :param int max_results: + Specifies the maximum number of blobs to return, + including all :class:`~azure.storage.blob.models.BlobPrefix` elements. If the request does not specify + max_results or specifies a value greater than 5,000, the server will + return up to 5,000 items. Setting max_results to a value less than + or equal to zero results in error response code 400 (Bad Request). + :param str include: + Specifies one or more datasets to include in the + response. To specify more than one of these options on the URI, + you must separate each option with a comma. Valid values are: + snapshots: + Specifies that snapshots should be included in the + enumeration. Snapshots are listed from oldest to newest in + the response. + metadata: + Specifies that blob metadata be returned in the response. + uncommittedblobs: + Specifies that blobs for which blocks have been uploaded, + but which have not been committed using Put Block List + (REST API), be included in the response. + copy: + Version 2012-02-12 and newer. Specifies that metadata + related to any current or previous Copy Blob operation + should be included in the response. + deleted: + Version 2017-07-29 and newer. Specifies that soft deleted blobs + which are retained by the service should be included + in the response. + :param str delimiter: + When the request includes this parameter, the operation + returns a :class:`~azure.storage.blob.models.BlobPrefix` element in the response body that acts as a + placeholder for all blobs whose names begin with the same + substring up to the appearance of the delimiter character. The + delimiter may be a single character or a string. + :param int timeout: + The timeout parameter is expressed in seconds. + ''' + _validate_not_none('container_name', container_name) + request = HTTPRequest() + request.method = 'GET' + request.host_locations = self._get_host_locations(secondary=True) + request.path = _get_path(container_name) + request.query = { + 'restype': 'container', + 'comp': 'list', + 'prefix': _to_str(prefix), + 'delimiter': _to_str(delimiter), + 'marker': _to_str(marker), + 'maxresults': _int_to_str(max_results), + 'include': _to_str(include), + 'timeout': _int_to_str(timeout), + } + + return self._perform_request(request, _convert_xml_to_blob_list, operation_context=_context) + + def get_blob_account_information(self, container_name=None, blob_name=None, timeout=None): + """ + Gets information related to the storage account. + The information can also be retrieved if the user has a SAS to a container or blob. + + :param str container_name: + Name of existing container. + Optional, unless using a SAS token to a specific container or blob, in which case it's required. + :param str blob_name: + Name of existing blob. + Optional, unless using a SAS token to a specific blob, in which case it's required. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: The :class:`~azure.storage.blob.models.AccountInformation`. + """ + request = HTTPRequest() + request.method = 'HEAD' + request.host_locations = self._get_host_locations(secondary=True) + request.path = _get_path(container_name, blob_name) + request.query = { + 'restype': 'account', + 'comp': 'properties', + 'timeout': _int_to_str(timeout), + } + + return self._perform_request(request, _parse_account_information) + + def get_blob_service_stats(self, timeout=None): + ''' + Retrieves statistics related to replication for the Blob service. It is + only available when read-access geo-redundant replication is enabled for + the storage account. + + With geo-redundant replication, Azure Storage maintains your data durable + in two locations. In both locations, Azure Storage constantly maintains + multiple healthy replicas of your data. The location where you read, + create, update, or delete data is the primary storage account location. + The primary location exists in the region you choose at the time you + create an account via the Azure Management Azure classic portal, for + example, North Central US. The location to which your data is replicated + is the secondary location. The secondary location is automatically + determined based on the location of the primary; it is in a second data + center that resides in the same region as the primary location. Read-only + access is available from the secondary location, if read-access geo-redundant + replication is enabled for your storage account. + + :param int timeout: + The timeout parameter is expressed in seconds. + :return: The blob service stats. + :rtype: :class:`~..common.models.ServiceStats` + ''' + request = HTTPRequest() + request.method = 'GET' + request.host_locations = self._get_host_locations(primary=False, secondary=True) + request.path = _get_path() + request.query = { + 'restype': 'service', + 'comp': 'stats', + 'timeout': _int_to_str(timeout), + } + + return self._perform_request(request, _convert_xml_to_service_stats) + + def set_blob_service_properties( + self, logging=None, hour_metrics=None, minute_metrics=None, + cors=None, target_version=None, timeout=None, delete_retention_policy=None, static_website=None): + ''' + Sets the properties of a storage account's Blob service, including + Azure Storage Analytics. If an element (ex Logging) is left as None, the + existing settings on the service for that functionality are preserved. + + :param logging: + Groups the Azure Analytics Logging settings. + :type logging: + :class:`~..common.models.Logging` + :param hour_metrics: + The hour metrics settings provide a summary of request + statistics grouped by API in hourly aggregates for blobs. + :type hour_metrics: + :class:`~..common.models.Metrics` + :param minute_metrics: + The minute metrics settings provide request statistics + for each minute for blobs. + :type minute_metrics: + :class:`~..common.models.Metrics` + :param cors: + You can include up to five CorsRule elements in the + list. If an empty list is specified, all CORS rules will be deleted, + and CORS will be disabled for the service. + :type cors: list(:class:`~..common.models.CorsRule`) + :param str target_version: + Indicates the default version to use for requests if an incoming + request's version is not specified. + :param int timeout: + The timeout parameter is expressed in seconds. + :param delete_retention_policy: + The delete retention policy specifies whether to retain deleted blobs. + It also specifies the number of days and versions of blob to keep. + :type delete_retention_policy: + :class:`~..common.models.DeleteRetentionPolicy` + :param static_website: + Specifies whether the static website feature is enabled, + and if yes, indicates the index document and 404 error document to use. + :type static_website: + :class:`~..common.models.StaticWebsite` + ''' + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path() + request.query = { + 'restype': 'service', + 'comp': 'properties', + 'timeout': _int_to_str(timeout), + } + request.body = _get_request_body( + _convert_service_properties_to_xml(logging, hour_metrics, minute_metrics, + cors, target_version, delete_retention_policy, static_website)) + + self._perform_request(request) + + def get_blob_service_properties(self, timeout=None): + ''' + Gets the properties of a storage account's Blob service, including + Azure Storage Analytics. + + :param int timeout: + The timeout parameter is expressed in seconds. + :return: The blob :class:`~..common.models.ServiceProperties` with an attached + target_version property. + ''' + request = HTTPRequest() + request.method = 'GET' + request.host_locations = self._get_host_locations(secondary=True) + request.path = _get_path() + request.query = { + 'restype': 'service', + 'comp': 'properties', + 'timeout': _int_to_str(timeout), + } + + return self._perform_request(request, _convert_xml_to_service_properties) + + def get_blob_properties( + self, container_name, blob_name, snapshot=None, lease_id=None, + if_modified_since=None, if_unmodified_since=None, if_match=None, + if_none_match=None, timeout=None): + ''' + Returns all user-defined metadata, standard HTTP properties, and + system properties for the blob. It does not return the content of the blob. + Returns :class:`~azure.storage.blob.models.Blob` + with :class:`~azure.storage.blob.models.BlobProperties` and a metadata dict. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param str snapshot: + The snapshot parameter is an opaque DateTime value that, + when present, specifies the blob snapshot to retrieve. + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: a blob object including properties and metadata. + :rtype: :class:`~azure.storage.blob.models.Blob` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + request = HTTPRequest() + request.method = 'HEAD' + request.host_locations = self._get_host_locations(secondary=True) + request.path = _get_path(container_name, blob_name) + request.query = { + 'snapshot': _to_str(snapshot), + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-lease-id': _to_str(lease_id), + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'If-Match': _to_str(if_match), + 'If-None-Match': _to_str(if_none_match), + } + + return self._perform_request(request, _parse_blob, [blob_name, snapshot]) + + def set_blob_properties( + self, container_name, blob_name, content_settings=None, lease_id=None, + if_modified_since=None, if_unmodified_since=None, if_match=None, + if_none_match=None, timeout=None): + ''' + Sets system properties on the blob. If one property is set for the + content_settings, all properties will be overriden. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param ~azure.storage.blob.models.ContentSettings content_settings: + ContentSettings object used to set blob properties. + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: ETag and last modified properties for the updated Blob + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.query = { + 'comp': 'properties', + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'If-Match': _to_str(if_match), + 'If-None-Match': _to_str(if_none_match), + 'x-ms-lease-id': _to_str(lease_id) + } + if content_settings is not None: + request.headers.update(content_settings._to_headers()) + + return self._perform_request(request, _parse_base_properties) + + def exists(self, container_name, blob_name=None, snapshot=None, timeout=None): + ''' + Returns a boolean indicating whether the container exists (if blob_name + is None), or otherwise a boolean indicating whether the blob exists. + + :param str container_name: + Name of a container. + :param str blob_name: + Name of a blob. If None, the container will be checked for existence. + :param str snapshot: + The snapshot parameter is an opaque DateTime value that, + when present, specifies the snapshot. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: A boolean indicating whether the resource exists. + :rtype: bool + ''' + _validate_not_none('container_name', container_name) + try: + # make head request to see if container/blob/snapshot exists + request = HTTPRequest() + request.method = 'GET' if blob_name is None else 'HEAD' + request.host_locations = self._get_host_locations(secondary=True) + request.path = _get_path(container_name, blob_name) + request.query = { + 'snapshot': _to_str(snapshot), + 'timeout': _int_to_str(timeout), + 'restype': 'container' if blob_name is None else None, + } + + expected_errors = [_CONTAINER_NOT_FOUND_ERROR_CODE] if blob_name is None \ + else [_CONTAINER_NOT_FOUND_ERROR_CODE, _BLOB_NOT_FOUND_ERROR_CODE] + self._perform_request(request, expected_errors=expected_errors) + + return True + except AzureHttpError as ex: + _dont_fail_not_exist(ex) + return False + + def _get_blob( + self, container_name, blob_name, snapshot=None, start_range=None, + end_range=None, validate_content=False, lease_id=None, if_modified_since=None, + if_unmodified_since=None, if_match=None, if_none_match=None, timeout=None, + _context=None): + ''' + Downloads a blob's content, metadata, and properties. You can also + call this API to read a snapshot. You can specify a range if you don't + need to download the blob in its entirety. If no range is specified, + the full blob will be downloaded. + + See get_blob_to_* for high level functions that handle the download + of large blobs with automatic chunking and progress notifications. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param str snapshot: + The snapshot parameter is an opaque DateTime value that, + when present, specifies the blob snapshot to retrieve. + :param int start_range: + Start of byte range to use for downloading a section of the blob. + If no end_range is given, all bytes after the start_range will be downloaded. + The start_range and end_range params are inclusive. + Ex: start_range=0, end_range=511 will download first 512 bytes of blob. + :param int end_range: + End of byte range to use for downloading a section of the blob. + If end_range is given, start_range must be provided. + The start_range and end_range params are inclusive. + Ex: start_range=0, end_range=511 will download first 512 bytes of blob. + :param bool validate_content: + When this is set to True and specified together with the Range header, + the service returns the MD5 hash for the range, as long as the range + is less than or equal to 4 MB in size. + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: A Blob with content, properties, and metadata. + :rtype: :class:`~azure.storage.blob.models.Blob` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_decryption_required(self.require_encryption, + self.key_encryption_key, + self.key_resolver_function) + + start_offset, end_offset = 0, 0 + if self.key_encryption_key is not None or self.key_resolver_function is not None: + if start_range is not None: + # Align the start of the range along a 16 byte block + start_offset = start_range % 16 + start_range -= start_offset + + # Include an extra 16 bytes for the IV if necessary + # Because of the previous offsetting, start_range will always + # be a multiple of 16. + if start_range > 0: + start_offset += 16 + start_range -= 16 + + if end_range is not None: + # Align the end of the range along a 16 byte block + end_offset = 15 - (end_range % 16) + end_range += end_offset + + request = HTTPRequest() + request.method = 'GET' + request.host_locations = self._get_host_locations(secondary=True) + request.path = _get_path(container_name, blob_name) + request.query = { + 'snapshot': _to_str(snapshot), + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-lease-id': _to_str(lease_id), + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'If-Match': _to_str(if_match), + 'If-None-Match': _to_str(if_none_match), + } + _validate_and_format_range_headers( + request, + start_range, + end_range, + start_range_required=False, + end_range_required=False, + check_content_md5=validate_content) + + return self._perform_request(request, _parse_blob, + [blob_name, snapshot, validate_content, self.require_encryption, + self.key_encryption_key, self.key_resolver_function, + start_offset, end_offset], + operation_context=_context) + + def get_blob_to_path( + self, container_name, blob_name, file_path, open_mode='wb', + snapshot=None, start_range=None, end_range=None, + validate_content=False, progress_callback=None, + max_connections=2, lease_id=None, if_modified_since=None, + if_unmodified_since=None, if_match=None, if_none_match=None, + timeout=None): + ''' + Downloads a blob to a file path, with automatic chunking and progress + notifications. Returns an instance of :class:`~azure.storage.blob.models.Blob` with + properties and metadata. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param str file_path: + Path of file to write out to. + :param str open_mode: + Mode to use when opening the file. Note that specifying append only + open_mode prevents parallel download. So, max_connections must be set + to 1 if this open_mode is used. + :param str snapshot: + The snapshot parameter is an opaque DateTime value that, + when present, specifies the blob snapshot to retrieve. + :param int start_range: + Start of byte range to use for downloading a section of the blob. + If no end_range is given, all bytes after the start_range will be downloaded. + The start_range and end_range params are inclusive. + Ex: start_range=0, end_range=511 will download first 512 bytes of blob. + :param int end_range: + End of byte range to use for downloading a section of the blob. + If end_range is given, start_range must be provided. + The start_range and end_range params are inclusive. + Ex: start_range=0, end_range=511 will download first 512 bytes of blob. + :param bool validate_content: + If set to true, validates an MD5 hash for each retrieved portion of + the blob. This is primarily valuable for detecting bitflips on the wire + if using http instead of https as https (the default) will already + validate. Note that the service will only return transactional MD5s + for chunks 4MB or less so the first get request will be of size + self.MAX_CHUNK_GET_SIZE instead of self.MAX_SINGLE_GET_SIZE. If + self.MAX_CHUNK_GET_SIZE was set to greater than 4MB an error will be + thrown. As computing the MD5 takes processing time and more requests + will need to be done due to the reduced chunk size there may be some + increase in latency. + :param progress_callback: + Callback for progress with signature function(current, total) + where current is the number of bytes transfered so far, and total is + the size of the blob if known. + :type progress_callback: func(current, total) + :param int max_connections: + If set to 2 or greater, an initial get will be done for the first + self.MAX_SINGLE_GET_SIZE bytes of the blob. If this is the entire blob, + the method returns at this point. If it is not, it will download the + remaining data parallel using the number of threads equal to + max_connections. Each chunk will be of size self.MAX_CHUNK_GET_SIZE. + If set to 1, a single large get request will be done. This is not + generally recommended but available if very few threads should be + used, network requests are very expensive, or a non-seekable stream + prevents parallel download. This may also be useful if many blobs are + expected to be empty as an extra request is required for empty blobs + if max_connections is greater than 1. + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. This method may make + multiple calls to the Azure service and the timeout will apply to + each call individually. + :return: A Blob with properties and metadata. If max_connections is greater + than 1, the content_md5 (if set on the blob) will not be returned. If you + require this value, either use get_blob_properties or set max_connections + to 1. + :rtype: :class:`~azure.storage.blob.models.Blob` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('file_path', file_path) + _validate_not_none('open_mode', open_mode) + + if max_connections > 1 and 'a' in open_mode: + raise ValueError(_ERROR_PARALLEL_NOT_SEEKABLE) + + with open(file_path, open_mode) as stream: + blob = self.get_blob_to_stream( + container_name, + blob_name, + stream, + snapshot, + start_range, + end_range, + validate_content, + progress_callback, + max_connections, + lease_id, + if_modified_since, + if_unmodified_since, + if_match, + if_none_match, + timeout) + + return blob + + def get_blob_to_stream( + self, container_name, blob_name, stream, snapshot=None, + start_range=None, end_range=None, validate_content=False, + progress_callback=None, max_connections=2, lease_id=None, + if_modified_since=None, if_unmodified_since=None, if_match=None, + if_none_match=None, timeout=None): + + ''' + Downloads a blob to a stream, with automatic chunking and progress + notifications. Returns an instance of :class:`~azure.storage.blob.models.Blob` with + properties and metadata. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param io.IOBase stream: + Opened stream to write to. + :param str snapshot: + The snapshot parameter is an opaque DateTime value that, + when present, specifies the blob snapshot to retrieve. + :param int start_range: + Start of byte range to use for downloading a section of the blob. + If no end_range is given, all bytes after the start_range will be downloaded. + The start_range and end_range params are inclusive. + Ex: start_range=0, end_range=511 will download first 512 bytes of blob. + :param int end_range: + End of byte range to use for downloading a section of the blob. + If end_range is given, start_range must be provided. + The start_range and end_range params are inclusive. + Ex: start_range=0, end_range=511 will download first 512 bytes of blob. + :param bool validate_content: + If set to true, validates an MD5 hash for each retrieved portion of + the blob. This is primarily valuable for detecting bitflips on the wire + if using http instead of https as https (the default) will already + validate. Note that the service will only return transactional MD5s + for chunks 4MB or less so the first get request will be of size + self.MAX_CHUNK_GET_SIZE instead of self.MAX_SINGLE_GET_SIZE. If + self.MAX_CHUNK_GET_SIZE was set to greater than 4MB an error will be + thrown. As computing the MD5 takes processing time and more requests + will need to be done due to the reduced chunk size there may be some + increase in latency. + :param progress_callback: + Callback for progress with signature function(current, total) + where current is the number of bytes transfered so far, and total is + the size of the blob if known. + :type progress_callback: func(current, total) + :param int max_connections: + If set to 2 or greater, an initial get will be done for the first + self.MAX_SINGLE_GET_SIZE bytes of the blob. If this is the entire blob, + the method returns at this point. If it is not, it will download the + remaining data parallel using the number of threads equal to + max_connections. Each chunk will be of size self.MAX_CHUNK_GET_SIZE. + If set to 1, a single large get request will be done. This is not + generally recommended but available if very few threads should be + used, network requests are very expensive, or a non-seekable stream + prevents parallel download. This may also be useful if many blobs are + expected to be empty as an extra request is required for empty blobs + if max_connections is greater than 1. + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. This method may make + multiple calls to the Azure service and the timeout will apply to + each call individually. + :return: A Blob with properties and metadata. If max_connections is greater + than 1, the content_md5 (if set on the blob) will not be returned. If you + require this value, either use get_blob_properties or set max_connections + to 1. + :rtype: :class:`~azure.storage.blob.models.Blob` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('stream', stream) + + if end_range is not None: + _validate_not_none("start_range", start_range) + + # the stream must be seekable if parallel download is required + if max_connections > 1: + if sys.version_info >= (3,) and not stream.seekable(): + raise ValueError(_ERROR_PARALLEL_NOT_SEEKABLE) + else: + try: + stream.seek(stream.tell()) + except (NotImplementedError, AttributeError): + raise ValueError(_ERROR_PARALLEL_NOT_SEEKABLE) + + # The service only provides transactional MD5s for chunks under 4MB. + # If validate_content is on, get only self.MAX_CHUNK_GET_SIZE for the first + # chunk so a transactional MD5 can be retrieved. + first_get_size = self.MAX_SINGLE_GET_SIZE if not validate_content else self.MAX_CHUNK_GET_SIZE + + initial_request_start = start_range if start_range is not None else 0 + + if end_range is not None and end_range - start_range < first_get_size: + initial_request_end = end_range + else: + initial_request_end = initial_request_start + first_get_size - 1 + + # Send a context object to make sure we always retry to the initial location + operation_context = _OperationContext(location_lock=True) + try: + blob = self._get_blob(container_name, + blob_name, + snapshot, + start_range=initial_request_start, + end_range=initial_request_end, + validate_content=validate_content, + lease_id=lease_id, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + if_match=if_match, + if_none_match=if_none_match, + timeout=timeout, + _context=operation_context) + + # Parse the total blob size and adjust the download size if ranges + # were specified + blob_size = _parse_length_from_content_range(blob.properties.content_range) + if end_range is not None: + # Use the end_range unless it is over the end of the blob + download_size = min(blob_size, end_range - start_range + 1) + elif start_range is not None: + download_size = blob_size - start_range + else: + download_size = blob_size + except AzureHttpError as ex: + if start_range is None and ex.status_code == 416: + # Get range will fail on an empty blob. If the user did not + # request a range, do a regular get request in order to get + # any properties. + blob = self._get_blob(container_name, + blob_name, + snapshot, + validate_content=validate_content, + lease_id=lease_id, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + if_match=if_match, + if_none_match=if_none_match, + timeout=timeout, + _context=operation_context) + + # Set the download size to empty + download_size = 0 + else: + raise ex + + # Mark the first progress chunk. If the blob is small or this is a single + # shot download, this is the only call + if progress_callback: + progress_callback(blob.properties.content_length, download_size) + + # Write the content to the user stream + # Clear blob content since output has been written to user stream + if blob.content is not None: + stream.write(blob.content) + blob.content = None + + # If the blob is small, the download is complete at this point. + # If blob size is large, download the rest of the blob in chunks. + if blob.properties.content_length != download_size: + # Lock on the etag. This can be overriden by the user by specifying '*' + if_match = if_match if if_match is not None else blob.properties.etag + + end_blob = blob_size + if end_range is not None: + # Use the end_range unless it is over the end of the blob + end_blob = min(blob_size, end_range + 1) + + _download_blob_chunks( + self, + container_name, + blob_name, + snapshot, + download_size, + self.MAX_CHUNK_GET_SIZE, + first_get_size, + initial_request_end + 1, # start where the first download ended + end_blob, + stream, + max_connections, + progress_callback, + validate_content, + lease_id, + if_modified_since, + if_unmodified_since, + if_match, + if_none_match, + timeout, + operation_context + ) + + # Set the content length to the download size instead of the size of + # the last range + blob.properties.content_length = download_size + + # Overwrite the content range to the user requested range + blob.properties.content_range = 'bytes {0}-{1}/{2}'.format(start_range, end_range, blob_size) + + # Overwrite the content MD5 as it is the MD5 for the last range instead + # of the stored MD5 + # TODO: Set to the stored MD5 when the service returns this + blob.properties.content_md5 = None + + return blob + + def get_blob_to_bytes( + self, container_name, blob_name, snapshot=None, + start_range=None, end_range=None, validate_content=False, + progress_callback=None, max_connections=2, lease_id=None, + if_modified_since=None, if_unmodified_since=None, if_match=None, + if_none_match=None, timeout=None): + ''' + Downloads a blob as an array of bytes, with automatic chunking and + progress notifications. Returns an instance of :class:`~azure.storage.blob.models.Blob` with + properties, metadata, and content. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param str snapshot: + The snapshot parameter is an opaque DateTime value that, + when present, specifies the blob snapshot to retrieve. + :param int start_range: + Start of byte range to use for downloading a section of the blob. + If no end_range is given, all bytes after the start_range will be downloaded. + The start_range and end_range params are inclusive. + Ex: start_range=0, end_range=511 will download first 512 bytes of blob. + :param int end_range: + End of byte range to use for downloading a section of the blob. + If end_range is given, start_range must be provided. + The start_range and end_range params are inclusive. + Ex: start_range=0, end_range=511 will download first 512 bytes of blob. + :param bool validate_content: + If set to true, validates an MD5 hash for each retrieved portion of + the blob. This is primarily valuable for detecting bitflips on the wire + if using http instead of https as https (the default) will already + validate. Note that the service will only return transactional MD5s + for chunks 4MB or less so the first get request will be of size + self.MAX_CHUNK_GET_SIZE instead of self.MAX_SINGLE_GET_SIZE. If + self.MAX_CHUNK_GET_SIZE was set to greater than 4MB an error will be + thrown. As computing the MD5 takes processing time and more requests + will need to be done due to the reduced chunk size there may be some + increase in latency. + :param progress_callback: + Callback for progress with signature function(current, total) + where current is the number of bytes transfered so far, and total is + the size of the blob if known. + :type progress_callback: func(current, total) + :param int max_connections: + If set to 2 or greater, an initial get will be done for the first + self.MAX_SINGLE_GET_SIZE bytes of the blob. If this is the entire blob, + the method returns at this point. If it is not, it will download the + remaining data parallel using the number of threads equal to + max_connections. Each chunk will be of size self.MAX_CHUNK_GET_SIZE. + If set to 1, a single large get request will be done. This is not + generally recommended but available if very few threads should be + used, network requests are very expensive, or a non-seekable stream + prevents parallel download. This may also be useful if many blobs are + expected to be empty as an extra request is required for empty blobs + if max_connections is greater than 1. + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. This method may make + multiple calls to the Azure service and the timeout will apply to + each call individually. + :return: A Blob with properties and metadata. If max_connections is greater + than 1, the content_md5 (if set on the blob) will not be returned. If you + require this value, either use get_blob_properties or set max_connections + to 1. + :rtype: :class:`~azure.storage.blob.models.Blob` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + + stream = BytesIO() + blob = self.get_blob_to_stream( + container_name, + blob_name, + stream, + snapshot, + start_range, + end_range, + validate_content, + progress_callback, + max_connections, + lease_id, + if_modified_since, + if_unmodified_since, + if_match, + if_none_match, + timeout) + + blob.content = stream.getvalue() + return blob + + def get_blob_to_text( + self, container_name, blob_name, encoding='utf-8', snapshot=None, + start_range=None, end_range=None, validate_content=False, + progress_callback=None, max_connections=2, lease_id=None, + if_modified_since=None, if_unmodified_since=None, if_match=None, + if_none_match=None, timeout=None): + ''' + Downloads a blob as unicode text, with automatic chunking and progress + notifications. Returns an instance of :class:`~azure.storage.blob.models.Blob` with + properties, metadata, and content. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param str encoding: + Python encoding to use when decoding the blob data. + :param str snapshot: + The snapshot parameter is an opaque DateTime value that, + when present, specifies the blob snapshot to retrieve. + :param int start_range: + Start of byte range to use for downloading a section of the blob. + If no end_range is given, all bytes after the start_range will be downloaded. + The start_range and end_range params are inclusive. + Ex: start_range=0, end_range=511 will download first 512 bytes of blob. + :param int end_range: + End of byte range to use for downloading a section of the blob. + If end_range is given, start_range must be provided. + The start_range and end_range params are inclusive. + Ex: start_range=0, end_range=511 will download first 512 bytes of blob. + :param bool validate_content: + If set to true, validates an MD5 hash for each retrieved portion of + the blob. This is primarily valuable for detecting bitflips on the wire + if using http instead of https as https (the default) will already + validate. Note that the service will only return transactional MD5s + for chunks 4MB or less so the first get request will be of size + self.MAX_CHUNK_GET_SIZE instead of self.MAX_SINGLE_GET_SIZE. If + self.MAX_CHUNK_GET_SIZE was set to greater than 4MB an error will be + thrown. As computing the MD5 takes processing time and more requests + will need to be done due to the reduced chunk size there may be some + increase in latency. + :param progress_callback: + Callback for progress with signature function(current, total) + where current is the number of bytes transfered so far, and total is + the size of the blob if known. + :type progress_callback: func(current, total) + :param int max_connections: + If set to 2 or greater, an initial get will be done for the first + self.MAX_SINGLE_GET_SIZE bytes of the blob. If this is the entire blob, + the method returns at this point. If it is not, it will download the + remaining data parallel using the number of threads equal to + max_connections. Each chunk will be of size self.MAX_CHUNK_GET_SIZE. + If set to 1, a single large get request will be done. This is not + generally recommended but available if very few threads should be + used, network requests are very expensive, or a non-seekable stream + prevents parallel download. This may also be useful if many blobs are + expected to be empty as an extra request is required for empty blobs + if max_connections is greater than 1. + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. This method may make + multiple calls to the Azure service and the timeout will apply to + each call individually. + :return: A Blob with properties and metadata. If max_connections is greater + than 1, the content_md5 (if set on the blob) will not be returned. If you + require this value, either use get_blob_properties or set max_connections + to 1. + :rtype: :class:`~azure.storage.blob.models.Blob` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('encoding', encoding) + + blob = self.get_blob_to_bytes(container_name, + blob_name, + snapshot, + start_range, + end_range, + validate_content, + progress_callback, + max_connections, + lease_id, + if_modified_since, + if_unmodified_since, + if_match, + if_none_match, + timeout) + blob.content = blob.content.decode(encoding) + return blob + + def get_blob_metadata( + self, container_name, blob_name, snapshot=None, lease_id=None, + if_modified_since=None, if_unmodified_since=None, if_match=None, + if_none_match=None, timeout=None): + ''' + Returns all user-defined metadata for the specified blob or snapshot. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param str snapshot: + The snapshot parameter is an opaque value that, + when present, specifies the blob snapshot to retrieve. + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: + A dictionary representing the blob metadata name, value pairs. + :rtype: dict(str, str) + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + request = HTTPRequest() + request.method = 'GET' + request.host_locations = self._get_host_locations(secondary=True) + request.path = _get_path(container_name, blob_name) + request.query = { + 'snapshot': _to_str(snapshot), + 'comp': 'metadata', + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-lease-id': _to_str(lease_id), + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'If-Match': _to_str(if_match), + 'If-None-Match': _to_str(if_none_match), + } + + return self._perform_request(request, _parse_metadata) + + def set_blob_metadata(self, container_name, blob_name, + metadata=None, lease_id=None, + if_modified_since=None, if_unmodified_since=None, + if_match=None, if_none_match=None, timeout=None): + ''' + Sets user-defined metadata for the specified blob as one or more + name-value pairs. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param metadata: + Dict containing name and value pairs. Each call to this operation + replaces all existing metadata attached to the blob. To remove all + metadata from the blob, call this operation with no metadata headers. + :type metadata: dict(str, str) + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: ETag and last modified properties for the updated Blob + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.query = { + 'comp': 'metadata', + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'If-Match': _to_str(if_match), + 'If-None-Match': _to_str(if_none_match), + 'x-ms-lease-id': _to_str(lease_id), + } + _add_metadata_headers(metadata, request) + + return self._perform_request(request, _parse_base_properties) + + def _lease_blob_impl(self, container_name, blob_name, + lease_action, lease_id, + lease_duration, lease_break_period, + proposed_lease_id, if_modified_since, + if_unmodified_since, if_match, if_none_match, timeout=None): + ''' + Establishes and manages a lease on a blob for write and delete operations. + The Lease Blob operation can be called in one of five modes: + Acquire, to request a new lease. + Renew, to renew an existing lease. + Change, to change the ID of an existing lease. + Release, to free the lease if it is no longer needed so that another + client may immediately acquire a lease against the blob. + Break, to end the lease but ensure that another client cannot acquire + a new lease until the current lease period has expired. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param str lease_action: + Possible _LeaseActions acquire|renew|release|break|change + :param str lease_id: + Required if the blob has an active lease. + :param int lease_duration: + Specifies the duration of the lease, in seconds, or negative one + (-1) for a lease that never expires. A non-infinite lease can be + between 15 and 60 seconds. A lease duration cannot be changed + using renew or change. + :param int lease_break_period: + For a break operation, this is the proposed duration of + seconds that the lease should continue before it is broken, between + 0 and 60 seconds. This break period is only used if it is shorter + than the time remaining on the lease. If longer, the time remaining + on the lease is used. A new lease will not be available before the + break period has expired, but the lease may be held for longer than + the break period. If this header does not appear with a break + operation, a fixed-duration lease breaks after the remaining lease + period elapses, and an infinite lease breaks immediately. + :param str proposed_lease_id: + Optional for acquire, required for change. Proposed lease ID, in a + GUID string format. The Blob service returns 400 (Invalid request) + if the proposed lease ID is not in the correct format. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: + Response headers returned from the service call. + :rtype: dict(str, str) + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('lease_action', lease_action) + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.query = { + 'comp': 'lease', + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-lease-id': _to_str(lease_id), + 'x-ms-lease-action': _to_str(lease_action), + 'x-ms-lease-duration': _to_str(lease_duration), + 'x-ms-lease-break-period': _to_str(lease_break_period), + 'x-ms-proposed-lease-id': _to_str(proposed_lease_id), + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'If-Match': _to_str(if_match), + 'If-None-Match': _to_str(if_none_match), + } + + return self._perform_request(request, _parse_lease) + + def acquire_blob_lease(self, container_name, blob_name, + lease_duration=-1, + proposed_lease_id=None, + if_modified_since=None, + if_unmodified_since=None, + if_match=None, + if_none_match=None, timeout=None): + ''' + Requests a new lease. If the blob does not have an active lease, the Blob + service creates a lease on the blob and returns a new lease ID. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param int lease_duration: + Specifies the duration of the lease, in seconds, or negative one + (-1) for a lease that never expires. A non-infinite lease can be + between 15 and 60 seconds. A lease duration cannot be changed + using renew or change. Default is -1 (infinite lease). + :param str proposed_lease_id: + Proposed lease ID, in a GUID string format. The Blob service + returns 400 (Invalid request) if the proposed lease ID is not + in the correct format. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: the lease ID of the newly created lease. + :return: str + ''' + _validate_not_none('lease_duration', lease_duration) + + if lease_duration is not -1 and \ + (lease_duration < 15 or lease_duration > 60): + raise ValueError(_ERROR_INVALID_LEASE_DURATION) + lease = self._lease_blob_impl(container_name, + blob_name, + _LeaseActions.Acquire, + None, # lease_id + lease_duration, + None, # lease_break_period + proposed_lease_id, + if_modified_since, + if_unmodified_since, + if_match, + if_none_match, + timeout) + return lease['id'] + + def renew_blob_lease(self, container_name, blob_name, + lease_id, if_modified_since=None, + if_unmodified_since=None, if_match=None, + if_none_match=None, timeout=None): + ''' + Renews the lease. The lease can be renewed if the lease ID specified on + the request matches that associated with the blob. Note that the lease may + be renewed even if it has expired as long as the blob has not been modified + or leased again since the expiration of that lease. When you renew a lease, + the lease duration clock resets. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param str lease_id: + Lease ID for active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: the lease ID of the renewed lease. + :return: str + ''' + _validate_not_none('lease_id', lease_id) + + lease = self._lease_blob_impl(container_name, + blob_name, + _LeaseActions.Renew, + lease_id, + None, # lease_duration + None, # lease_break_period + None, # proposed_lease_id + if_modified_since, + if_unmodified_since, + if_match, + if_none_match, + timeout) + return lease['id'] + + def release_blob_lease(self, container_name, blob_name, + lease_id, if_modified_since=None, + if_unmodified_since=None, if_match=None, + if_none_match=None, timeout=None): + ''' + Releases the lease. The lease may be released if the lease ID specified on the + request matches that associated with the blob. Releasing the lease allows another + client to immediately acquire the lease for the blob as soon as the release is complete. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param str lease_id: + Lease ID for active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + ''' + _validate_not_none('lease_id', lease_id) + + self._lease_blob_impl(container_name, + blob_name, + _LeaseActions.Release, + lease_id, + None, # lease_duration + None, # lease_break_period + None, # proposed_lease_id + if_modified_since, + if_unmodified_since, + if_match, + if_none_match, + timeout) + + def break_blob_lease(self, container_name, blob_name, + lease_break_period=None, + if_modified_since=None, + if_unmodified_since=None, + if_match=None, + if_none_match=None, timeout=None): + ''' + Breaks the lease, if the blob has an active lease. Once a lease is broken, + it cannot be renewed. Any authorized request can break the lease; the request + is not required to specify a matching lease ID. When a lease is broken, + the lease break period is allowed to elapse, during which time no lease operation + except break and release can be performed on the blob. When a lease is successfully + broken, the response indicates the interval in seconds until a new lease can be acquired. + + A lease that has been broken can also be released, in which case another client may + immediately acquire the lease on the blob. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param int lease_break_period: + For a break operation, this is the proposed duration of + seconds that the lease should continue before it is broken, between + 0 and 60 seconds. This break period is only used if it is shorter + than the time remaining on the lease. If longer, the time remaining + on the lease is used. A new lease will not be available before the + break period has expired, but the lease may be held for longer than + the break period. If this header does not appear with a break + operation, a fixed-duration lease breaks after the remaining lease + period elapses, and an infinite lease breaks immediately. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: Approximate time remaining in the lease period, in seconds. + :return: int + ''' + if (lease_break_period is not None) and (lease_break_period < 0 or lease_break_period > 60): + raise ValueError(_ERROR_INVALID_LEASE_BREAK_PERIOD) + + lease = self._lease_blob_impl(container_name, + blob_name, + _LeaseActions.Break, + None, # lease_id + None, # lease_duration + lease_break_period, + None, # proposed_lease_id + if_modified_since, + if_unmodified_since, + if_match, + if_none_match, + timeout) + return lease['time'] + + def change_blob_lease(self, container_name, blob_name, + lease_id, + proposed_lease_id, + if_modified_since=None, + if_unmodified_since=None, + if_match=None, + if_none_match=None, timeout=None): + ''' + Changes the lease ID of an active lease. A change must include the current + lease ID and a new lease ID. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param str lease_id: + Required if the blob has an active lease. + :param str proposed_lease_id: + Proposed lease ID, in a GUID string format. The Blob service returns + 400 (Invalid request) if the proposed lease ID is not in the correct format. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + ''' + self._lease_blob_impl(container_name, + blob_name, + _LeaseActions.Change, + lease_id, + None, # lease_duration + None, # lease_break_period + proposed_lease_id, + if_modified_since, + if_unmodified_since, + if_match, + if_none_match, + timeout) + + def snapshot_blob(self, container_name, blob_name, + metadata=None, if_modified_since=None, + if_unmodified_since=None, if_match=None, + if_none_match=None, lease_id=None, timeout=None): + ''' + Creates a read-only snapshot of a blob. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param metadata: + Specifies a user-defined name-value pair associated with the blob. + If no name-value pairs are specified, the operation will copy the + base blob metadata to the snapshot. If one or more name-value pairs + are specified, the snapshot is created with the specified metadata, + and metadata is not copied from the base blob. + :type metadata: dict(str, str) + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param str lease_id: + Required if the blob has an active lease. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: snapshot properties + :rtype: :class:`~azure.storage.blob.models.Blob` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.query = { + 'comp': 'snapshot', + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'If-Match': _to_str(if_match), + 'If-None-Match': _to_str(if_none_match), + 'x-ms-lease-id': _to_str(lease_id) + } + _add_metadata_headers(metadata, request) + + return self._perform_request(request, _parse_snapshot_blob, [blob_name]) + + def copy_blob(self, container_name, blob_name, copy_source, + metadata=None, + source_if_modified_since=None, + source_if_unmodified_since=None, + source_if_match=None, source_if_none_match=None, + destination_if_modified_since=None, + destination_if_unmodified_since=None, + destination_if_match=None, + destination_if_none_match=None, + destination_lease_id=None, + source_lease_id=None, timeout=None): + ''' + Copies a blob asynchronously. This operation returns a copy operation + properties object, including a copy ID you can use to check or abort the + copy operation. The Blob service copies blobs on a best-effort basis. + + The source blob for a copy operation may be a block blob, an append blob, + or a page blob. If the destination blob already exists, it must be of the + same blob type as the source blob. Any existing destination blob will be + overwritten. The destination blob cannot be modified while a copy operation + is in progress. + + When copying from a page blob, the Blob service creates a destination page + blob of the source blob's length, initially containing all zeroes. Then + the source page ranges are enumerated, and non-empty ranges are copied. + + For a block blob or an append blob, the Blob service creates a committed + blob of zero length before returning from this operation. When copying + from a block blob, all committed blocks and their block IDs are copied. + Uncommitted blocks are not copied. At the end of the copy operation, the + destination blob will have the same committed block count as the source. + + When copying from an append blob, all committed blocks are copied. At the + end of the copy operation, the destination blob will have the same committed + block count as the source. + + For all blob types, you can call get_blob_properties on the destination + blob to check the status of the copy operation. The final blob will be + committed when the copy completes. + + :param str container_name: + Name of the destination container. The container must exist. + :param str blob_name: + Name of the destination blob. If the destination blob exists, it will + be overwritten. Otherwise, it will be created. + :param str copy_source: + A URL of up to 2 KB in length that specifies an Azure file or blob. + The value should be URL-encoded as it would appear in a request URI. + If the source is in another account, the source must either be public + or must be authenticated via a shared access signature. If the source + is public, no authentication is required. + Examples: + https://myaccount.blob.core.windows.net/mycontainer/myblob + https://myaccount.blob.core.windows.net/mycontainer/myblob?snapshot= + https://otheraccount.blob.core.windows.net/mycontainer/myblob?sastoken + :param metadata: + Name-value pairs associated with the blob as metadata. If no name-value + pairs are specified, the operation will copy the metadata from the + source blob or file to the destination blob. If one or more name-value + pairs are specified, the destination blob is created with the specified + metadata, and metadata is not copied from the source blob or file. + :type metadata: dict(str, str) + :param datetime source_if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this conditional header to copy the blob only if the source + blob has been modified since the specified date/time. + :param datetime source_if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this conditional header to copy the blob only if the source blob + has not been modified since the specified date/time. + :param ETag source_if_match: + An ETag value, or the wildcard character (*). Specify this conditional + header to copy the source blob only if its ETag matches the value + specified. If the ETag values do not match, the Blob service returns + status code 412 (Precondition Failed). This header cannot be specified + if the source is an Azure File. + :param ETag source_if_none_match: + An ETag value, or the wildcard character (*). Specify this conditional + header to copy the blob only if its ETag does not match the value + specified. If the values are identical, the Blob service returns status + code 412 (Precondition Failed). This header cannot be specified if the + source is an Azure File. + :param datetime destination_if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this conditional header to copy the blob only + if the destination blob has been modified since the specified date/time. + If the destination blob has not been modified, the Blob service returns + status code 412 (Precondition Failed). + :param datetime destination_if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this conditional header to copy the blob only + if the destination blob has not been modified since the specified + date/time. If the destination blob has been modified, the Blob service + returns status code 412 (Precondition Failed). + :param ETag destination_if_match: + An ETag value, or the wildcard character (*). Specify an ETag value for + this conditional header to copy the blob only if the specified ETag value + matches the ETag value for an existing destination blob. If the ETag for + the destination blob does not match the ETag specified for If-Match, the + Blob service returns status code 412 (Precondition Failed). + :param ETag destination_if_none_match: + An ETag value, or the wildcard character (*). Specify an ETag value for + this conditional header to copy the blob only if the specified ETag value + does not match the ETag value for the destination blob. Specify the wildcard + character (*) to perform the operation only if the destination blob does not + exist. If the specified condition isn't met, the Blob service returns status + code 412 (Precondition Failed). + :param str destination_lease_id: + The lease ID specified for this header must match the lease ID of the + destination blob. If the request does not include the lease ID or it is not + valid, the operation fails with status code 412 (Precondition Failed). + :param str source_lease_id: + Specify this to perform the Copy Blob operation only if + the lease ID given matches the active lease ID of the source blob. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: Copy operation properties such as status, source, and ID. + :rtype: :class:`~azure.storage.blob.models.CopyProperties` + ''' + return self._copy_blob(container_name, blob_name, copy_source, + metadata, + None, + source_if_modified_since, source_if_unmodified_since, + source_if_match, source_if_none_match, + destination_if_modified_since, + destination_if_unmodified_since, + destination_if_match, + destination_if_none_match, + destination_lease_id, + source_lease_id, timeout, + False) + + def _copy_blob(self, container_name, blob_name, copy_source, + metadata=None, + premium_page_blob_tier=None, + source_if_modified_since=None, + source_if_unmodified_since=None, + source_if_match=None, source_if_none_match=None, + destination_if_modified_since=None, + destination_if_unmodified_since=None, + destination_if_match=None, + destination_if_none_match=None, + destination_lease_id=None, + source_lease_id=None, timeout=None, + incremental_copy=False): + ''' + See copy_blob for more details. This helper method + allows for standard copies as well as incremental copies which are only supported for page blobs. + :param bool incremental_copy: + The timeout parameter is expressed in seconds. + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('copy_source', copy_source) + + if copy_source.startswith('/'): + # Backwards compatibility for earlier versions of the SDK where + # the copy source can be in the following formats: + # - Blob in named container: + # /accountName/containerName/blobName + # - Snapshot in named container: + # /accountName/containerName/blobName?snapshot= + # - Blob in root container: + # /accountName/blobName + # - Snapshot in root container: + # /accountName/blobName?snapshot= + account, _, source = \ + copy_source.partition('/')[2].partition('/') + copy_source = self.protocol + '://' + \ + self.primary_endpoint + '/' + source + + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + + if incremental_copy: + request.query = { + 'comp': 'incrementalcopy', + 'timeout': _int_to_str(timeout), + } + else: + request.query = {'timeout': _int_to_str(timeout)} + + request.headers = { + 'x-ms-copy-source': _to_str(copy_source), + 'x-ms-source-if-modified-since': _to_str(source_if_modified_since), + 'x-ms-source-if-unmodified-since': _to_str(source_if_unmodified_since), + 'x-ms-source-if-match': _to_str(source_if_match), + 'x-ms-source-if-none-match': _to_str(source_if_none_match), + 'If-Modified-Since': _datetime_to_utc_string(destination_if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(destination_if_unmodified_since), + 'If-Match': _to_str(destination_if_match), + 'If-None-Match': _to_str(destination_if_none_match), + 'x-ms-lease-id': _to_str(destination_lease_id), + 'x-ms-source-lease-id': _to_str(source_lease_id), + 'x-ms-access-tier': _to_str(premium_page_blob_tier) + } + _add_metadata_headers(metadata, request) + + return self._perform_request(request, _parse_properties, [BlobProperties]).copy + + def abort_copy_blob(self, container_name, blob_name, copy_id, + lease_id=None, timeout=None): + ''' + Aborts a pending copy_blob operation, and leaves a destination blob + with zero length and full metadata. + + :param str container_name: + Name of destination container. + :param str blob_name: + Name of destination blob. + :param str copy_id: + Copy identifier provided in the copy.id of the original + copy_blob operation. + :param str lease_id: + Required if the destination blob has an active infinite lease. + :param int timeout: + The timeout parameter is expressed in seconds. + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('copy_id', copy_id) + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.query = { + 'comp': 'copy', + 'copyid': _to_str(copy_id), + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-lease-id': _to_str(lease_id), + 'x-ms-copy-action': 'abort', + } + + self._perform_request(request) + + def delete_blob(self, container_name, blob_name, snapshot=None, + lease_id=None, delete_snapshots=None, + if_modified_since=None, if_unmodified_since=None, + if_match=None, if_none_match=None, timeout=None): + ''' + Marks the specified blob or snapshot for deletion. + The blob is later deleted during garbage collection. + + Note that in order to delete a blob, you must delete all of its + snapshots. You can delete both at the same time with the Delete + Blob operation. + + If a delete retention policy is enabled for the service, then this operation soft deletes the blob or snapshot + and retains the blob or snapshot for specified number of days. + After specified number of days, blob's data is removed from the service during garbage collection. + Soft deleted blob or snapshot is accessible through List Blobs API specifying include=Include.Deleted option. + Soft-deleted blob or snapshot can be restored using Undelete API. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param str snapshot: + The snapshot parameter is an opaque DateTime value that, + when present, specifies the blob snapshot to delete. + :param str lease_id: + Required if the blob has an active lease. + :param ~azure.storage.blob.models.DeleteSnapshot delete_snapshots: + Required if the blob has associated snapshots. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + request = HTTPRequest() + request.method = 'DELETE' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.headers = { + 'x-ms-lease-id': _to_str(lease_id), + 'x-ms-delete-snapshots': _to_str(delete_snapshots), + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'If-Match': _to_str(if_match), + 'If-None-Match': _to_str(if_none_match), + } + request.query = { + 'snapshot': _to_str(snapshot), + 'timeout': _int_to_str(timeout) + } + + self._perform_request(request) + + def undelete_blob(self, container_name, blob_name, timeout=None): + ''' + The undelete Blob operation restores the contents and metadata of soft deleted blob or snapshot. + Attempting to undelete a blob or snapshot that is not soft deleted will succeed without any changes. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param int timeout: + The timeout parameter is expressed in seconds. + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.query = { + 'comp': 'undelete', + 'timeout': _int_to_str(timeout) + } + + self._perform_request(request) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/blockblobservice.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/blockblobservice.py new file mode 100644 index 000000000000..abd693974656 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/blockblobservice.py @@ -0,0 +1,1063 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from io import ( + BytesIO +) +from os import ( + path, +) + +from ..common._common_conversion import ( + _encode_base64, + _to_str, + _int_to_str, + _datetime_to_utc_string, + _get_content_md5, +) +from ..common._constants import ( + SERVICE_HOST_BASE, + DEFAULT_PROTOCOL, +) +from ..common._error import ( + _validate_not_none, + _validate_type_bytes, + _validate_encryption_required, + _validate_encryption_unsupported, + _ERROR_VALUE_NEGATIVE, + _ERROR_VALUE_SHOULD_BE_STREAM +) +from ..common._http import HTTPRequest +from ..common._serialization import ( + _get_request_body, + _get_data_bytes_only, + _get_data_bytes_or_stream_only, + _add_metadata_headers, +) +from ..common._serialization import ( + _len_plus +) + +from ._deserialization import ( + _convert_xml_to_block_list, + _parse_base_properties, +) +from ._encryption import ( + _encrypt_blob, + _generate_blob_encryption_data, +) +from ._serialization import ( + _convert_block_list_to_xml, + _get_path, +) +from ._upload_chunking import ( + _BlockBlobChunkUploader, + _upload_blob_chunks, + _upload_blob_substream_blocks, +) +from .baseblobservice import BaseBlobService +from .models import ( + _BlobTypes, +) + + +class BlockBlobService(BaseBlobService): + ''' + Block blobs let you upload large blobs efficiently. Block blobs are comprised + of blocks, each of which is identified by a block ID. You create or modify a + block blob by writing a set of blocks and committing them by their block IDs. + Each block can be a different size, up to a maximum of 100 MB, and a block blob + can include up to 50,000 blocks. The maximum size of a block blob is therefore + approximately 4.75 TB (100 MB X 50,000 blocks). If you are writing a block + blob that is no more than 64 MB in size, you can upload it in its entirety with + a single write operation; see create_blob_from_bytes. + + :ivar int MAX_SINGLE_PUT_SIZE: + The largest size upload supported in a single put call. This is used by + the create_blob_from_* methods if the content length is known and is less + than this value. + :ivar int MAX_BLOCK_SIZE: + The size of the blocks put by create_blob_from_* methods if the content + length is unknown or is larger than MAX_SINGLE_PUT_SIZE. Smaller blocks + may be put. The maximum block size the service supports is 100MB. + :ivar int MIN_LARGE_BLOCK_UPLOAD_THRESHOLD: + The minimum block size at which the the memory-optimized, block upload + algorithm is considered. This algorithm is only applicable to the create_blob_from_file and + create_blob_from_stream methods and will prevent the full buffering of blocks. + In addition to the block size, ContentMD5 validation and Encryption must be disabled as + these options require the blocks to be buffered. + ''' + + MAX_SINGLE_PUT_SIZE = 64 * 1024 * 1024 + MAX_BLOCK_SIZE = 4 * 1024 * 1024 + MIN_LARGE_BLOCK_UPLOAD_THRESHOLD = 4 * 1024 * 1024 + 1 + + def __init__(self, account_name=None, account_key=None, sas_token=None, is_emulated=False, + protocol=DEFAULT_PROTOCOL, endpoint_suffix=SERVICE_HOST_BASE, custom_domain=None, + request_session=None, connection_string=None, socket_timeout=None, token_credential=None): + ''' + :param str account_name: + The storage account name. This is used to authenticate requests + signed with an account key and to construct the storage endpoint. It + is required unless a connection string is given, or if a custom + domain is used with anonymous authentication. + :param str account_key: + The storage account key. This is used for shared key authentication. + If neither account key or sas token is specified, anonymous access + will be used. + :param str sas_token: + A shared access signature token to use to authenticate requests + instead of the account key. If account key and sas token are both + specified, account key will be used to sign. If neither are + specified, anonymous access will be used. + :param bool is_emulated: + Whether to use the emulator. Defaults to False. If specified, will + override all other parameters besides connection string and request + session. + :param str protocol: + The protocol to use for requests. Defaults to https. + :param str endpoint_suffix: + The host base component of the url, minus the account name. Defaults + to Azure (core.windows.net). Override this to use the China cloud + (core.chinacloudapi.cn). + :param str custom_domain: + The custom domain to use. This can be set in the Azure Portal. For + example, 'www.mydomain.com'. + :param requests.Session request_session: + The session object to use for http requests. + :param str connection_string: + If specified, this will override all other parameters besides + request session. See + http://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/ + for the connection string format. + :param int socket_timeout: + If specified, this will override the default socket timeout. The timeout specified is in seconds. + See DEFAULT_SOCKET_TIMEOUT in _constants.py for the default value. + :param token_credential: + A token credential used to authenticate HTTPS requests. The token value + should be updated before its expiration. + :type `~azure.storage.common.TokenCredential` + ''' + self.blob_type = _BlobTypes.BlockBlob + super(BlockBlobService, self).__init__( + account_name, account_key, sas_token, is_emulated, protocol, endpoint_suffix, + custom_domain, request_session, connection_string, socket_timeout, token_credential) + + def put_block(self, container_name, blob_name, block, block_id, + validate_content=False, lease_id=None, timeout=None): + ''' + Creates a new block to be committed as part of a blob. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of blob. + :param block: Content of the block. + :type block: io.IOBase or bytes + Content of the block. + :param str block_id: + A valid Base64 string value that identifies the block. Prior to + encoding, the string must be less than or equal to 64 bytes in size. + For a given blob, the length of the value specified for the blockid + parameter must be the same size for each block. Note that the Base64 + string must be URL-encoded. + :param bool validate_content: + If true, calculates an MD5 hash of the block content. The storage + service checks the hash of the content that has arrived + with the hash that was sent. This is primarily valuable for detecting + bitflips on the wire if using http instead of https as https (the default) + will already validate. Note that this MD5 hash is not stored with the + blob. + :param str lease_id: + Required if the blob has an active lease. + :param int timeout: + The timeout parameter is expressed in seconds. + ''' + _validate_encryption_unsupported(self.require_encryption, self.key_encryption_key) + + self._put_block( + container_name, + blob_name, + block, + block_id, + validate_content=validate_content, + lease_id=lease_id, + timeout=timeout + ) + + def put_block_list( + self, container_name, blob_name, block_list, content_settings=None, + metadata=None, validate_content=False, lease_id=None, if_modified_since=None, + if_unmodified_since=None, if_match=None, if_none_match=None, + timeout=None): + ''' + Writes a blob by specifying the list of block IDs that make up the blob. + In order to be written as part of a blob, a block must have been + successfully written to the server in a prior Put Block operation. + + You can call Put Block List to update a blob by uploading only those + blocks that have changed, then committing the new and existing blocks + together. You can do this by specifying whether to commit a block from + the committed block list or from the uncommitted block list, or to commit + the most recently uploaded version of the block, whichever list it may + belong to. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param block_list: + A list of :class:`~azure.storeage.blob.models.BlobBlock` containing the block ids and block state. + :type block_list: list(:class:`~azure.storage.blob.models.BlobBlock`) + :param ~azure.storage.blob.models.ContentSettings content_settings: + ContentSettings object used to set properties on the blob. + :param metadata: + Name-value pairs associated with the blob as metadata. + :type metadata: dict(str, str) + :param bool validate_content: + If true, calculates an MD5 hash of the block list content. The storage + service checks the hash of the block list content that has arrived + with the hash that was sent. This is primarily valuable for detecting + bitflips on the wire if using http instead of https as https (the default) + will already validate. Note that this check is associated with + the block list content, and not with the content of the blob itself. + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: ETag and last modified properties for the updated Block Blob + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + + _validate_encryption_unsupported(self.require_encryption, self.key_encryption_key) + + return self._put_block_list( + container_name, + blob_name, + block_list, + content_settings=content_settings, + metadata=metadata, + validate_content=validate_content, + lease_id=lease_id, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + if_match=if_match, + if_none_match=if_none_match, + timeout=timeout + ) + + def get_block_list(self, container_name, blob_name, snapshot=None, + block_list_type=None, lease_id=None, timeout=None): + ''' + Retrieves the list of blocks that have been uploaded as part of a + block blob. There are two block lists maintained for a blob: + Committed Block List: + The list of blocks that have been successfully committed to a + given blob with Put Block List. + Uncommitted Block List: + The list of blocks that have been uploaded for a blob using + Put Block, but that have not yet been committed. These blocks + are stored in Azure in association with a blob, but do not yet + form part of the blob. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param str snapshot: + Datetime to determine the time to retrieve the blocks. + :param str block_list_type: + Specifies whether to return the list of committed blocks, the list + of uncommitted blocks, or both lists together. Valid values are: + committed, uncommitted, or all. + :param str lease_id: + Required if the blob has an active lease. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: list committed and/or uncommitted blocks for Block Blob + :rtype: :class:`~azure.storage.blob.models.BlobBlockList` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + request = HTTPRequest() + request.method = 'GET' + request.host_locations = self._get_host_locations(secondary=True) + request.path = _get_path(container_name, blob_name) + request.query = { + 'comp': 'blocklist', + 'snapshot': _to_str(snapshot), + 'blocklisttype': _to_str(block_list_type), + 'timeout': _int_to_str(timeout), + } + request.headers = {'x-ms-lease-id': _to_str(lease_id)} + + return self._perform_request(request, _convert_xml_to_block_list) + + def put_block_from_url(self, container_name, blob_name, copy_source_url, source_range_start, source_range_end, + block_id, source_content_md5=None, lease_id=None, timeout=None): + """ + Creates a new block to be committed as part of a blob. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of blob. + :param str copy_source_url: + The URL of the source data. It can point to any Azure Blob or File, that is either public or has a + shared access signature attached. + :param int source_range_start: + This indicates the start of the range of bytes(inclusive) that has to be taken from the copy source. + :param int source_range_end: + This indicates the end of the range of bytes(inclusive) that has to be taken from the copy source. + :param str block_id: + A valid Base64 string value that identifies the block. Prior to + encoding, the string must be less than or equal to 64 bytes in size. + For a given blob, the length of the value specified for the blockid + parameter must be the same size for each block. Note that the Base64 + string must be URL-encoded. + :param str source_content_md5: + If given, the service will calculate the MD5 hash of the block content and compare against this value. + :param str lease_id: + Required if the blob has an active lease. + :param int timeout: + The timeout parameter is expressed in seconds. + """ + _validate_encryption_unsupported(self.require_encryption, self.key_encryption_key) + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('copy_source_url', copy_source_url) + _validate_not_none('source_range_start', source_range_start) + _validate_not_none('source_range_end', source_range_end) + _validate_not_none('block_id', block_id) + + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.query = { + 'comp': 'block', + 'blockid': _encode_base64(_to_str(block_id)), + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-lease-id': _to_str(lease_id), + 'x-ms-copy-source': copy_source_url, + 'x-ms-source-range': 'bytes=' + _to_str(source_range_start) + '-' + _to_str(source_range_end), + 'x-ms-source-content-md5': source_content_md5, + } + + self._perform_request(request) + + # ----Convenience APIs----------------------------------------------------- + + def create_blob_from_path( + self, container_name, blob_name, file_path, content_settings=None, + metadata=None, validate_content=False, progress_callback=None, + max_connections=2, lease_id=None, if_modified_since=None, + if_unmodified_since=None, if_match=None, if_none_match=None, timeout=None): + ''' + Creates a new blob from a file path, or updates the content of an + existing blob, with automatic chunking and progress notifications. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of blob to create or update. + :param str file_path: + Path of the file to upload as the blob content. + :param ~azure.storage.blob.models.ContentSettings content_settings: + ContentSettings object used to set blob properties. + :param metadata: + Name-value pairs associated with the blob as metadata. + :type metadata: dict(str, str) + :param bool validate_content: + If true, calculates an MD5 hash for each chunk of the blob. The storage + service checks the hash of the content that has arrived with the hash + that was sent. This is primarily valuable for detecting bitflips on + the wire if using http instead of https as https (the default) will + already validate. Note that this MD5 hash is not stored with the + blob. Also note that if enabled, the memory-efficient upload algorithm + will not be used, because computing the MD5 hash requires buffering + entire blocks, and doing so defeats the purpose of the memory-efficient algorithm. + :param progress_callback: + Callback for progress with signature function(current, total) where + current is the number of bytes transfered so far, and total is the + size of the blob, or None if the total size is unknown. + :type progress_callback: func(current, total) + :param int max_connections: + Maximum number of parallel connections to use when the blob size exceeds + 64MB. + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. This method may make + multiple calls to the Azure service and the timeout will apply to + each call individually. + :return: ETag and last modified properties for the Block Blob + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('file_path', file_path) + + count = path.getsize(file_path) + with open(file_path, 'rb') as stream: + return self.create_blob_from_stream( + container_name=container_name, + blob_name=blob_name, + stream=stream, + count=count, + content_settings=content_settings, + metadata=metadata, + validate_content=validate_content, + lease_id=lease_id, + progress_callback=progress_callback, + max_connections=max_connections, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + if_match=if_match, + if_none_match=if_none_match, + timeout=timeout) + + def create_blob_from_stream( + self, container_name, blob_name, stream, count=None, + content_settings=None, metadata=None, validate_content=False, + progress_callback=None, max_connections=2, lease_id=None, + if_modified_since=None, if_unmodified_since=None, if_match=None, + if_none_match=None, timeout=None, use_byte_buffer=False): + ''' + Creates a new blob from a file/stream, or updates the content of + an existing blob, with automatic chunking and progress + notifications. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of blob to create or update. + :param io.IOBase stream: + Opened file/stream to upload as the blob content. + :param int count: + Number of bytes to read from the stream. This is optional, but + should be supplied for optimal performance. + :param ~azure.storage.blob.models.ContentSettings content_settings: + ContentSettings object used to set blob properties. + :param metadata: + Name-value pairs associated with the blob as metadata. + :type metadata: dict(str, str) + :param bool validate_content: + If true, calculates an MD5 hash for each chunk of the blob. The storage + service checks the hash of the content that has arrived with the hash + that was sent. This is primarily valuable for detecting bitflips on + the wire if using http instead of https as https (the default) will + already validate. Note that this MD5 hash is not stored with the + blob. Also note that if enabled, the memory-efficient upload algorithm + will not be used, because computing the MD5 hash requires buffering + entire blocks, and doing so defeats the purpose of the memory-efficient algorithm. + :param progress_callback: + Callback for progress with signature function(current, total) where + current is the number of bytes transfered so far, and total is the + size of the blob, or None if the total size is unknown. + :type progress_callback: func(current, total) + :param int max_connections: + Maximum number of parallel connections to use when the blob size exceeds + 64MB. Note that parallel upload requires the stream to be seekable. + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. This method may make + multiple calls to the Azure service and the timeout will apply to + each call individually. + :param bool use_byte_buffer: + If True, this will force usage of the original full block buffering upload path. + By default, this value is False and will employ a memory-efficient, + streaming upload algorithm under the following conditions: + The provided stream is seekable, 'require_encryption' is False, and + MAX_BLOCK_SIZE >= MIN_LARGE_BLOCK_UPLOAD_THRESHOLD. + One should consider the drawbacks of using this approach. In order to achieve + memory-efficiency, a IOBase stream or file-like object is segmented into logical blocks + using a SubStream wrapper. In order to read the correct data, each SubStream must acquire + a lock so that it can safely seek to the right position on the shared, underlying stream. + If max_connections > 1, the concurrency will result in a considerable amount of seeking on + the underlying stream. For the most common inputs such as a file-like stream object, seeking + is an inexpensive operation and this is not much of a concern. However, for other variants of streams + this may not be the case. The trade-off for memory-efficiency must be weighed against the cost of seeking + with your input stream. + The SubStream class will attempt to buffer up to 4 MB internally to reduce the amount of + seek and read calls to the underlying stream. This is particularly beneficial when uploading larger blocks. + :return: ETag and last modified properties for the Block Blob + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('stream', stream) + _validate_encryption_required(self.require_encryption, self.key_encryption_key) + + # Adjust count to include padding if we are expected to encrypt. + adjusted_count = count + if (self.key_encryption_key is not None) and (adjusted_count is not None): + adjusted_count += (16 - (count % 16)) + + # Do single put if the size is smaller than MAX_SINGLE_PUT_SIZE + if adjusted_count is not None and (adjusted_count < self.MAX_SINGLE_PUT_SIZE): + if progress_callback: + progress_callback(0, count) + + data = stream.read(count) + resp = self._put_blob( + container_name=container_name, + blob_name=blob_name, + blob=data, + content_settings=content_settings, + metadata=metadata, + validate_content=validate_content, + lease_id=lease_id, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + if_match=if_match, + if_none_match=if_none_match, + timeout=timeout) + + if progress_callback: + progress_callback(count, count) + + return resp + else: # Size is larger than MAX_SINGLE_PUT_SIZE, must upload with multiple put_block calls + cek, iv, encryption_data = None, None, None + + use_original_upload_path = use_byte_buffer or validate_content or self.require_encryption or \ + self.MAX_BLOCK_SIZE < self.MIN_LARGE_BLOCK_UPLOAD_THRESHOLD or \ + hasattr(stream, 'seekable') and not stream.seekable() or \ + not hasattr(stream, 'seek') or not hasattr(stream, 'tell') + + if use_original_upload_path: + if self.key_encryption_key: + cek, iv, encryption_data = _generate_blob_encryption_data(self.key_encryption_key) + + block_ids = _upload_blob_chunks( + blob_service=self, + container_name=container_name, + blob_name=blob_name, + blob_size=count, + block_size=self.MAX_BLOCK_SIZE, + stream=stream, + max_connections=max_connections, + progress_callback=progress_callback, + validate_content=validate_content, + lease_id=lease_id, + uploader_class=_BlockBlobChunkUploader, + timeout=timeout, + content_encryption_key=cek, + initialization_vector=iv + ) + else: + block_ids = _upload_blob_substream_blocks( + blob_service=self, + container_name=container_name, + blob_name=blob_name, + blob_size=count, + block_size=self.MAX_BLOCK_SIZE, + stream=stream, + max_connections=max_connections, + progress_callback=progress_callback, + validate_content=validate_content, + lease_id=lease_id, + uploader_class=_BlockBlobChunkUploader, + timeout=timeout, + ) + + return self._put_block_list( + container_name=container_name, + blob_name=blob_name, + block_list=block_ids, + content_settings=content_settings, + metadata=metadata, + validate_content=validate_content, + lease_id=lease_id, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + if_match=if_match, + if_none_match=if_none_match, + timeout=timeout, + encryption_data=encryption_data + ) + + def create_blob_from_bytes( + self, container_name, blob_name, blob, index=0, count=None, + content_settings=None, metadata=None, validate_content=False, + progress_callback=None, max_connections=2, lease_id=None, + if_modified_since=None, if_unmodified_since=None, if_match=None, + if_none_match=None, timeout=None): + ''' + Creates a new blob from an array of bytes, or updates the content + of an existing blob, with automatic chunking and progress + notifications. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of blob to create or update. + :param bytes blob: + Content of blob as an array of bytes. + :param int index: + Start index in the array of bytes. + :param int count: + Number of bytes to upload. Set to None or negative value to upload + all bytes starting from index. + :param ~azure.storage.blob.models.ContentSettings content_settings: + ContentSettings object used to set blob properties. + :param metadata: + Name-value pairs associated with the blob as metadata. + :type metadata: dict(str, str) + :param bool validate_content: + If true, calculates an MD5 hash for each chunk of the blob. The storage + service checks the hash of the content that has arrived with the hash + that was sent. This is primarily valuable for detecting bitflips on + the wire if using http instead of https as https (the default) will + already validate. Note that this MD5 hash is not stored with the + blob. + :param progress_callback: + Callback for progress with signature function(current, total) where + current is the number of bytes transfered so far, and total is the + size of the blob, or None if the total size is unknown. + :type progress_callback: func(current, total) + :param int max_connections: + Maximum number of parallel connections to use when the blob size exceeds + 64MB. + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. This method may make + multiple calls to the Azure service and the timeout will apply to + each call individually. + :return: ETag and last modified properties for the Block Blob + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('blob', blob) + _validate_not_none('index', index) + _validate_type_bytes('blob', blob) + + if index < 0: + raise IndexError(_ERROR_VALUE_NEGATIVE.format('index')) + + if count is None or count < 0: + count = len(blob) - index + + stream = BytesIO(blob) + stream.seek(index) + + return self.create_blob_from_stream( + container_name=container_name, + blob_name=blob_name, + stream=stream, + count=count, + content_settings=content_settings, + metadata=metadata, + validate_content=validate_content, + progress_callback=progress_callback, + max_connections=max_connections, + lease_id=lease_id, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + if_match=if_match, + if_none_match=if_none_match, + timeout=timeout, + use_byte_buffer=True + ) + + def create_blob_from_text( + self, container_name, blob_name, text, encoding='utf-8', + content_settings=None, metadata=None, validate_content=False, + progress_callback=None, max_connections=2, lease_id=None, + if_modified_since=None, if_unmodified_since=None, if_match=None, + if_none_match=None, timeout=None): + ''' + Creates a new blob from str/unicode, or updates the content of an + existing blob, with automatic chunking and progress notifications. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of blob to create or update. + :param str text: + Text to upload to the blob. + :param str encoding: + Python encoding to use to convert the text to bytes. + :param ~azure.storage.blob.models.ContentSettings content_settings: + ContentSettings object used to set blob properties. + :param metadata: + Name-value pairs associated with the blob as metadata. + :type metadata: dict(str, str) + :param bool validate_content: + If true, calculates an MD5 hash for each chunk of the blob. The storage + service checks the hash of the content that has arrived with the hash + that was sent. This is primarily valuable for detecting bitflips on + the wire if using http instead of https as https (the default) will + already validate. Note that this MD5 hash is not stored with the + blob. + :param progress_callback: + Callback for progress with signature function(current, total) where + current is the number of bytes transfered so far, and total is the + size of the blob, or None if the total size is unknown. + :type progress_callback: func(current, total) + :param int max_connections: + Maximum number of parallel connections to use when the blob size exceeds + 64MB. + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. This method may make + multiple calls to the Azure service and the timeout will apply to + each call individually. + :return: ETag and last modified properties for the Block Blob + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('text', text) + + if not isinstance(text, bytes): + _validate_not_none('encoding', encoding) + text = text.encode(encoding) + + return self.create_blob_from_bytes( + container_name=container_name, + blob_name=blob_name, + blob=text, + index=0, + count=len(text), + content_settings=content_settings, + metadata=metadata, + validate_content=validate_content, + lease_id=lease_id, + progress_callback=progress_callback, + max_connections=max_connections, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + if_match=if_match, + if_none_match=if_none_match, + timeout=timeout) + + def set_standard_blob_tier( + self, container_name, blob_name, standard_blob_tier, timeout=None): + ''' + Sets the block blob tiers on the blob. This API is only supported for block blobs on standard storage accounts. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of blob to update. + :param StandardBlobTier standard_blob_tier: + A standard blob tier value to set the blob to. For this version of the library, + this is only applicable to block blobs on standard storage accounts. + :param int timeout: + The timeout parameter is expressed in seconds. This method may make + multiple calls to the Azure service and the timeout will apply to + each call individually. + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('standard_blob_tier', standard_blob_tier) + + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.query = { + 'comp': 'tier', + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-access-tier': _to_str(standard_blob_tier) + } + + self._perform_request(request) + + # -----Helper methods------------------------------------ + def _put_blob(self, container_name, blob_name, blob, content_settings=None, + metadata=None, validate_content=False, lease_id=None, if_modified_since=None, + if_unmodified_since=None, if_match=None, if_none_match=None, + timeout=None): + ''' + Creates a blob or updates an existing blob. + + See create_blob_from_* for high level + functions that handle the creation and upload of large blobs with + automatic chunking and progress notifications. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of blob to create or update. + :param bytes blob: + Content of blob as bytes (size < 64MB). For larger size, you + must call put_block and put_block_list to set content of blob. + :param ~azure.storage.blob.models.ContentSettings content_settings: + ContentSettings object used to set properties on the blob. + :param metadata: + Name-value pairs associated with the blob as metadata. + :param bool validate_content: + If true, calculates an MD5 hash of the blob content. The storage + service checks the hash of the content that has arrived + with the hash that was sent. This is primarily valuable for detecting + bitflips on the wire if using http instead of https as https (the default) + will already validate. Note that this MD5 hash is not stored with the + blob. + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: ETag and last modified properties for the new Block Blob + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_encryption_required(self.require_encryption, self.key_encryption_key) + + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.query = {'timeout': _int_to_str(timeout)} + request.headers = { + 'x-ms-blob-type': _to_str(self.blob_type), + 'x-ms-lease-id': _to_str(lease_id), + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'If-Match': _to_str(if_match), + 'If-None-Match': _to_str(if_none_match) + } + _add_metadata_headers(metadata, request) + if content_settings is not None: + request.headers.update(content_settings._to_headers()) + blob = _get_data_bytes_only('blob', blob) + if self.key_encryption_key: + encryption_data, blob = _encrypt_blob(blob, self.key_encryption_key) + request.headers['x-ms-meta-encryptiondata'] = encryption_data + request.body = blob + + if validate_content: + computed_md5 = _get_content_md5(request.body) + request.headers['Content-MD5'] = _to_str(computed_md5) + + return self._perform_request(request, _parse_base_properties) + + def _put_block(self, container_name, blob_name, block, block_id, + validate_content=False, lease_id=None, timeout=None): + ''' + See put_block for more details. This helper method + allows for encryption or other such special behavior because + it is safely handled by the library. These behaviors are + prohibited in the public version of this function. + ''' + + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('block', block) + _validate_not_none('block_id', block_id) + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.query = { + 'comp': 'block', + 'blockid': _encode_base64(_to_str(block_id)), + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-lease-id': _to_str(lease_id) + } + request.body = _get_data_bytes_or_stream_only('block', block) + if hasattr(request.body, 'read'): + if _len_plus(request.body) is None: + try: + data = b'' + for chunk in iter(lambda: request.body.read(4096), b""): + data += chunk + request.body = data + except AttributeError: + raise ValueError(_ERROR_VALUE_SHOULD_BE_STREAM.format('request.body')) + + if validate_content: + computed_md5 = _get_content_md5(request.body) + request.headers['Content-MD5'] = _to_str(computed_md5) + + self._perform_request(request) + + def _put_block_list( + self, container_name, blob_name, block_list, content_settings=None, + metadata=None, validate_content=False, lease_id=None, if_modified_since=None, + if_unmodified_since=None, if_match=None, if_none_match=None, + timeout=None, encryption_data=None): + ''' + See put_block_list for more details. This helper method + allows for encryption or other such special behavior because + it is safely handled by the library. These behaviors are + prohibited in the public version of this function. + :param str encryption_data: + A JSON formatted string containing the encryption metadata generated for this + blob if it was encrypted all at once upon upload. This should only be passed + in by internal methods. + ''' + + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('block_list', block_list) + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.query = { + 'comp': 'blocklist', + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-lease-id': _to_str(lease_id), + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'If-Match': _to_str(if_match), + 'If-None-Match': _to_str(if_none_match), + } + _add_metadata_headers(metadata, request) + if content_settings is not None: + request.headers.update(content_settings._to_headers()) + request.body = _get_request_body( + _convert_block_list_to_xml(block_list)) + + if validate_content: + computed_md5 = _get_content_md5(request.body) + request.headers['Content-MD5'] = _to_str(computed_md5) + + if encryption_data is not None: + request.headers['x-ms-meta-encryptiondata'] = encryption_data + + return self._perform_request(request, _parse_base_properties) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/models.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/models.py new file mode 100644 index 000000000000..e39067aa3ac1 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/models.py @@ -0,0 +1,781 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +from ..common._common_conversion import _to_str + + +class Container(object): + ''' + Blob container class. + + :ivar str name: + The name of the container. + :ivar metadata: + A dict containing name-value pairs associated with the container as metadata. + This var is set to None unless the include=metadata param was included + for the list containers operation. If this parameter was specified but the + container has no metadata, metadata will be set to an empty dictionary. + :vartype metadata: dict(str, str) + :ivar ContainerProperties properties: + System properties for the container. + ''' + + def __init__(self, name=None, props=None, metadata=None): + self.name = name + self.properties = props or ContainerProperties() + self.metadata = metadata + + +class ContainerProperties(object): + ''' + Blob container's properties class. + + :ivar datetime last_modified: + A datetime object representing the last time the container was modified. + :ivar str etag: + The ETag contains a value that you can use to perform operations + conditionally. + :ivar LeaseProperties lease: + Stores all the lease information for the container. + :ivar bool has_immutability_policy: + Represents whether the container has an immutability policy. + :ivar bool has_legal_hold: + Represents whether the container has a legal hold. + ''' + + def __init__(self): + self.last_modified = None + self.etag = None + self.lease = LeaseProperties() + self.public_access = None + self.has_immutability_policy = None + self.has_legal_hold = None + + +class Blob(object): + ''' + Blob class. + + :ivar str name: + Name of blob. + :ivar str snapshot: + A DateTime value that uniquely identifies the snapshot. The value of + this header indicates the snapshot version, and may be used in + subsequent requests to access the snapshot. + :ivar content: + Blob content. + :vartype content: str or bytes + :ivar BlobProperties properties: + Stores all the system properties for the blob. + :ivar metadata: + Name-value pairs associated with the blob as metadata. + :ivar bool deleted: + Specify whether the blob was soft deleted. + In other words, if the blob is being retained by the delete retention policy, + this field would be True. The blob could be undeleted or it will be garbage collected after the specified + time period. + ''' + + def __init__(self, name=None, snapshot=None, content=None, props=None, metadata=None, deleted=False): + self.name = name + self.snapshot = snapshot + self.content = content + self.properties = props or BlobProperties() + self.metadata = metadata + self.deleted = deleted + + +class BlobProperties(object): + ''' + Blob Properties + + :ivar str blob_type: + String indicating this blob's type. + :ivar datetime last_modified: + A datetime object representing the last time the blob was modified. + :ivar str etag: + The ETag contains a value that you can use to perform operations + conditionally. + :ivar int content_length: + The length of the content returned. If the entire blob was requested, + the length of blob in bytes. If a subset of the blob was requested, the + length of the returned subset. + :ivar str content_range: + Indicates the range of bytes returned in the event that the client + requested a subset of the blob. + :ivar int append_blob_committed_block_count: + (For Append Blobs) Number of committed blocks in the blob. + :ivar int page_blob_sequence_number: + (For Page Blobs) Sequence number for page blob used for coordinating + concurrent writes. + :ivar bool server_encrypted: + Set to true if the blob is encrypted on the server. + :ivar ~azure.storage.blob.models.CopyProperties copy: + Stores all the copy properties for the blob. + :ivar ~azure.storage.blob.models.ContentSettings content_settings: + Stores all the content settings for the blob. + :ivar ~azure.storage.blob.models.LeaseProperties lease: + Stores all the lease information for the blob. + :ivar StandardBlobTier blob_tier: + Indicates the access tier of the blob. The hot tier is optimized + for storing data that is accessed frequently. The cool storage tier + is optimized for storing data that is infrequently accessed and stored + for at least a month. The archive tier is optimized for storing + data that is rarely accessed and stored for at least six months + with flexible latency requirements. + :ivar datetime blob_tier_change_time: + Indicates when the access tier was last changed. + :ivar bool blob_tier_inferred: + Indicates whether the access tier was inferred by the service. + If false, it indicates that the tier was set explicitly. + :ivar datetime deleted_time: + A datetime object representing the time at which the blob was deleted. + :ivar int remaining_retention_days: + The number of days that the blob will be retained before being permanently deleted by the service. + :ivar datetime creation_time: + Indicates when the blob was created, in UTC. + ''' + + def __init__(self): + self.blob_type = None + self.last_modified = None + self.etag = None + self.content_length = None + self.content_range = None + self.append_blob_committed_block_count = None + self.page_blob_sequence_number = None + self.server_encrypted = None + self.copy = CopyProperties() + self.content_settings = ContentSettings() + self.lease = LeaseProperties() + self.blob_tier = None + self.blob_tier_change_time = None + self.blob_tier_inferred = False + self.deleted_time = None + self.remaining_retention_days = None + self.creation_time = None + + +class ContentSettings(object): + ''' + Used to store the content settings of a blob. + + :ivar str content_type: + The content type specified for the blob. If no content type was + specified, the default content type is application/octet-stream. + :ivar str content_encoding: + If the content_encoding has previously been set + for the blob, that value is stored. + :ivar str content_language: + If the content_language has previously been set + for the blob, that value is stored. + :ivar str content_disposition: + content_disposition conveys additional information about how to + process the response payload, and also can be used to attach + additional metadata. If content_disposition has previously been set + for the blob, that value is stored. + :ivar str cache_control: + If the cache_control has previously been set for + the blob, that value is stored. + :ivar str content_md5: + If the content_md5 has been set for the blob, this response + header is stored so that the client can check for message content + integrity. + ''' + + def __init__( + self, content_type=None, content_encoding=None, + content_language=None, content_disposition=None, + cache_control=None, content_md5=None): + self.content_type = content_type + self.content_encoding = content_encoding + self.content_language = content_language + self.content_disposition = content_disposition + self.cache_control = cache_control + self.content_md5 = content_md5 + + def _to_headers(self): + return { + 'x-ms-blob-cache-control': _to_str(self.cache_control), + 'x-ms-blob-content-type': _to_str(self.content_type), + 'x-ms-blob-content-disposition': _to_str(self.content_disposition), + 'x-ms-blob-content-md5': _to_str(self.content_md5), + 'x-ms-blob-content-encoding': _to_str(self.content_encoding), + 'x-ms-blob-content-language': _to_str(self.content_language), + } + + +class CopyProperties(object): + ''' + Blob Copy Properties. + + :ivar str id: + String identifier for the last attempted Copy Blob operation where this blob + was the destination blob. This header does not appear if this blob has never + been the destination in a Copy Blob operation, or if this blob has been + modified after a concluded Copy Blob operation using Set Blob Properties, + Put Blob, or Put Block List. + :ivar str source: + URL up to 2 KB in length that specifies the source blob used in the last attempted + Copy Blob operation where this blob was the destination blob. This header does not + appear if this blob has never been the destination in a Copy Blob operation, or if + this blob has been modified after a concluded Copy Blob operation using + Set Blob Properties, Put Blob, or Put Block List. + :ivar str status: + State of the copy operation identified by Copy ID, with these values: + success: + Copy completed successfully. + pending: + Copy is in progress. Check copy_status_description if intermittent, + non-fatal errors impede copy progress but don't cause failure. + aborted: + Copy was ended by Abort Copy Blob. + failed: + Copy failed. See copy_status_description for failure details. + :ivar str progress: + Contains the number of bytes copied and the total bytes in the source in the last + attempted Copy Blob operation where this blob was the destination blob. Can show + between 0 and Content-Length bytes copied. + :ivar datetime completion_time: + Conclusion time of the last attempted Copy Blob operation where this blob was the + destination blob. This value can specify the time of a completed, aborted, or + failed copy attempt. + :ivar str status_description: + only appears when x-ms-copy-status is failed or pending. Describes cause of fatal + or non-fatal copy operation failure. + ''' + + def __init__(self): + self.id = None + self.source = None + self.status = None + self.progress = None + self.completion_time = None + self.status_description = None + + +class LeaseProperties(object): + ''' + Blob Lease Properties. + + :ivar str status: + The lease status of the blob. + Possible values: locked|unlocked + :ivar str state: + Lease state of the blob. + Possible values: available|leased|expired|breaking|broken + :ivar str duration: + When a blob is leased, specifies whether the lease is of infinite or fixed duration. + ''' + + def __init__(self): + self.status = None + self.state = None + self.duration = None + + +class BlobPrefix(object): + ''' + BlobPrefix objects may potentially returned in the blob list when + :func:`~azure.storage.blob.baseblobservice.BaseBlobService.list_blobs` is + used with a delimiter. Prefixes can be thought of as virtual blob directories. + + :ivar str name: The name of the blob prefix. + ''' + + def __init__(self): + self.name = None + + +class BlobBlockState(object): + '''Block blob block types.''' + + Committed = 'Committed' + '''Committed blocks.''' + + Latest = 'Latest' + '''Latest blocks.''' + + Uncommitted = 'Uncommitted' + '''Uncommitted blocks.''' + + +class BlobBlock(object): + ''' + BlockBlob Block class. + + :ivar str id: + Block id. + :ivar str state: + Block state. + Possible valuse: committed|uncommitted + :ivar int size: + Block size in bytes. + ''' + + def __init__(self, id=None, state=BlobBlockState.Latest): + self.id = id + self.state = state + + def _set_size(self, size): + self.size = size + + +class BlobBlockList(object): + ''' + Blob Block List class. + + :ivar committed_blocks: + List of committed blocks. + :vartype committed_blocks: list(:class:`~azure.storage.blob.models.BlobBlock`) + :ivar uncommitted_blocks: + List of uncommitted blocks. + :vartype uncommitted_blocks: list(:class:`~azure.storage.blob.models.BlobBlock`) + ''' + + def __init__(self): + self.committed_blocks = list() + self.uncommitted_blocks = list() + + +class PageRange(object): + ''' + Page Range for page blob. + + :ivar int start: + Start of page range in bytes. + :ivar int end: + End of page range in bytes. + :ivar bool is_cleared: + Indicates if a page range is cleared or not. Only applicable + for get_page_range_diff API. + ''' + + def __init__(self, start=None, end=None, is_cleared=False): + self.start = start + self.end = end + self.is_cleared = is_cleared + + +class ResourceProperties(object): + ''' + Base response for a resource request. + + :ivar str etag: + Opaque etag value that can be used to check if resource + has been modified. + :ivar datetime last_modified: + Datetime for last time resource was modified. + ''' + + def __init__(self): + self.last_modified = None + self.etag = None + + +class AppendBlockProperties(ResourceProperties): + ''' + Response for an append block request. + + :ivar int append_offset: + Position to start next append. + :ivar int committed_block_count: + Number of committed append blocks. + ''' + + def __init__(self): + super(ResourceProperties, self).__init__() + self.append_offset = None + self.committed_block_count = None + + +class PageBlobProperties(ResourceProperties): + ''' + Response for a page request. + + :ivar int sequence_number: + Identifer for page blobs to help handle concurrent writes. + ''' + + def __init__(self): + super(ResourceProperties, self).__init__() + self.sequence_number = None + + +class PublicAccess(object): + ''' + Specifies whether data in the container may be accessed publicly and the level of access. + ''' + + OFF = 'off' + ''' + Specifies that there is no public read access for both the container and blobs within the container. + Clients cannot enumerate the containers within the storage account as well as the blobs within the container. + ''' + + Blob = 'blob' + ''' + Specifies public read access for blobs. Blob data within this container can be read + via anonymous request, but container data is not available. Clients cannot enumerate + blobs within the container via anonymous request. + ''' + + Container = 'container' + ''' + Specifies full public read access for container and blob data. Clients can enumerate + blobs within the container via anonymous request, but cannot enumerate containers + within the storage account. + ''' + + +class DeleteSnapshot(object): + ''' + Required if the blob has associated snapshots. Specifies how to handle the snapshots. + ''' + + Include = 'include' + ''' + Delete the base blob and all of its snapshots. + ''' + + Only = 'only' + ''' + Delete only the blob's snapshots and not the blob itself. + ''' + + +class BlockListType(object): + ''' + Specifies whether to return the list of committed blocks, the list of uncommitted + blocks, or both lists together. + ''' + + All = 'all' + '''Both committed and uncommitted blocks.''' + + Committed = 'committed' + '''Committed blocks.''' + + Uncommitted = 'uncommitted' + '''Uncommitted blocks.''' + + +class SequenceNumberAction(object): + '''Sequence number actions.''' + + Increment = 'increment' + ''' + Increments the value of the sequence number by 1. If specifying this option, + do not include the x-ms-blob-sequence-number header. + ''' + + Max = 'max' + ''' + Sets the sequence number to be the higher of the value included with the + request and the value currently stored for the blob. + ''' + + Update = 'update' + '''Sets the sequence number to the value included with the request.''' + + +class _LeaseActions(object): + '''Actions for a lease.''' + + Acquire = 'acquire' + '''Acquire the lease.''' + + Break = 'break' + '''Break the lease.''' + + Change = 'change' + '''Change the lease ID.''' + + Release = 'release' + '''Release the lease.''' + + Renew = 'renew' + '''Renew the lease.''' + + +class _BlobTypes(object): + '''Blob type options.''' + + AppendBlob = 'AppendBlob' + '''Append blob type.''' + + BlockBlob = 'BlockBlob' + '''Block blob type.''' + + PageBlob = 'PageBlob' + '''Page blob type.''' + + +class Include(object): + ''' + Specifies the datasets to include in the blob list response. + + :ivar ~azure.storage.blob.models.Include Include.COPY: + Specifies that metadata related to any current or previous Copy Blob operation + should be included in the response. + :ivar ~azure.storage.blob.models.Include Include.METADATA: + Specifies that metadata be returned in the response. + :ivar ~azure.storage.blob.models.Include Include.SNAPSHOTS: + Specifies that snapshots should be included in the enumeration. + :ivar ~azure.storage.blob.models.Include Include.UNCOMMITTED_BLOBS: + Specifies that blobs for which blocks have been uploaded, but which have not + been committed using Put Block List, be included in the response. + :ivar ~azure.storage.blob.models.Include Include.DELETED: + Specifies that deleted blobs should be returned in the response. + ''' + + def __init__(self, snapshots=False, metadata=False, uncommitted_blobs=False, + copy=False, deleted=False, _str=None): + ''' + :param bool snapshots: + Specifies that snapshots should be included in the enumeration. + :param bool metadata: + Specifies that metadata be returned in the response. + :param bool uncommitted_blobs: + Specifies that blobs for which blocks have been uploaded, but which have + not been committed using Put Block List, be included in the response. + :param bool copy: + Specifies that metadata related to any current or previous Copy Blob + operation should be included in the response. + :param bool deleted: + Specifies that deleted blobs should be returned in the response. + :param str _str: + A string representing the includes. + ''' + if not _str: + _str = '' + components = _str.split(',') + self.snapshots = snapshots or ('snapshots' in components) + self.metadata = metadata or ('metadata' in components) + self.uncommitted_blobs = uncommitted_blobs or ('uncommittedblobs' in components) + self.copy = copy or ('copy' in components) + self.deleted = deleted or ('deleted' in components) + + def __or__(self, other): + return Include(_str=str(self) + str(other)) + + def __add__(self, other): + return Include(_str=str(self) + str(other)) + + def __str__(self): + include = (('snapshots,' if self.snapshots else '') + + ('metadata,' if self.metadata else '') + + ('uncommittedblobs,' if self.uncommitted_blobs else '') + + ('copy,' if self.copy else '') + + ('deleted,' if self.deleted else '')) + return include.rstrip(',') + + +Include.COPY = Include(copy=True) +Include.METADATA = Include(metadata=True) +Include.SNAPSHOTS = Include(snapshots=True) +Include.UNCOMMITTED_BLOBS = Include(uncommitted_blobs=True) +Include.DELETED = Include(deleted=True) + + +class BlobPermissions(object): + ''' + BlobPermissions class to be used with + :func:`~azure.storage.blob.baseblobservice.BaseBlobService.generate_blob_shared_access_signature` API. + + :ivar BlobPermissions BlobPermissions.ADD: + Add a block to an append blob. + :ivar BlobPermissions BlobPermissions.CREATE: + Write a new blob, snapshot a blob, or copy a blob to a new blob. + :ivar BlobPermissions BlobPermissions.DELETE: + Delete the blob. + :ivar BlobPermissions BlobPermissions.READ: + Read the content, properties, metadata and block list. Use the blob as the source of a copy operation. + :ivar BlobPermissions BlobPermissions.WRITE: + Create or write content, properties, metadata, or block list. Snapshot or lease + the blob. Resize the blob (page blob only). Use the blob as the destination of a + copy operation within the same account. + ''' + + def __init__(self, read=False, add=False, create=False, write=False, + delete=False, _str=None): + ''' + :param bool read: + Read the content, properties, metadata and block list. Use the blob as + the source of a copy operation. + :param bool add: + Add a block to an append blob. + :param bool create: + Write a new blob, snapshot a blob, or copy a blob to a new blob. + :param bool write: + Create or write content, properties, metadata, or block list. Snapshot + or lease the blob. Resize the blob (page blob only). Use the blob as the + destination of a copy operation within the same account. + :param bool delete: + Delete the blob. + :param str _str: + A string representing the permissions. + ''' + if not _str: + _str = '' + self.read = read or ('r' in _str) + self.add = add or ('a' in _str) + self.create = create or ('c' in _str) + self.write = write or ('w' in _str) + self.delete = delete or ('d' in _str) + + def __or__(self, other): + return BlobPermissions(_str=str(self) + str(other)) + + def __add__(self, other): + return BlobPermissions(_str=str(self) + str(other)) + + def __str__(self): + return (('r' if self.read else '') + + ('a' if self.add else '') + + ('c' if self.create else '') + + ('w' if self.write else '') + + ('d' if self.delete else '')) + + +BlobPermissions.ADD = BlobPermissions(add=True) +BlobPermissions.CREATE = BlobPermissions(create=True) +BlobPermissions.DELETE = BlobPermissions(delete=True) +BlobPermissions.READ = BlobPermissions(read=True) +BlobPermissions.WRITE = BlobPermissions(write=True) + + +class ContainerPermissions(object): + ''' + ContainerPermissions class to be used with :func:`~azure.storage.blob.baseblobservice.BaseBlobService.generate_container_shared_access_signature` + API and for the AccessPolicies used with :func:`~azure.storage.blob.baseblobservice.BaseBlobService.set_container_acl`. + + :ivar ContainerPermissions ContainerPermissions.DELETE: + Delete any blob in the container. Note: You cannot grant permissions to + delete a container with a container SAS. Use an account SAS instead. + :ivar ContainerPermissions ContainerPermissions.LIST: + List blobs in the container. + :ivar ContainerPermissions ContainerPermissions.READ: + Read the content, properties, metadata or block list of any blob in the + container. Use any blob in the container as the source of a copy operation. + :ivar ContainerPermissions ContainerPermissions.WRITE: + For any blob in the container, create or write content, properties, + metadata, or block list. Snapshot or lease the blob. Resize the blob + (page blob only). Use the blob as the destination of a copy operation + within the same account. Note: You cannot grant permissions to read or + write container properties or metadata, nor to lease a container, with + a container SAS. Use an account SAS instead. + ''' + + def __init__(self, read=False, write=False, delete=False, list=False, + _str=None): + ''' + :param bool read: + Read the content, properties, metadata or block list of any blob in the + container. Use any blob in the container as the source of a copy operation. + :param bool write: + For any blob in the container, create or write content, properties, + metadata, or block list. Snapshot or lease the blob. Resize the blob + (page blob only). Use the blob as the destination of a copy operation + within the same account. Note: You cannot grant permissions to read or + write container properties or metadata, nor to lease a container, with + a container SAS. Use an account SAS instead. + :param bool delete: + Delete any blob in the container. Note: You cannot grant permissions to + delete a container with a container SAS. Use an account SAS instead. + :param bool list: + List blobs in the container. + :param str _str: + A string representing the permissions. + ''' + if not _str: + _str = '' + self.read = read or ('r' in _str) + self.write = write or ('w' in _str) + self.delete = delete or ('d' in _str) + self.list = list or ('l' in _str) + + def __or__(self, other): + return ContainerPermissions(_str=str(self) + str(other)) + + def __add__(self, other): + return ContainerPermissions(_str=str(self) + str(other)) + + def __str__(self): + return (('r' if self.read else '') + + ('w' if self.write else '') + + ('d' if self.delete else '') + + ('l' if self.list else '')) + + +ContainerPermissions.DELETE = ContainerPermissions(delete=True) +ContainerPermissions.LIST = ContainerPermissions(list=True) +ContainerPermissions.READ = ContainerPermissions(read=True) +ContainerPermissions.WRITE = ContainerPermissions(write=True) + + +class PremiumPageBlobTier(object): + ''' + Specifies the page blob tier to set the blob to. This is only applicable to page + blobs on premium storage accounts. + Please take a look at https://docs.microsoft.com/en-us/azure/storage/storage-premium-storage#scalability-and-performance-targets + for detailed information on the corresponding IOPS and throughtput per PageBlobTier. + ''' + + P4 = 'P4' + ''' P4 Tier ''' + + P6 = 'P6' + ''' P6 Tier ''' + + P10 = 'P10' + ''' P10 Tier ''' + + P20 = 'P20' + ''' P20 Tier ''' + + P30 = 'P30' + ''' P30 Tier ''' + + P40 = 'P40' + ''' P40 Tier ''' + + P50 = 'P50' + ''' P50 Tier ''' + + P60 = 'P60' + ''' P60 Tier ''' + + +class StandardBlobTier(object): + ''' + Specifies the blob tier to set the blob to. This is only applicable for block blobs on standard storage accounts. + ''' + + Archive = 'Archive' + ''' Archive ''' + + Cool = 'Cool' + ''' Cool ''' + + Hot = 'Hot' + ''' Hot ''' + + +class AccountInformation(object): + """ + Holds information related to the storage account. + + :ivar str sku_name: + Name of the storage SKU, also known as account type. + Example: Standard_LRS, Standard_ZRS, Standard_GRS, Standard_RAGRS, Premium_LRS, Premium_ZRS + :ivar str account_kind: + Describes the flavour of the storage account, also known as account kind. + Example: Storage, StorageV2, BlobStorage + """ + def __init__(self): + self.sku_name = None + self.account_kind = None diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/pageblobservice.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/pageblobservice.py new file mode 100644 index 000000000000..476d55a49071 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/pageblobservice.py @@ -0,0 +1,1394 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import sys +from os import path + + +from ..common._common_conversion import ( + _int_to_str, + _to_str, + _datetime_to_utc_string, + _get_content_md5, +) +from ..common._constants import ( + SERVICE_HOST_BASE, + DEFAULT_PROTOCOL, +) +from ..common._error import ( + _validate_not_none, + _validate_type_bytes, + _validate_encryption_required, + _validate_encryption_unsupported, + _ERROR_VALUE_NEGATIVE, +) +from ..common._http import HTTPRequest +from ..common._serialization import ( + _get_data_bytes_only, + _add_metadata_headers, +) + +from ._deserialization import ( + _convert_xml_to_page_ranges, + _parse_page_properties, + _parse_base_properties, +) +from ._encryption import _generate_blob_encryption_data +from ._error import ( + _ERROR_PAGE_BLOB_SIZE_ALIGNMENT, +) +from ._serialization import ( + _get_path, + _validate_and_format_range_headers, +) +from ._upload_chunking import ( + _PageBlobChunkUploader, + _upload_blob_chunks, +) +from .baseblobservice import BaseBlobService +from .models import ( + _BlobTypes, + ResourceProperties) + +if sys.version_info >= (3,): + from io import BytesIO +else: + from cStringIO import StringIO as BytesIO + +# Keep this value sync with _ERROR_PAGE_BLOB_SIZE_ALIGNMENT +_PAGE_ALIGNMENT = 512 + + +class PageBlobService(BaseBlobService): + ''' + Page blobs are a collection of 512-byte pages optimized for random read and + write operations. To create a page blob, you initialize the page blob and + specify the maximum size the page blob will grow. To add or update the + contents of a page blob, you write a page or pages by specifying an offset + and a range that align to 512-byte page boundaries. A write to a page blob + can overwrite just one page, some pages, or up to 4 MB of the page blob. + Writes to page blobs happen in-place and are immediately committed to the + blob. The maximum size for a page blob is 8 TB. + + :ivar int MAX_PAGE_SIZE: + The size of the pages put by create_blob_from_* methods. Smaller pages + may be put if there is less data provided. The maximum page size the service + supports is 4MB. When using the create_blob_from_* methods, empty pages are skipped. + ''' + + MAX_PAGE_SIZE = 4 * 1024 * 1024 + + def __init__(self, account_name=None, account_key=None, sas_token=None, is_emulated=False, + protocol=DEFAULT_PROTOCOL, endpoint_suffix=SERVICE_HOST_BASE, custom_domain=None, + request_session=None, connection_string=None, socket_timeout=None, token_credential=None): + ''' + :param str account_name: + The storage account name. This is used to authenticate requests + signed with an account key and to construct the storage endpoint. It + is required unless a connection string is given, or if a custom + domain is used with anonymous authentication. + :param str account_key: + The storage account key. This is used for shared key authentication. + If neither account key or sas token is specified, anonymous access + will be used. + :param str sas_token: + A shared access signature token to use to authenticate requests + instead of the account key. If account key and sas token are both + specified, account key will be used to sign. If neither are + specified, anonymous access will be used. + :param bool is_emulated: + Whether to use the emulator. Defaults to False. If specified, will + override all other parameters besides connection string and request + session. + :param str protocol: + The protocol to use for requests. Defaults to https. + :param str endpoint_suffix: + The host base component of the url, minus the account name. Defaults + to Azure (core.windows.net). Override this to use the China cloud + (core.chinacloudapi.cn). + :param str custom_domain: + The custom domain to use. This can be set in the Azure Portal. For + example, 'www.mydomain.com'. + :param requests.Session request_session: + The session object to use for http requests. + :param str connection_string: + If specified, this will override all other parameters besides + request session. See + http://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/ + for the connection string format. + :param int socket_timeout: + If specified, this will override the default socket timeout. The timeout specified is in seconds. + See DEFAULT_SOCKET_TIMEOUT in _constants.py for the default value. + :param token_credential: + A token credential used to authenticate HTTPS requests. The token value + should be updated before its expiration. + :type `~azure.storage.common.TokenCredential` + ''' + self.blob_type = _BlobTypes.PageBlob + super(PageBlobService, self).__init__( + account_name, account_key, sas_token, is_emulated, protocol, endpoint_suffix, + custom_domain, request_session, connection_string, socket_timeout, token_credential) + + def create_blob( + self, container_name, blob_name, content_length, content_settings=None, + sequence_number=None, metadata=None, lease_id=None, if_modified_since=None, + if_unmodified_since=None, if_match=None, if_none_match=None, timeout=None, premium_page_blob_tier=None): + ''' + Creates a new Page Blob. + + See create_blob_from_* for high level functions that handle the + creation and upload of large blobs with automatic chunking and + progress notifications. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of blob to create or update. + :param int content_length: + Required. This header specifies the maximum size + for the page blob, up to 1 TB. The page blob size must be aligned + to a 512-byte boundary. + :param ~azure.storage.blob.models.ContentSettings content_settings: + ContentSettings object used to set properties on the blob. + :param int sequence_number: + The sequence number is a user-controlled value that you can use to + track requests. The value of the sequence number must be between 0 + and 2^63 - 1.The default value is 0. + :param metadata: + Name-value pairs associated with the blob as metadata. + :type metadata: dict(str, str) + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + :param PremiumPageBlobTier premium_page_blob_tier: + A page blob tier value to set the blob to. The tier correlates to the size of the + blob and number of allowed IOPS. This is only applicable to page blobs on + premium storage accounts. + :return: ETag and last modified properties for the new Page Blob + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + _validate_encryption_unsupported(self.require_encryption, self.key_encryption_key) + + return self._create_blob( + container_name, + blob_name, + content_length, + content_settings=content_settings, + sequence_number=sequence_number, + metadata=metadata, + lease_id=lease_id, + premium_page_blob_tier=premium_page_blob_tier, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + if_match=if_match, + if_none_match=if_none_match, + timeout=timeout + ) + + def incremental_copy_blob(self, container_name, blob_name, copy_source, + metadata=None, destination_if_modified_since=None, destination_if_unmodified_since=None, + destination_if_match=None, destination_if_none_match=None, destination_lease_id=None, + source_lease_id=None, timeout=None): + ''' + Copies an incremental copy of a blob asynchronously. This operation returns a copy operation + properties object, including a copy ID you can use to check or abort the + copy operation. The Blob service copies blobs on a best-effort basis. + + The source blob for an incremental copy operation must be a page blob. + Call get_blob_properties on the destination blob to check the status of the copy operation. + The final blob will be committed when the copy completes. + + :param str container_name: + Name of the destination container. The container must exist. + :param str blob_name: + Name of the destination blob. If the destination blob exists, it will + be overwritten. Otherwise, it will be created. + :param str copy_source: + A URL of up to 2 KB in length that specifies an Azure page blob. + The value should be URL-encoded as it would appear in a request URI. + The copy source must be a snapshot and include a valid SAS token or be public. + Example: + https://myaccount.blob.core.windows.net/mycontainer/myblob?snapshot=&sastoken + :param metadata: + Name-value pairs associated with the blob as metadata. If no name-value + pairs are specified, the operation will copy the metadata from the + source blob or file to the destination blob. If one or more name-value + pairs are specified, the destination blob is created with the specified + metadata, and metadata is not copied from the source blob or file. + :type metadata: dict(str, str). + :param datetime destination_if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this conditional header to copy the blob only + if the destination blob has been modified since the specified date/time. + If the destination blob has not been modified, the Blob service returns + status code 412 (Precondition Failed). + :param datetime destination_if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this conditional header to copy the blob only if the destination blob + has not been modified since the specified ate/time. If the destination blob + has been modified, the Blob service returns status code 412 (Precondition Failed). + :param ETag destination_if_match: + An ETag value, or the wildcard character (*). Specify an ETag value for + this conditional header to copy the blob only if the specified ETag value + matches the ETag value for an existing destination blob. If the ETag for + the destination blob does not match the ETag specified for If-Match, the + Blob service returns status code 412 (Precondition Failed). + :param ETag destination_if_none_match: + An ETag value, or the wildcard character (*). Specify an ETag value for + this conditional header to copy the blob only if the specified ETag value + does not match the ETag value for the destination blob. Specify the wildcard + character (*) to perform the operation only if the destination blob does not + exist. If the specified condition isn't met, the Blob service returns status + code 412 (Precondition Failed). + :param str destination_lease_id: + The lease ID specified for this header must match the lease ID of the + destination blob. If the request does not include the lease ID or it is not + valid, the operation fails with status code 412 (Precondition Failed). + :param str source_lease_id: + Specify this to perform the Copy Blob operation only if + the lease ID given matches the active lease ID of the source blob. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: Copy operation properties such as status, source, and ID. + :rtype: :class:`~azure.storage.blob.models.CopyProperties` + ''' + return self._copy_blob(container_name, blob_name, copy_source, + metadata, + source_if_modified_since=None, source_if_unmodified_since=None, + source_if_match=None, source_if_none_match=None, + destination_if_modified_since=destination_if_modified_since, + destination_if_unmodified_since=destination_if_unmodified_since, + destination_if_match=destination_if_match, + destination_if_none_match=destination_if_none_match, + destination_lease_id=destination_lease_id, + source_lease_id=source_lease_id, timeout=timeout, + incremental_copy=True) + + def update_page( + self, container_name, blob_name, page, start_range, end_range, + validate_content=False, lease_id=None, if_sequence_number_lte=None, + if_sequence_number_lt=None, if_sequence_number_eq=None, + if_modified_since=None, if_unmodified_since=None, + if_match=None, if_none_match=None, timeout=None): + ''' + Updates a range of pages. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param bytes page: + Content of the page. + :param int start_range: + Start of byte range to use for writing to a section of the blob. + Pages must be aligned with 512-byte boundaries, the start offset + must be a modulus of 512 and the end offset must be a modulus of + 512-1. Examples of valid byte ranges are 0-511, 512-1023, etc. + :param int end_range: + End of byte range to use for writing to a section of the blob. + Pages must be aligned with 512-byte boundaries, the start offset + must be a modulus of 512 and the end offset must be a modulus of + 512-1. Examples of valid byte ranges are 0-511, 512-1023, etc. + :param bool validate_content: + If true, calculates an MD5 hash of the page content. The storage + service checks the hash of the content that has arrived + with the hash that was sent. This is primarily valuable for detecting + bitflips on the wire if using http instead of https as https (the default) + will already validate. Note that this MD5 hash is not stored with the + blob. + :param str lease_id: + Required if the blob has an active lease. + :param int if_sequence_number_lte: + If the blob's sequence number is less than or equal to + the specified value, the request proceeds; otherwise it fails. + :param int if_sequence_number_lt: + If the blob's sequence number is less than the specified + value, the request proceeds; otherwise it fails. + :param int if_sequence_number_eq: + If the blob's sequence number is equal to the specified + value, the request proceeds; otherwise it fails. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify an ETag value for this conditional + header to write the page only if the blob's ETag value matches the + value specified. If the values do not match, the Blob service fails. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify an ETag value for this conditional + header to write the page only if the blob's ETag value does not + match the value specified. If the values are identical, the Blob + service fails. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: ETag and last modified properties for the updated Page Blob + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + + _validate_encryption_unsupported(self.require_encryption, self.key_encryption_key) + + return self._update_page( + container_name, + blob_name, + page, + start_range, + end_range, + validate_content=validate_content, + lease_id=lease_id, + if_sequence_number_lte=if_sequence_number_lte, + if_sequence_number_lt=if_sequence_number_lt, + if_sequence_number_eq=if_sequence_number_eq, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + if_match=if_match, + if_none_match=if_none_match, + timeout=timeout + ) + + def clear_page( + self, container_name, blob_name, start_range, end_range, + lease_id=None, if_sequence_number_lte=None, + if_sequence_number_lt=None, if_sequence_number_eq=None, + if_modified_since=None, if_unmodified_since=None, + if_match=None, if_none_match=None, timeout=None): + ''' + Clears a range of pages. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param int start_range: + Start of byte range to use for writing to a section of the blob. + Pages must be aligned with 512-byte boundaries, the start offset + must be a modulus of 512 and the end offset must be a modulus of + 512-1. Examples of valid byte ranges are 0-511, 512-1023, etc. + :param int end_range: + End of byte range to use for writing to a section of the blob. + Pages must be aligned with 512-byte boundaries, the start offset + must be a modulus of 512 and the end offset must be a modulus of + 512-1. Examples of valid byte ranges are 0-511, 512-1023, etc. + :param str lease_id: + Required if the blob has an active lease. + :param int if_sequence_number_lte: + If the blob's sequence number is less than or equal to + the specified value, the request proceeds; otherwise it fails. + :param int if_sequence_number_lt: + If the blob's sequence number is less than the specified + value, the request proceeds; otherwise it fails. + :param int if_sequence_number_eq: + If the blob's sequence number is equal to the specified + value, the request proceeds; otherwise it fails. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify an ETag value for this conditional + header to write the page only if the blob's ETag value matches the + value specified. If the values do not match, the Blob service fails. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify an ETag value for this conditional + header to write the page only if the blob's ETag value does not + match the value specified. If the values are identical, the Blob + service fails. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: ETag and last modified properties for the updated Page Blob + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.query = { + 'comp': 'page', + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-page-write': 'clear', + 'x-ms-lease-id': _to_str(lease_id), + 'x-ms-if-sequence-number-le': _to_str(if_sequence_number_lte), + 'x-ms-if-sequence-number-lt': _to_str(if_sequence_number_lt), + 'x-ms-if-sequence-number-eq': _to_str(if_sequence_number_eq), + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'If-Match': _to_str(if_match), + 'If-None-Match': _to_str(if_none_match) + } + _validate_and_format_range_headers( + request, + start_range, + end_range, + align_to_page=True) + + return self._perform_request(request, _parse_page_properties) + + def get_page_ranges( + self, container_name, blob_name, snapshot=None, start_range=None, + end_range=None, lease_id=None, if_modified_since=None, + if_unmodified_since=None, if_match=None, if_none_match=None, timeout=None): + ''' + Returns the list of valid page ranges for a Page Blob or snapshot + of a page blob. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param str snapshot: + The snapshot parameter is an opaque DateTime value that, + when present, specifies the blob snapshot to retrieve information + from. + :param int start_range: + Start of byte range to use for getting valid page ranges. + If no end_range is given, all bytes after the start_range will be searched. + Pages must be aligned with 512-byte boundaries, the start offset + must be a modulus of 512 and the end offset must be a modulus of + 512-1. Examples of valid byte ranges are 0-511, 512-, etc. + :param int end_range: + End of byte range to use for getting valid page ranges. + If end_range is given, start_range must be provided. + This range will return valid page ranges for from the offset start up to + offset end. + Pages must be aligned with 512-byte boundaries, the start offset + must be a modulus of 512 and the end offset must be a modulus of + 512-1. Examples of valid byte ranges are 0-511, 512-, etc. + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: A list of valid Page Ranges for the Page Blob. + :rtype: list(:class:`~azure.storage.blob.models.PageRange`) + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + request = HTTPRequest() + request.method = 'GET' + request.host_locations = self._get_host_locations(secondary=True) + request.path = _get_path(container_name, blob_name) + request.query = { + 'comp': 'pagelist', + 'snapshot': _to_str(snapshot), + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-lease-id': _to_str(lease_id), + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'If-Match': _to_str(if_match), + 'If-None-Match': _to_str(if_none_match), + } + if start_range is not None: + _validate_and_format_range_headers( + request, + start_range, + end_range, + start_range_required=False, + end_range_required=False, + align_to_page=True) + + return self._perform_request(request, _convert_xml_to_page_ranges) + + def get_page_ranges_diff( + self, container_name, blob_name, previous_snapshot, snapshot=None, + start_range=None, end_range=None, lease_id=None, if_modified_since=None, + if_unmodified_since=None, if_match=None, if_none_match=None, timeout=None): + ''' + The response will include only the pages that are different between either a + recent snapshot or the current blob and a previous snapshot, including pages + that were cleared. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param str previous_snapshot: + The snapshot parameter is an opaque DateTime value that + specifies a previous blob snapshot to be compared + against a more recent snapshot or the current blob. + :param str snapshot: + The snapshot parameter is an opaque DateTime value that + specifies a more recent blob snapshot to be compared + against a previous snapshot (previous_snapshot). + :param int start_range: + Start of byte range to use for getting different page ranges. + If no end_range is given, all bytes after the start_range will be searched. + Pages must be aligned with 512-byte boundaries, the start offset + must be a modulus of 512 and the end offset must be a modulus of + 512-1. Examples of valid byte ranges are 0-511, 512-, etc. + :param int end_range: + End of byte range to use for getting different page ranges. + If end_range is given, start_range must be provided. + This range will return valid page ranges for from the offset start up to + offset end. + Pages must be aligned with 512-byte boundaries, the start offset + must be a modulus of 512 and the end offset must be a modulus of + 512-1. Examples of valid byte ranges are 0-511, 512-, etc. + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: A list of different Page Ranges for the Page Blob. + :rtype: list(:class:`~azure.storage.blob.models.PageRange`) + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('previous_snapshot', previous_snapshot) + request = HTTPRequest() + request.method = 'GET' + request.host_locations = self._get_host_locations(secondary=True) + request.path = _get_path(container_name, blob_name) + request.query = { + 'comp': 'pagelist', + 'snapshot': _to_str(snapshot), + 'prevsnapshot': _to_str(previous_snapshot), + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-lease-id': _to_str(lease_id), + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'If-Match': _to_str(if_match), + 'If-None-Match': _to_str(if_none_match), + } + if start_range is not None: + _validate_and_format_range_headers( + request, + start_range, + end_range, + start_range_required=False, + end_range_required=False, + align_to_page=True) + + return self._perform_request(request, _convert_xml_to_page_ranges) + + def set_sequence_number( + self, container_name, blob_name, sequence_number_action, sequence_number=None, + lease_id=None, if_modified_since=None, if_unmodified_since=None, + if_match=None, if_none_match=None, timeout=None): + + ''' + Sets the blob sequence number. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param str sequence_number_action: + This property indicates how the service should modify the blob's sequence + number. See :class:`~azure.storage.blob.models.SequenceNumberAction` for more information. + :param str sequence_number: + This property sets the blob's sequence number. The sequence number is a + user-controlled property that you can use to track requests and manage + concurrency issues. + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: ETag and last modified properties for the updated Page Blob + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('sequence_number_action', sequence_number_action) + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.query = { + 'comp': 'properties', + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-blob-sequence-number': _to_str(sequence_number), + 'x-ms-sequence-number-action': _to_str(sequence_number_action), + 'x-ms-lease-id': _to_str(lease_id), + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'If-Match': _to_str(if_match), + 'If-None-Match': _to_str(if_none_match), + } + + return self._perform_request(request, _parse_page_properties) + + def resize_blob( + self, container_name, blob_name, content_length, + lease_id=None, if_modified_since=None, if_unmodified_since=None, + if_match=None, if_none_match=None, timeout=None): + + ''' + Resizes a page blob to the specified size. If the specified value is less + than the current size of the blob, then all pages above the specified value + are cleared. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of existing blob. + :param int content_length: + Size to resize blob to. + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: ETag and last modified properties for the updated Page Blob + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('content_length', content_length) + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.query = { + 'comp': 'properties', + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-blob-content-length': _to_str(content_length), + 'x-ms-lease-id': _to_str(lease_id), + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'If-Match': _to_str(if_match), + 'If-None-Match': _to_str(if_none_match), + } + + return self._perform_request(request, _parse_page_properties) + + # ----Convenience APIs----------------------------------------------------- + + def create_blob_from_path( + self, container_name, blob_name, file_path, content_settings=None, + metadata=None, validate_content=False, progress_callback=None, max_connections=2, + lease_id=None, if_modified_since=None, if_unmodified_since=None, + if_match=None, if_none_match=None, timeout=None, premium_page_blob_tier=None): + ''' + Creates a new blob from a file path, or updates the content of an + existing blob, with automatic chunking and progress notifications. + Empty chunks are skipped, while non-emtpy ones(even if only partly filled) are uploaded. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of blob to create or update. + :param str file_path: + Path of the file to upload as the blob content. + :param ~azure.storage.blob.models.ContentSettings content_settings: + ContentSettings object used to set blob properties. + :param metadata: + Name-value pairs associated with the blob as metadata. + :type metadata: dict(str, str) + :param bool validate_content: + If true, calculates an MD5 hash for each page of the blob. The storage + service checks the hash of the content that has arrived with the hash + that was sent. This is primarily valuable for detecting bitflips on + the wire if using http instead of https as https (the default) will + already validate. Note that this MD5 hash is not stored with the + blob. + :param progress_callback: + Callback for progress with signature function(current, total) where + current is the number of bytes transfered so far, and total is the + size of the blob, or None if the total size is unknown. + :type progress_callback: func(current, total) + :param int max_connections: + Maximum number of parallel connections to use. + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. This method may make + multiple calls to the Azure service and the timeout will apply to + each call individually. + :param premium_page_blob_tier: + A page blob tier value to set the blob to. The tier correlates to the size of the + blob and number of allowed IOPS. This is only applicable to page blobs on + premium storage accounts. + :return: ETag and last modified properties for the Page Blob + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('file_path', file_path) + + count = path.getsize(file_path) + with open(file_path, 'rb') as stream: + return self.create_blob_from_stream( + container_name=container_name, + blob_name=blob_name, + stream=stream, + count=count, + content_settings=content_settings, + metadata=metadata, + validate_content=validate_content, + progress_callback=progress_callback, + max_connections=max_connections, + lease_id=lease_id, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + if_match=if_match, + if_none_match=if_none_match, + timeout=timeout, + premium_page_blob_tier=premium_page_blob_tier) + + def create_blob_from_stream( + self, container_name, blob_name, stream, count, content_settings=None, + metadata=None, validate_content=False, progress_callback=None, + max_connections=2, lease_id=None, if_modified_since=None, + if_unmodified_since=None, if_match=None, if_none_match=None, timeout=None, + premium_page_blob_tier=None): + ''' + Creates a new blob from a file/stream, or updates the content of an + existing blob, with automatic chunking and progress notifications. + Empty chunks are skipped, while non-emtpy ones(even if only partly filled) are uploaded. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of blob to create or update. + :param io.IOBase stream: + Opened file/stream to upload as the blob content. + :param int count: + Number of bytes to read from the stream. This is required, a page + blob cannot be created if the count is unknown. + :param ~azure.storage.blob.models.ContentSettings content_settings: + ContentSettings object used to set the blob properties. + :param metadata: + Name-value pairs associated with the blob as metadata. + :type metadata: dict(str, str) + :param bool validate_content: + If true, calculates an MD5 hash for each page of the blob. The storage + service checks the hash of the content that has arrived with the hash + that was sent. This is primarily valuable for detecting bitflips on + the wire if using http instead of https as https (the default) will + already validate. Note that this MD5 hash is not stored with the + blob. + :param progress_callback: + Callback for progress with signature function(current, total) where + current is the number of bytes transfered so far, and total is the + size of the blob, or None if the total size is unknown. + :type progress_callback: func(current, total) + :param int max_connections: + Maximum number of parallel connections to use. Note that parallel upload + requires the stream to be seekable. + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. This method may make + multiple calls to the Azure service and the timeout will apply to + each call individually. + :param premium_page_blob_tier: + A page blob tier value to set the blob to. The tier correlates to the size of the + blob and number of allowed IOPS. This is only applicable to page blobs on + premium storage accounts. + :return: ETag and last modified properties for the Page Blob + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('stream', stream) + _validate_not_none('count', count) + _validate_encryption_required(self.require_encryption, self.key_encryption_key) + + if count < 0: + raise ValueError(_ERROR_VALUE_NEGATIVE.format('count')) + + if count % _PAGE_ALIGNMENT != 0: + raise ValueError(_ERROR_PAGE_BLOB_SIZE_ALIGNMENT.format(count)) + + cek, iv, encryption_data = None, None, None + if self.key_encryption_key is not None: + cek, iv, encryption_data = _generate_blob_encryption_data(self.key_encryption_key) + + response = self._create_blob( + container_name=container_name, + blob_name=blob_name, + content_length=count, + content_settings=content_settings, + metadata=metadata, + lease_id=lease_id, + premium_page_blob_tier=premium_page_blob_tier, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + if_match=if_match, + if_none_match=if_none_match, + timeout=timeout, + encryption_data=encryption_data + ) + + if count == 0: + return response + + # _upload_blob_chunks returns the block ids for block blobs so resource_properties + # is passed as a parameter to get the last_modified and etag for page and append blobs. + # this info is not needed for block_blobs since _put_block_list is called after which gets this info + resource_properties = ResourceProperties() + _upload_blob_chunks( + blob_service=self, + container_name=container_name, + blob_name=blob_name, + blob_size=count, + block_size=self.MAX_PAGE_SIZE, + stream=stream, + max_connections=max_connections, + progress_callback=progress_callback, + validate_content=validate_content, + lease_id=lease_id, + uploader_class=_PageBlobChunkUploader, + if_match=response.etag, + timeout=timeout, + content_encryption_key=cek, + initialization_vector=iv, + resource_properties=resource_properties + ) + + return resource_properties + + def create_blob_from_bytes( + self, container_name, blob_name, blob, index=0, count=None, + content_settings=None, metadata=None, validate_content=False, + progress_callback=None, max_connections=2, lease_id=None, + if_modified_since=None, if_unmodified_since=None, if_match=None, + if_none_match=None, timeout=None, premium_page_blob_tier=None): + ''' + Creates a new blob from an array of bytes, or updates the content + of an existing blob, with automatic chunking and progress + notifications. Empty chunks are skipped, while non-emtpy ones(even if only partly filled) are uploaded. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of blob to create or update. + :param bytes blob: + Content of blob as an array of bytes. + :param int index: + Start index in the byte array. + :param int count: + Number of bytes to upload. Set to None or negative value to upload + all bytes starting from index. + :param ~azure.storage.blob.models.ContentSettings content_settings: + ContentSettings object used to set blob properties. + :param metadata: + Name-value pairs associated with the blob as metadata. + :type metadata: dict(str, str) + :param bool validate_content: + If true, calculates an MD5 hash for each page of the blob. The storage + service checks the hash of the content that has arrived with the hash + that was sent. This is primarily valuable for detecting bitflips on + the wire if using http instead of https as https (the default) will + already validate. Note that this MD5 hash is not stored with the + blob. + :param progress_callback: + Callback for progress with signature function(current, total) where + current is the number of bytes transfered so far, and total is the + size of the blob, or None if the total size is unknown. + :type progress_callback: func(current, total) + :param int max_connections: + Maximum number of parallel connections to use. + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. This method may make + multiple calls to the Azure service and the timeout will apply to + each call individually. + :param premium_page_blob_tier: + A page blob tier value to set the blob to. The tier correlates to the size of the + blob and number of allowed IOPS. This is only applicable to page blobs on + premium storage accounts. + :return: ETag and last modified properties for the Page Blob + :rtype: :class:`~azure.storage.blob.models.ResourceProperties` + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('blob', blob) + _validate_type_bytes('blob', blob) + + if index < 0: + raise IndexError(_ERROR_VALUE_NEGATIVE.format('index')) + + if count is None or count < 0: + count = len(blob) - index + + stream = BytesIO(blob) + stream.seek(index) + + return self.create_blob_from_stream( + container_name=container_name, + blob_name=blob_name, + stream=stream, + count=count, + content_settings=content_settings, + metadata=metadata, + validate_content=validate_content, + lease_id=lease_id, + progress_callback=progress_callback, + max_connections=max_connections, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + if_match=if_match, + if_none_match=if_none_match, + timeout=timeout, + premium_page_blob_tier=premium_page_blob_tier) + + def set_premium_page_blob_tier( + self, container_name, blob_name, premium_page_blob_tier, + timeout=None): + ''' + Sets the page blob tiers on the blob. This API is only supported for page blobs on premium accounts. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of blob to update. + :param PremiumPageBlobTier premium_page_blob_tier: + A page blob tier value to set the blob to. The tier correlates to the size of the + blob and number of allowed IOPS. This is only applicable to page blobs on + premium storage accounts. + :param int timeout: + The timeout parameter is expressed in seconds. This method may make + multiple calls to the Azure service and the timeout will apply to + each call individually. + ''' + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('premium_page_blob_tier', premium_page_blob_tier) + + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.query = { + 'comp': 'tier', + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-access-tier': _to_str(premium_page_blob_tier) + } + + self._perform_request(request) + + def copy_blob(self, container_name, blob_name, copy_source, + metadata=None, + source_if_modified_since=None, + source_if_unmodified_since=None, + source_if_match=None, source_if_none_match=None, + destination_if_modified_since=None, + destination_if_unmodified_since=None, + destination_if_match=None, + destination_if_none_match=None, + destination_lease_id=None, + source_lease_id=None, timeout=None, + premium_page_blob_tier=None): + ''' + Copies a blob asynchronously. This operation returns a copy operation + properties object, including a copy ID you can use to check or abort the + copy operation. The Blob service copies blobs on a best-effort basis. + + The source blob for a copy operation must be a page blob. If the destination + blob already exists, it must be of the same blob type as the source blob. + Any existing destination blob will be overwritten. + The destination blob cannot be modified while a copy operation is in progress. + + When copying from a page blob, the Blob service creates a destination page + blob of the source blob's length, initially containing all zeroes. Then + the source page ranges are enumerated, and non-empty ranges are copied. + + If the tier on the source blob is larger than the tier being passed to this + copy operation or if the size of the blob exceeds the tier being passed to + this copy operation then the operation will fail. + + You can call get_blob_properties on the destination + blob to check the status of the copy operation. The final blob will be + committed when the copy completes. + + :param str container_name: + Name of the destination container. The container must exist. + :param str blob_name: + Name of the destination blob. If the destination blob exists, it will + be overwritten. Otherwise, it will be created. + :param str copy_source: + A URL of up to 2 KB in length that specifies an Azure file or blob. + The value should be URL-encoded as it would appear in a request URI. + If the source is in another account, the source must either be public + or must be authenticated via a shared access signature. If the source + is public, no authentication is required. + Examples: + https://myaccount.blob.core.windows.net/mycontainer/myblob + https://myaccount.blob.core.windows.net/mycontainer/myblob?snapshot= + https://otheraccount.blob.core.windows.net/mycontainer/myblob?sastoken + :param metadata: + Name-value pairs associated with the blob as metadata. If no name-value + pairs are specified, the operation will copy the metadata from the + source blob or file to the destination blob. If one or more name-value + pairs are specified, the destination blob is created with the specified + metadata, and metadata is not copied from the source blob or file. + :type metadata: dict(str, str). + :param datetime source_if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this conditional header to copy the blob only if the source + blob has been modified since the specified date/time. + :param datetime source_if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this conditional header to copy the blob only if the source blob + has not been modified since the specified date/time. + :param ETag source_if_match: + An ETag value, or the wildcard character (*). Specify this conditional + header to copy the source blob only if its ETag matches the value + specified. If the ETag values do not match, the Blob service returns + status code 412 (Precondition Failed). This header cannot be specified + if the source is an Azure File. + :param ETag source_if_none_match: + An ETag value, or the wildcard character (*). Specify this conditional + header to copy the blob only if its ETag does not match the value + specified. If the values are identical, the Blob service returns status + code 412 (Precondition Failed). This header cannot be specified if the + source is an Azure File. + :param datetime destination_if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this conditional header to copy the blob only + if the destination blob has been modified since the specified date/time. + If the destination blob has not been modified, the Blob service returns + status code 412 (Precondition Failed). + :param datetime destination_if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this conditional header to copy the blob only + if the destination blob has not been modified since the specified + date/time. If the destination blob has been modified, the Blob service + returns status code 412 (Precondition Failed). + :param ETag destination_if_match: + An ETag value, or the wildcard character (*). Specify an ETag value for + this conditional header to copy the blob only if the specified ETag value + matches the ETag value for an existing destination blob. If the ETag for + the destination blob does not match the ETag specified for If-Match, the + Blob service returns status code 412 (Precondition Failed). + :param ETag destination_if_none_match: + An ETag value, or the wildcard character (*). Specify an ETag value for + this conditional header to copy the blob only if the specified ETag value + does not match the ETag value for the destination blob. Specify the wildcard + character (*) to perform the operation only if the destination blob does not + exist. If the specified condition isn't met, the Blob service returns status + code 412 (Precondition Failed). + :param str destination_lease_id: + The lease ID specified for this header must match the lease ID of the + destination blob. If the request does not include the lease ID or it is not + valid, the operation fails with status code 412 (Precondition Failed). + :param str source_lease_id: + Specify this to perform the Copy Blob operation only if + the lease ID given matches the active lease ID of the source blob. + :param int timeout: + The timeout parameter is expressed in seconds. + :param PageBlobTier premium_page_blob_tier: + A page blob tier value to set on the destination blob. The tier correlates to + the size of the blob and number of allowed IOPS. This is only applicable to + page blobs on premium storage accounts. + If the tier on the source blob is larger than the tier being passed to this + copy operation or if the size of the blob exceeds the tier being passed to + this copy operation then the operation will fail. + :return: Copy operation properties such as status, source, and ID. + :rtype: :class:`~azure.storage.blob.models.CopyProperties` + ''' + return self._copy_blob(container_name, blob_name, copy_source, + metadata, premium_page_blob_tier, + source_if_modified_since, source_if_unmodified_since, + source_if_match, source_if_none_match, + destination_if_modified_since, + destination_if_unmodified_since, + destination_if_match, + destination_if_none_match, + destination_lease_id, + source_lease_id, timeout, + False) + + # -----Helper methods----------------------------------------------------- + + def _create_blob( + self, container_name, blob_name, content_length, content_settings=None, + sequence_number=None, metadata=None, lease_id=None, premium_page_blob_tier=None, if_modified_since=None, + if_unmodified_since=None, if_match=None, if_none_match=None, timeout=None, + encryption_data=None): + ''' + See create_blob for more details. This helper method + allows for encryption or other such special behavior because + it is safely handled by the library. These behaviors are + prohibited in the public version of this function. + :param str encryption_data: + The JSON formatted encryption metadata to upload as a part of the blob. + This should only be passed internally from other methods and only applied + when uploading entire blob contents immediately follows creation of the blob. + ''' + + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('content_length', content_length) + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.query = {'timeout': _int_to_str(timeout)} + request.headers = { + 'x-ms-blob-type': _to_str(self.blob_type), + 'x-ms-blob-content-length': _to_str(content_length), + 'x-ms-lease-id': _to_str(lease_id), + 'x-ms-blob-sequence-number': _to_str(sequence_number), + 'x-ms-access-tier': _to_str(premium_page_blob_tier), + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'If-Match': _to_str(if_match), + 'If-None-Match': _to_str(if_none_match) + } + _add_metadata_headers(metadata, request) + if content_settings is not None: + request.headers.update(content_settings._to_headers()) + + if encryption_data is not None: + request.headers['x-ms-meta-encryptiondata'] = encryption_data + + return self._perform_request(request, _parse_base_properties) + + def _update_page( + self, container_name, blob_name, page, start_range, end_range, + validate_content=False, lease_id=None, if_sequence_number_lte=None, + if_sequence_number_lt=None, if_sequence_number_eq=None, + if_modified_since=None, if_unmodified_since=None, + if_match=None, if_none_match=None, timeout=None): + ''' + See update_page for more details. This helper method + allows for encryption or other such special behavior because + it is safely handled by the library. These behaviors are + prohibited in the public version of this function. + ''' + + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.query = { + 'comp': 'page', + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-page-write': 'update', + 'x-ms-lease-id': _to_str(lease_id), + 'x-ms-if-sequence-number-le': _to_str(if_sequence_number_lte), + 'x-ms-if-sequence-number-lt': _to_str(if_sequence_number_lt), + 'x-ms-if-sequence-number-eq': _to_str(if_sequence_number_eq), + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'If-Match': _to_str(if_match), + 'If-None-Match': _to_str(if_none_match) + } + _validate_and_format_range_headers( + request, + start_range, + end_range, + align_to_page=True) + request.body = _get_data_bytes_only('page', page) + + if validate_content: + computed_md5 = _get_content_md5(request.body) + request.headers['Content-MD5'] = _to_str(computed_md5) + + return self._perform_request(request, _parse_page_properties) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/sharedaccesssignature.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/sharedaccesssignature.py new file mode 100644 index 000000000000..6947e7e10ff7 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/sharedaccesssignature.py @@ -0,0 +1,180 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +from ..common.sharedaccesssignature import ( + SharedAccessSignature, + _SharedAccessHelper, +) + +from ._constants import X_MS_VERSION + + +class BlobSharedAccessSignature(SharedAccessSignature): + ''' + Provides a factory for creating blob and container access + signature tokens with a common account name and account key. Users can either + use the factory or can construct the appropriate service and use the + generate_*_shared_access_signature method directly. + ''' + + def __init__(self, account_name, account_key): + ''' + :param str account_name: + The storage account name used to generate the shared access signatures. + :param str account_key: + The access key to generate the shares access signatures. + ''' + super(BlobSharedAccessSignature, self).__init__(account_name, account_key, x_ms_version=X_MS_VERSION) + + def generate_blob(self, container_name, blob_name, permission=None, + expiry=None, start=None, id=None, ip=None, protocol=None, + cache_control=None, content_disposition=None, + content_encoding=None, content_language=None, + content_type=None): + ''' + Generates a shared access signature for the blob. + Use the returned signature with the sas_token parameter of any BlobService. + + :param str container_name: + Name of container. + :param str blob_name: + Name of blob. + :param BlobPermissions permission: + The permissions associated with the shared access signature. The + user is restricted to operations allowed by the permissions. + Permissions must be ordered read, write, delete, list. + Required unless an id is given referencing a stored access policy + which contains this field. This field must be omitted if it has been + specified in an associated stored access policy. + :param expiry: + The time at which the shared access signature becomes invalid. + Required unless an id is given referencing a stored access policy + which contains this field. This field must be omitted if it has + been specified in an associated stored access policy. Azure will always + convert values to UTC. If a date is passed in without timezone info, it + is assumed to be UTC. + :type expiry: datetime or str + :param start: + The time at which the shared access signature becomes valid. If + omitted, start time for this call is assumed to be the time when the + storage service receives the request. Azure will always convert values + to UTC. If a date is passed in without timezone info, it is assumed to + be UTC. + :type start: datetime or str + :param str id: + A unique value up to 64 characters in length that correlates to a + stored access policy. To create a stored access policy, use + set_blob_service_properties. + :param str ip: + Specifies an IP address or a range of IP addresses from which to accept requests. + If the IP address from which the request originates does not match the IP address + or address range specified on the SAS token, the request is not authenticated. + For example, specifying sip=168.1.5.65 or sip=168.1.5.60-168.1.5.70 on the SAS + restricts the request to those IP addresses. + :param str protocol: + Specifies the protocol permitted for a request made. The default value + is https,http. See :class:`~azure.storage.common.models.Protocol` for possible values. + :param str cache_control: + Response header value for Cache-Control when resource is accessed + using this shared access signature. + :param str content_disposition: + Response header value for Content-Disposition when resource is accessed + using this shared access signature. + :param str content_encoding: + Response header value for Content-Encoding when resource is accessed + using this shared access signature. + :param str content_language: + Response header value for Content-Language when resource is accessed + using this shared access signature. + :param str content_type: + Response header value for Content-Type when resource is accessed + using this shared access signature. + ''' + resource_path = container_name + '/' + blob_name + + sas = _SharedAccessHelper() + sas.add_base(permission, expiry, start, ip, protocol, self.x_ms_version) + sas.add_id(id) + sas.add_resource('b') + sas.add_override_response_headers(cache_control, content_disposition, + content_encoding, content_language, + content_type) + sas.add_resource_signature(self.account_name, self.account_key, 'blob', resource_path) + + return sas.get_token() + + def generate_container(self, container_name, permission=None, expiry=None, + start=None, id=None, ip=None, protocol=None, + cache_control=None, content_disposition=None, + content_encoding=None, content_language=None, + content_type=None): + ''' + Generates a shared access signature for the container. + Use the returned signature with the sas_token parameter of any BlobService. + + :param str container_name: + Name of container. + :param ContainerPermissions permission: + The permissions associated with the shared access signature. The + user is restricted to operations allowed by the permissions. + Permissions must be ordered read, write, delete, list. + Required unless an id is given referencing a stored access policy + which contains this field. This field must be omitted if it has been + specified in an associated stored access policy. + :param expiry: + The time at which the shared access signature becomes invalid. + Required unless an id is given referencing a stored access policy + which contains this field. This field must be omitted if it has + been specified in an associated stored access policy. Azure will always + convert values to UTC. If a date is passed in without timezone info, it + is assumed to be UTC. + :type expiry: datetime or str + :param start: + The time at which the shared access signature becomes valid. If + omitted, start time for this call is assumed to be the time when the + storage service receives the request. Azure will always convert values + to UTC. If a date is passed in without timezone info, it is assumed to + be UTC. + :type start: datetime or str + :param str id: + A unique value up to 64 characters in length that correlates to a + stored access policy. To create a stored access policy, use + set_blob_service_properties. + :param str ip: + Specifies an IP address or a range of IP addresses from which to accept requests. + If the IP address from which the request originates does not match the IP address + or address range specified on the SAS token, the request is not authenticated. + For example, specifying sip=168.1.5.65 or sip=168.1.5.60-168.1.5.70 on the SAS + restricts the request to those IP addresses. + :param str protocol: + Specifies the protocol permitted for a request made. The default value + is https,http. See :class:`~azure.storage.common.models.Protocol` for possible values. + :param str cache_control: + Response header value for Cache-Control when resource is accessed + using this shared access signature. + :param str content_disposition: + Response header value for Content-Disposition when resource is accessed + using this shared access signature. + :param str content_encoding: + Response header value for Content-Encoding when resource is accessed + using this shared access signature. + :param str content_language: + Response header value for Content-Language when resource is accessed + using this shared access signature. + :param str content_type: + Response header value for Content-Type when resource is accessed + using this shared access signature. + ''' + sas = _SharedAccessHelper() + sas.add_base(permission, expiry, start, ip, protocol, self.x_ms_version) + sas.add_id(id) + sas.add_resource('c') + sas.add_override_response_headers(cache_control, content_disposition, + content_encoding, content_language, + content_type) + sas.add_resource_signature(self.account_name, self.account_key, 'blob', container_name) + + return sas.get_token() diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/__init__.py new file mode 100644 index 000000000000..797c97069ee1 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/__init__.py @@ -0,0 +1,38 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from ._constants import ( + __author__, + __version__, + DEFAULT_X_MS_VERSION, +) +from .cloudstorageaccount import CloudStorageAccount +from .models import ( + RetentionPolicy, + Logging, + Metrics, + CorsRule, + DeleteRetentionPolicy, + StaticWebsite, + ServiceProperties, + AccessPolicy, + ResourceTypes, + Services, + AccountPermissions, + Protocol, + ServiceStats, + GeoReplication, + LocationMode, + RetryContext, +) +from .retry import ( + ExponentialRetry, + LinearRetry, + no_retry, +) +from .sharedaccesssignature import ( + SharedAccessSignature, +) +from .tokencredential import TokenCredential diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_auth.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_auth.py new file mode 100644 index 000000000000..15c15b9ea560 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_auth.py @@ -0,0 +1,117 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from ._common_conversion import ( + _sign_string, +) +from ._constants import ( + DEV_ACCOUNT_NAME, + DEV_ACCOUNT_SECONDARY_NAME +) + +import logging +logger = logging.getLogger(__name__) + + +class _StorageSharedKeyAuthentication(object): + def __init__(self, account_name, account_key, is_emulated=False): + self.account_name = account_name + self.account_key = account_key + self.is_emulated = is_emulated + + def _get_headers(self, request, headers_to_sign): + headers = dict((name.lower(), value) for name, value in request.headers.items() if value) + if 'content-length' in headers and headers['content-length'] == '0': + del headers['content-length'] + return '\n'.join(headers.get(x, '') for x in headers_to_sign) + '\n' + + def _get_verb(self, request): + return request.method + '\n' + + def _get_canonicalized_resource(self, request): + uri_path = request.path.split('?')[0] + + # for emulator, use the DEV_ACCOUNT_NAME instead of DEV_ACCOUNT_SECONDARY_NAME + # as this is how the emulator works + if self.is_emulated and uri_path.find(DEV_ACCOUNT_SECONDARY_NAME) == 1: + # only replace the first instance + uri_path = uri_path.replace(DEV_ACCOUNT_SECONDARY_NAME, DEV_ACCOUNT_NAME, 1) + + return '/' + self.account_name + uri_path + + def _get_canonicalized_headers(self, request): + string_to_sign = '' + x_ms_headers = [] + for name, value in request.headers.items(): + if name.startswith('x-ms-'): + x_ms_headers.append((name.lower(), value)) + x_ms_headers.sort() + for name, value in x_ms_headers: + if value is not None: + string_to_sign += ''.join([name, ':', value, '\n']) + return string_to_sign + + def _add_authorization_header(self, request, string_to_sign): + signature = _sign_string(self.account_key, string_to_sign) + auth_string = 'SharedKey ' + self.account_name + ':' + signature + request.headers['Authorization'] = auth_string + + +class _StorageSharedKeyAuthentication(_StorageSharedKeyAuthentication): + def sign_request(self, request): + string_to_sign = \ + self._get_verb(request) + \ + self._get_headers( + request, + [ + 'content-encoding', 'content-language', 'content-length', + 'content-md5', 'content-type', 'date', 'if-modified-since', + 'if-match', 'if-none-match', 'if-unmodified-since', 'byte_range' + ] + ) + \ + self._get_canonicalized_headers(request) + \ + self._get_canonicalized_resource(request) + \ + self._get_canonicalized_resource_query(request) + + self._add_authorization_header(request, string_to_sign) + logger.debug("String_to_sign=%s", string_to_sign) + + def _get_canonicalized_resource_query(self, request): + sorted_queries = [(name, value) for name, value in request.query.items()] + sorted_queries.sort() + + string_to_sign = '' + for name, value in sorted_queries: + if value is not None: + string_to_sign += '\n' + name.lower() + ':' + value + + return string_to_sign + + +class _StorageNoAuthentication(object): + def sign_request(self, request): + pass + + +class _StorageSASAuthentication(object): + def __init__(self, sas_token): + # ignore ?-prefix (added by tools such as Azure Portal) on sas tokens + # doing so avoids double question marks when signing + if sas_token[0] == '?': + self.sas_token = sas_token[1:] + else: + self.sas_token = sas_token + + def sign_request(self, request): + # if 'sig=' is present, then the request has already been signed + # as is the case when performing retries + if 'sig=' in request.path: + return + if '?' in request.path: + request.path += '&' + else: + request.path += '?' + + request.path += self.sas_token diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_common_conversion.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_common_conversion.py new file mode 100644 index 000000000000..8b50afbe1afb --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_common_conversion.py @@ -0,0 +1,126 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +import base64 +import hashlib +import hmac +import sys +from io import (SEEK_SET) + +from dateutil.tz import tzutc + +from ._error import ( + _ERROR_VALUE_SHOULD_BE_BYTES_OR_STREAM, + _ERROR_VALUE_SHOULD_BE_SEEKABLE_STREAM, +) +from .models import ( + _unicode_type, +) + +if sys.version_info < (3,): + def _str(value): + if isinstance(value, unicode): + return value.encode('utf-8') + + return str(value) +else: + _str = str + + +def _to_str(value): + return _str(value) if value is not None else None + + +def _int_to_str(value): + return str(int(value)) if value is not None else None + + +def _bool_to_str(value): + if value is None: + return None + + if isinstance(value, bool): + if value: + return 'true' + else: + return 'false' + + return str(value) + + +def _to_utc_datetime(value): + return value.strftime('%Y-%m-%dT%H:%M:%SZ') + + +def _datetime_to_utc_string(value): + # Azure expects the date value passed in to be UTC. + # Azure will always return values as UTC. + # If a date is passed in without timezone info, it is assumed to be UTC. + if value is None: + return None + + if value.tzinfo: + value = value.astimezone(tzutc()) + + return value.strftime('%a, %d %b %Y %H:%M:%S GMT') + + +def _encode_base64(data): + if isinstance(data, _unicode_type): + data = data.encode('utf-8') + encoded = base64.b64encode(data) + return encoded.decode('utf-8') + + +def _decode_base64_to_bytes(data): + if isinstance(data, _unicode_type): + data = data.encode('utf-8') + return base64.b64decode(data) + + +def _decode_base64_to_text(data): + decoded_bytes = _decode_base64_to_bytes(data) + return decoded_bytes.decode('utf-8') + + +def _sign_string(key, string_to_sign, key_is_base64=True): + if key_is_base64: + key = _decode_base64_to_bytes(key) + else: + if isinstance(key, _unicode_type): + key = key.encode('utf-8') + if isinstance(string_to_sign, _unicode_type): + string_to_sign = string_to_sign.encode('utf-8') + signed_hmac_sha256 = hmac.HMAC(key, string_to_sign, hashlib.sha256) + digest = signed_hmac_sha256.digest() + encoded_digest = _encode_base64(digest) + return encoded_digest + + +def _get_content_md5(data): + md5 = hashlib.md5() + if isinstance(data, bytes): + md5.update(data) + elif hasattr(data, 'read'): + pos = 0 + try: + pos = data.tell() + except: + pass + for chunk in iter(lambda: data.read(4096), b""): + md5.update(chunk) + try: + data.seek(pos, SEEK_SET) + except (AttributeError, IOError): + raise ValueError(_ERROR_VALUE_SHOULD_BE_SEEKABLE_STREAM.format('data')) + else: + raise ValueError(_ERROR_VALUE_SHOULD_BE_BYTES_OR_STREAM.format('data')) + + return base64.b64encode(md5.digest()).decode('utf-8') + + +def _lower(text): + return text.lower() diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_connection.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_connection.py new file mode 100644 index 000000000000..1388fddeb625 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_connection.py @@ -0,0 +1,160 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import sys + +if sys.version_info >= (3,): + from urllib.parse import urlparse +else: + from urlparse import urlparse + +from ._constants import ( + SERVICE_HOST_BASE, + DEFAULT_PROTOCOL, + DEV_ACCOUNT_NAME, + DEV_ACCOUNT_SECONDARY_NAME, + DEV_ACCOUNT_KEY, + DEV_BLOB_HOST, + DEV_QUEUE_HOST, +) +from ._error import ( + _ERROR_STORAGE_MISSING_INFO, +) + +_EMULATOR_ENDPOINTS = { + 'blob': DEV_BLOB_HOST, + 'queue': DEV_QUEUE_HOST, + 'file': '', +} + +_CONNECTION_ENDPOINTS = { + 'blob': 'BlobEndpoint', + 'queue': 'QueueEndpoint', + 'file': 'FileEndpoint', +} + +_CONNECTION_ENDPOINTS_SECONDARY = { + 'blob': 'BlobSecondaryEndpoint', + 'queue': 'QueueSecondaryEndpoint', + 'file': 'FileSecondaryEndpoint', +} + + +class _ServiceParameters(object): + def __init__(self, service, account_name=None, account_key=None, sas_token=None, token_credential=None, + is_emulated=False, protocol=DEFAULT_PROTOCOL, endpoint_suffix=SERVICE_HOST_BASE, + custom_domain=None, custom_domain_secondary=None): + + self.account_name = account_name + self.account_key = account_key + self.sas_token = sas_token + self.token_credential = token_credential + self.protocol = protocol or DEFAULT_PROTOCOL + self.is_emulated = is_emulated + + if is_emulated: + self.account_name = DEV_ACCOUNT_NAME + self.protocol = 'http' + + # Only set the account key if a sas_token is not present to allow sas to be used with the emulator + self.account_key = DEV_ACCOUNT_KEY if not self.sas_token else None + + self.primary_endpoint = '{}/{}'.format(_EMULATOR_ENDPOINTS[service], DEV_ACCOUNT_NAME) + self.secondary_endpoint = '{}/{}'.format(_EMULATOR_ENDPOINTS[service], DEV_ACCOUNT_SECONDARY_NAME) + else: + # Strip whitespace from the key + if self.account_key: + self.account_key = self.account_key.strip() + + endpoint_suffix = endpoint_suffix or SERVICE_HOST_BASE + + # Setup the primary endpoint + if custom_domain: + parsed_url = urlparse(custom_domain) + + # Trim any trailing slashes from the path + path = parsed_url.path.rstrip('/') + + self.primary_endpoint = parsed_url.netloc + path + self.protocol = self.protocol if parsed_url.scheme is '' else parsed_url.scheme + else: + if not self.account_name: + raise ValueError(_ERROR_STORAGE_MISSING_INFO) + self.primary_endpoint = '{}.{}.{}'.format(self.account_name, service, endpoint_suffix) + + # Setup the secondary endpoint + if custom_domain_secondary: + if not custom_domain: + raise ValueError(_ERROR_STORAGE_MISSING_INFO) + + parsed_url = urlparse(custom_domain_secondary) + + # Trim any trailing slashes from the path + path = parsed_url.path.rstrip('/') + + self.secondary_endpoint = parsed_url.netloc + path + else: + if self.account_name: + self.secondary_endpoint = '{}-secondary.{}.{}'.format(self.account_name, service, endpoint_suffix) + else: + self.secondary_endpoint = None + + @staticmethod + def get_service_parameters(service, account_name=None, account_key=None, sas_token=None, token_credential= None, + is_emulated=None, protocol=None, endpoint_suffix=None, custom_domain=None, + request_session=None, connection_string=None, socket_timeout=None): + if connection_string: + params = _ServiceParameters._from_connection_string(connection_string, service) + elif is_emulated: + params = _ServiceParameters(service, is_emulated=True) + elif account_name: + if protocol.lower() != 'https' and token_credential is not None: + raise ValueError("Token credential is only supported with HTTPS.") + params = _ServiceParameters(service, + account_name=account_name, + account_key=account_key, + sas_token=sas_token, + token_credential=token_credential, + is_emulated=is_emulated, + protocol=protocol, + endpoint_suffix=endpoint_suffix, + custom_domain=custom_domain) + else: + raise ValueError(_ERROR_STORAGE_MISSING_INFO) + + params.request_session = request_session + params.socket_timeout = socket_timeout + return params + + @staticmethod + def _from_connection_string(connection_string, service): + # Split into key=value pairs removing empties, then split the pairs into a dict + config = dict(s.split('=', 1) for s in connection_string.split(';') if s) + + # Authentication + account_name = config.get('AccountName') + account_key = config.get('AccountKey') + sas_token = config.get('SharedAccessSignature') + + # Emulator + is_emulated = config.get('UseDevelopmentStorage') + + # Basic URL Configuration + protocol = config.get('DefaultEndpointsProtocol') + endpoint_suffix = config.get('EndpointSuffix') + + # Custom URLs + endpoint = config.get(_CONNECTION_ENDPOINTS[service]) + endpoint_secondary = config.get(_CONNECTION_ENDPOINTS_SECONDARY[service]) + + return _ServiceParameters(service, + account_name=account_name, + account_key=account_key, + sas_token=sas_token, + is_emulated=is_emulated, + protocol=protocol, + endpoint_suffix=endpoint_suffix, + custom_domain=endpoint, + custom_domain_secondary=endpoint_secondary) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_constants.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_constants.py new file mode 100644 index 000000000000..22516d640757 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_constants.py @@ -0,0 +1,47 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import platform +import sys + +__author__ = 'Microsoft Corp. ' +__version__ = '1.3.0' + +# UserAgent string sample: 'Azure-Storage/0.37.0-0.38.0 (Python CPython 3.4.2; Windows 8)' +# First version(0.37.0) is the common package, and the second version(0.38.0) is the service package +USER_AGENT_STRING_PREFIX = 'Azure-Storage/{}-'.format(__version__) +USER_AGENT_STRING_SUFFIX = '(Python {} {}; {} {})'.format(platform.python_implementation(), + platform.python_version(), platform.system(), + platform.release()) + +# default values for common package, in case it is used directly +DEFAULT_X_MS_VERSION = '2018-03-28' +DEFAULT_USER_AGENT_STRING = '{}None {}'.format(USER_AGENT_STRING_PREFIX, USER_AGENT_STRING_SUFFIX) + +# Live ServiceClient URLs +SERVICE_HOST_BASE = 'core.windows.net' +DEFAULT_PROTOCOL = 'https' + +# Development ServiceClient URLs +DEV_BLOB_HOST = '127.0.0.1:10000' +DEV_QUEUE_HOST = '127.0.0.1:10001' + +# Default credentials for Development Storage Service +DEV_ACCOUNT_NAME = 'devstoreaccount1' +DEV_ACCOUNT_SECONDARY_NAME = 'devstoreaccount1-secondary' +DEV_ACCOUNT_KEY = 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==' + +# Socket timeout in seconds +DEFAULT_SOCKET_TIMEOUT = 20 + +# for python 3.5+, there was a change to the definition of the socket timeout (as far as socket.sendall is concerned) +# The socket timeout is now the maximum total duration to send all data. +if sys.version_info >= (3, 5): + # the timeout to connect is 20 seconds, and the read timeout is 2000 seconds + # the 2000 seconds was calculated with: 100MB (max block size)/ 50KB/s (an arbitrarily chosen minimum upload speed) + DEFAULT_SOCKET_TIMEOUT = (20, 2000) + +# Encryption constants +_ENCRYPTION_PROTOCOL_V1 = '1.0' diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_deserialization.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_deserialization.py new file mode 100644 index 000000000000..80803da3e438 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_deserialization.py @@ -0,0 +1,384 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from dateutil import parser + +from ._common_conversion import _to_str + +try: + from xml.etree import cElementTree as ETree +except ImportError: + from xml.etree import ElementTree as ETree + +from .models import ( + ServiceProperties, + Logging, + Metrics, + CorsRule, + AccessPolicy, + _dict, + GeoReplication, + ServiceStats, + DeleteRetentionPolicy, + StaticWebsite, +) + + +def _to_int(value): + return value if value is None else int(value) + + +def _bool(value): + return value.lower() == 'true' + + +def _to_upper_str(value): + return _to_str(value).upper() if value is not None else None + + +def _get_download_size(start_range, end_range, resource_size): + if start_range is not None: + end_range = end_range if end_range else (resource_size if resource_size else None) + if end_range is not None: + return end_range - start_range + else: + return None + else: + return resource_size + + +GET_PROPERTIES_ATTRIBUTE_MAP = { + 'last-modified': (None, 'last_modified', parser.parse), + 'etag': (None, 'etag', _to_str), + 'x-ms-blob-type': (None, 'blob_type', _to_str), + 'content-length': (None, 'content_length', _to_int), + 'content-range': (None, 'content_range', _to_str), + 'x-ms-blob-sequence-number': (None, 'page_blob_sequence_number', _to_int), + 'x-ms-blob-committed-block-count': (None, 'append_blob_committed_block_count', _to_int), + 'x-ms-blob-public-access': (None, 'public_access', _to_str), + 'x-ms-access-tier': (None, 'blob_tier', _to_str), + 'x-ms-access-tier-change-time': (None, 'blob_tier_change_time', parser.parse), + 'x-ms-access-tier-inferred': (None, 'blob_tier_inferred', _bool), + 'x-ms-archive-status': (None, 'rehydration_status', _to_str), + 'x-ms-share-quota': (None, 'quota', _to_int), + 'x-ms-server-encrypted': (None, 'server_encrypted', _bool), + 'x-ms-creation-time': (None, 'creation_time', parser.parse), + 'content-type': ('content_settings', 'content_type', _to_str), + 'cache-control': ('content_settings', 'cache_control', _to_str), + 'content-encoding': ('content_settings', 'content_encoding', _to_str), + 'content-disposition': ('content_settings', 'content_disposition', _to_str), + 'content-language': ('content_settings', 'content_language', _to_str), + 'content-md5': ('content_settings', 'content_md5', _to_str), + 'x-ms-lease-status': ('lease', 'status', _to_str), + 'x-ms-lease-state': ('lease', 'state', _to_str), + 'x-ms-lease-duration': ('lease', 'duration', _to_str), + 'x-ms-copy-id': ('copy', 'id', _to_str), + 'x-ms-copy-source': ('copy', 'source', _to_str), + 'x-ms-copy-status': ('copy', 'status', _to_str), + 'x-ms-copy-progress': ('copy', 'progress', _to_str), + 'x-ms-copy-completion-time': ('copy', 'completion_time', parser.parse), + 'x-ms-copy-destination-snapshot': ('copy', 'destination_snapshot_time', _to_str), + 'x-ms-copy-status-description': ('copy', 'status_description', _to_str), + 'x-ms-has-immutability-policy': (None, 'has_immutability_policy', _bool), + 'x-ms-has-legal-hold': (None, 'has_legal_hold', _bool), +} + + +def _parse_metadata(response): + ''' + Extracts out resource metadata information. + ''' + + if response is None or response.headers is None: + return None + + metadata = _dict() + for key, value in response.headers.items(): + if key.lower().startswith('x-ms-meta-'): + metadata[key[10:]] = _to_str(value) + + return metadata + + +def _parse_properties(response, result_class): + ''' + Extracts out resource properties and metadata information. + Ignores the standard http headers. + ''' + + if response is None or response.headers is None: + return None + + props = result_class() + for key, value in response.headers.items(): + info = GET_PROPERTIES_ATTRIBUTE_MAP.get(key) + if info: + if info[0] is None: + setattr(props, info[1], info[2](value)) + else: + attr = getattr(props, info[0]) + setattr(attr, info[1], info[2](value)) + + if hasattr(props, 'blob_type') and props.blob_type == 'PageBlob' and hasattr(props, 'blob_tier') and props.blob_tier is not None: + props.blob_tier = _to_upper_str(props.blob_tier) + return props + + +def _parse_length_from_content_range(content_range): + ''' + Parses the blob length from the content range header: bytes 1-3/65537 + ''' + if content_range is None: + return None + + # First, split in space and take the second half: '1-3/65537' + # Next, split on slash and take the second half: '65537' + # Finally, convert to an int: 65537 + return int(content_range.split(' ', 1)[1].split('/', 1)[1]) + + +def _convert_xml_to_signed_identifiers(response): + ''' + + + + unique-value + + start-time + expiry-time + abbreviated-permission-list + + + + ''' + if response is None or response.body is None: + return None + + list_element = ETree.fromstring(response.body) + signed_identifiers = _dict() + + for signed_identifier_element in list_element.findall('SignedIdentifier'): + # Id element + id = signed_identifier_element.find('Id').text + + # Access policy element + access_policy = AccessPolicy() + access_policy_element = signed_identifier_element.find('AccessPolicy') + if access_policy_element is not None: + start_element = access_policy_element.find('Start') + if start_element is not None: + access_policy.start = parser.parse(start_element.text) + + expiry_element = access_policy_element.find('Expiry') + if expiry_element is not None: + access_policy.expiry = parser.parse(expiry_element.text) + + access_policy.permission = access_policy_element.findtext('Permission') + + signed_identifiers[id] = access_policy + + return signed_identifiers + + +def _convert_xml_to_service_stats(response): + ''' + + + + live|bootstrap|unavailable + sync-time| + + + ''' + if response is None or response.body is None: + return None + + service_stats_element = ETree.fromstring(response.body) + + geo_replication_element = service_stats_element.find('GeoReplication') + + geo_replication = GeoReplication() + geo_replication.status = geo_replication_element.find('Status').text + last_sync_time = geo_replication_element.find('LastSyncTime').text + geo_replication.last_sync_time = parser.parse(last_sync_time) if last_sync_time else None + + service_stats = ServiceStats() + service_stats.geo_replication = geo_replication + return service_stats + + +def _convert_xml_to_service_properties(response): + ''' + + + + version-number + true|false + true|false + true|false + + true|false + number-of-days + + + + version-number + true|false + true|false + + true|false + number-of-days + + + + version-number + true|false + true|false + + true|false + number-of-days + + + + + comma-separated-list-of-allowed-origins + comma-separated-list-of-HTTP-verb + max-caching-age-in-seconds + comma-seperated-list-of-response-headers + comma-seperated-list-of-request-headers + + + + true|false + number-of-days + + + true|false + + + + + ''' + if response is None or response.body is None: + return None + + service_properties_element = ETree.fromstring(response.body) + service_properties = ServiceProperties() + + # Logging + logging = service_properties_element.find('Logging') + if logging is not None: + service_properties.logging = Logging() + service_properties.logging.version = logging.find('Version').text + service_properties.logging.delete = _bool(logging.find('Delete').text) + service_properties.logging.read = _bool(logging.find('Read').text) + service_properties.logging.write = _bool(logging.find('Write').text) + + _convert_xml_to_retention_policy(logging.find('RetentionPolicy'), + service_properties.logging.retention_policy) + # HourMetrics + hour_metrics_element = service_properties_element.find('HourMetrics') + if hour_metrics_element is not None: + service_properties.hour_metrics = Metrics() + _convert_xml_to_metrics(hour_metrics_element, service_properties.hour_metrics) + + # MinuteMetrics + minute_metrics_element = service_properties_element.find('MinuteMetrics') + if minute_metrics_element is not None: + service_properties.minute_metrics = Metrics() + _convert_xml_to_metrics(minute_metrics_element, service_properties.minute_metrics) + + # CORS + cors = service_properties_element.find('Cors') + if cors is not None: + service_properties.cors = list() + for rule in cors.findall('CorsRule'): + allowed_origins = rule.find('AllowedOrigins').text.split(',') + + allowed_methods = rule.find('AllowedMethods').text.split(',') + + max_age_in_seconds = int(rule.find('MaxAgeInSeconds').text) + + cors_rule = CorsRule(allowed_origins, allowed_methods, max_age_in_seconds) + + exposed_headers = rule.find('ExposedHeaders').text + if exposed_headers is not None: + cors_rule.exposed_headers = exposed_headers.split(',') + + allowed_headers = rule.find('AllowedHeaders').text + if allowed_headers is not None: + cors_rule.allowed_headers = allowed_headers.split(',') + + service_properties.cors.append(cors_rule) + + # Target version + target_version = service_properties_element.find('DefaultServiceVersion') + if target_version is not None: + service_properties.target_version = target_version.text + + # DeleteRetentionPolicy + delete_retention_policy_element = service_properties_element.find('DeleteRetentionPolicy') + if delete_retention_policy_element is not None: + service_properties.delete_retention_policy = DeleteRetentionPolicy() + policy_enabled = _bool(delete_retention_policy_element.find('Enabled').text) + service_properties.delete_retention_policy.enabled = policy_enabled + + if policy_enabled: + service_properties.delete_retention_policy.days = int(delete_retention_policy_element.find('Days').text) + + # StaticWebsite + static_website_element = service_properties_element.find('StaticWebsite') + if static_website_element is not None: + service_properties.static_website = StaticWebsite() + service_properties.static_website.enabled = _bool(static_website_element.find('Enabled').text) + + index_document_element = static_website_element.find('IndexDocument') + if index_document_element is not None: + service_properties.static_website.index_document = index_document_element.text + + error_document_element = static_website_element.find('ErrorDocument404Path') + if error_document_element is not None: + service_properties.static_website.error_document_404_path = error_document_element.text + + return service_properties + + +def _convert_xml_to_metrics(xml, metrics): + ''' + version-number + true|false + true|false + + true|false + number-of-days + + ''' + # Version + metrics.version = xml.find('Version').text + + # Enabled + metrics.enabled = _bool(xml.find('Enabled').text) + + # IncludeAPIs + include_apis_element = xml.find('IncludeAPIs') + if include_apis_element is not None: + metrics.include_apis = _bool(include_apis_element.text) + + # RetentionPolicy + _convert_xml_to_retention_policy(xml.find('RetentionPolicy'), metrics.retention_policy) + + +def _convert_xml_to_retention_policy(xml, retention_policy): + ''' + true|false + number-of-days + ''' + # Enabled + retention_policy.enabled = _bool(xml.find('Enabled').text) + + # Days + days_element = xml.find('Days') + if days_element is not None: + retention_policy.days = int(days_element.text) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_encryption.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_encryption.py new file mode 100644 index 000000000000..cd7d92e66e0e --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_encryption.py @@ -0,0 +1,233 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from collections import OrderedDict + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.ciphers import Cipher +from cryptography.hazmat.primitives.ciphers.algorithms import AES +from cryptography.hazmat.primitives.ciphers.modes import CBC + +from ._common_conversion import ( + _encode_base64, + _decode_base64_to_bytes, +) +from ._constants import ( + _ENCRYPTION_PROTOCOL_V1, + __version__, +) +from ._error import ( + _ERROR_UNSUPPORTED_ENCRYPTION_VERSION, + _validate_not_none, + _validate_encryption_protocol_version, + _validate_key_encryption_key_unwrap, + _validate_kek_id, +) + + +class _EncryptionAlgorithm(object): + ''' + Specifies which client encryption algorithm is used. + ''' + AES_CBC_256 = 'AES_CBC_256' + + +class _WrappedContentKey: + ''' + Represents the envelope key details stored on the service. + ''' + + def __init__(self, algorithm, encrypted_key, key_id): + ''' + :param str algorithm: + The algorithm used for wrapping. + :param bytes encrypted_key: + The encrypted content-encryption-key. + :param str key_id: + The key-encryption-key identifier string. + ''' + + _validate_not_none('algorithm', algorithm) + _validate_not_none('encrypted_key', encrypted_key) + _validate_not_none('key_id', key_id) + + self.algorithm = algorithm + self.encrypted_key = encrypted_key + self.key_id = key_id + + +class _EncryptionAgent: + ''' + Represents the encryption agent stored on the service. + It consists of the encryption protocol version and encryption algorithm used. + ''' + + def __init__(self, encryption_algorithm, protocol): + ''' + :param _EncryptionAlgorithm encryption_algorithm: + The algorithm used for encrypting the message contents. + :param str protocol: + The protocol version used for encryption. + ''' + + _validate_not_none('encryption_algorithm', encryption_algorithm) + _validate_not_none('protocol', protocol) + + self.encryption_algorithm = str(encryption_algorithm) + self.protocol = protocol + + +class _EncryptionData: + ''' + Represents the encryption data that is stored on the service. + ''' + + def __init__(self, content_encryption_IV, encryption_agent, wrapped_content_key, + key_wrapping_metadata): + ''' + :param bytes content_encryption_IV: + The content encryption initialization vector. + :param _EncryptionAgent encryption_agent: + The encryption agent. + :param _WrappedContentKey wrapped_content_key: + An object that stores the wrapping algorithm, the key identifier, + and the encrypted key bytes. + :param dict key_wrapping_metadata: + A dict containing metadata related to the key wrapping. + ''' + + _validate_not_none('content_encryption_IV', content_encryption_IV) + _validate_not_none('encryption_agent', encryption_agent) + _validate_not_none('wrapped_content_key', wrapped_content_key) + + self.content_encryption_IV = content_encryption_IV + self.encryption_agent = encryption_agent + self.wrapped_content_key = wrapped_content_key + self.key_wrapping_metadata = key_wrapping_metadata + + +def _generate_encryption_data_dict(kek, cek, iv): + ''' + Generates and returns the encryption metadata as a dict. + + :param object kek: The key encryption key. See calling functions for more information. + :param bytes cek: The content encryption key. + :param bytes iv: The initialization vector. + :return: A dict containing all the encryption metadata. + :rtype: dict + ''' + # Encrypt the cek. + wrapped_cek = kek.wrap_key(cek) + + # Build the encryption_data dict. + # Use OrderedDict to comply with Java's ordering requirement. + wrapped_content_key = OrderedDict() + wrapped_content_key['KeyId'] = kek.get_kid() + wrapped_content_key['EncryptedKey'] = _encode_base64(wrapped_cek) + wrapped_content_key['Algorithm'] = kek.get_key_wrap_algorithm() + + encryption_agent = OrderedDict() + encryption_agent['Protocol'] = _ENCRYPTION_PROTOCOL_V1 + encryption_agent['EncryptionAlgorithm'] = _EncryptionAlgorithm.AES_CBC_256 + + encryption_data_dict = OrderedDict() + encryption_data_dict['WrappedContentKey'] = wrapped_content_key + encryption_data_dict['EncryptionAgent'] = encryption_agent + encryption_data_dict['ContentEncryptionIV'] = _encode_base64(iv) + encryption_data_dict['KeyWrappingMetadata'] = {'EncryptionLibrary': 'Python ' + __version__} + + return encryption_data_dict + + +def _dict_to_encryption_data(encryption_data_dict): + ''' + Converts the specified dictionary to an EncryptionData object for + eventual use in decryption. + + :param dict encryption_data_dict: + The dictionary containing the encryption data. + :return: an _EncryptionData object built from the dictionary. + :rtype: _EncryptionData + ''' + try: + if encryption_data_dict['EncryptionAgent']['Protocol'] != _ENCRYPTION_PROTOCOL_V1: + raise ValueError(_ERROR_UNSUPPORTED_ENCRYPTION_VERSION) + except KeyError: + raise ValueError(_ERROR_UNSUPPORTED_ENCRYPTION_VERSION) + wrapped_content_key = encryption_data_dict['WrappedContentKey'] + wrapped_content_key = _WrappedContentKey(wrapped_content_key['Algorithm'], + _decode_base64_to_bytes(wrapped_content_key['EncryptedKey']), + wrapped_content_key['KeyId']) + + encryption_agent = encryption_data_dict['EncryptionAgent'] + encryption_agent = _EncryptionAgent(encryption_agent['EncryptionAlgorithm'], + encryption_agent['Protocol']) + + if 'KeyWrappingMetadata' in encryption_data_dict: + key_wrapping_metadata = encryption_data_dict['KeyWrappingMetadata'] + else: + key_wrapping_metadata = None + + encryption_data = _EncryptionData(_decode_base64_to_bytes(encryption_data_dict['ContentEncryptionIV']), + encryption_agent, + wrapped_content_key, + key_wrapping_metadata) + + return encryption_data + + +def _generate_AES_CBC_cipher(cek, iv): + ''' + Generates and returns an encryption cipher for AES CBC using the given cek and iv. + + :param bytes[] cek: The content encryption key for the cipher. + :param bytes[] iv: The initialization vector for the cipher. + :return: A cipher for encrypting in AES256 CBC. + :rtype: ~cryptography.hazmat.primitives.ciphers.Cipher + ''' + + backend = default_backend() + algorithm = AES(cek) + mode = CBC(iv) + return Cipher(algorithm, mode, backend) + + +def _validate_and_unwrap_cek(encryption_data, key_encryption_key=None, key_resolver=None): + ''' + Extracts and returns the content_encryption_key stored in the encryption_data object + and performs necessary validation on all parameters. + :param _EncryptionData encryption_data: + The encryption metadata of the retrieved value. + :param obj key_encryption_key: + The key_encryption_key used to unwrap the cek. Please refer to high-level service object + instance variables for more details. + :param func key_resolver: + A function used that, given a key_id, will return a key_encryption_key. Please refer + to high-level service object instance variables for more details. + :return: the content_encryption_key stored in the encryption_data object. + :rtype: bytes[] + ''' + + _validate_not_none('content_encryption_IV', encryption_data.content_encryption_IV) + _validate_not_none('encrypted_key', encryption_data.wrapped_content_key.encrypted_key) + + _validate_encryption_protocol_version(encryption_data.encryption_agent.protocol) + + content_encryption_key = None + + # If the resolver exists, give priority to the key it finds. + if key_resolver is not None: + key_encryption_key = key_resolver(encryption_data.wrapped_content_key.key_id) + + _validate_not_none('key_encryption_key', key_encryption_key) + _validate_key_encryption_key_unwrap(key_encryption_key) + _validate_kek_id(encryption_data.wrapped_content_key.key_id, key_encryption_key.get_kid()) + + # Will throw an exception if the specified algorithm is not supported. + content_encryption_key = key_encryption_key.unwrap_key(encryption_data.wrapped_content_key.encrypted_key, + encryption_data.wrapped_content_key.algorithm) + _validate_not_none('content_encryption_key', content_encryption_key) + + return content_encryption_key diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_error.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_error.py new file mode 100644 index 000000000000..90faa0124ab2 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_error.py @@ -0,0 +1,183 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from sys import version_info + +if version_info < (3,): + def _str(value): + if isinstance(value, unicode): + return value.encode('utf-8') + + return str(value) +else: + _str = str + + +def _to_str(value): + return _str(value) if value is not None else None + + +from azure.common import ( + AzureHttpError, + AzureConflictHttpError, + AzureMissingResourceHttpError, + AzureException, +) +from ._constants import ( + _ENCRYPTION_PROTOCOL_V1, +) + +_ERROR_CONFLICT = 'Conflict ({0})' +_ERROR_NOT_FOUND = 'Not found ({0})' +_ERROR_UNKNOWN = 'Unknown error ({0})' +_ERROR_STORAGE_MISSING_INFO = \ + 'You need to provide an account name and either an account_key or sas_token when creating a storage service.' +_ERROR_EMULATOR_DOES_NOT_SUPPORT_FILES = \ + 'The emulator does not support the file service.' +_ERROR_ACCESS_POLICY = \ + 'share_access_policy must be either SignedIdentifier or AccessPolicy ' + \ + 'instance' +_ERROR_PARALLEL_NOT_SEEKABLE = 'Parallel operations require a seekable stream.' +_ERROR_VALUE_SHOULD_BE_BYTES = '{0} should be of type bytes.' +_ERROR_VALUE_SHOULD_BE_BYTES_OR_STREAM = '{0} should be of type bytes or a readable file-like/io.IOBase stream object.' +_ERROR_VALUE_SHOULD_BE_SEEKABLE_STREAM = '{0} should be a seekable file-like/io.IOBase type stream object.' +_ERROR_VALUE_SHOULD_BE_STREAM = '{0} should be a file-like/io.IOBase type stream object with a read method.' +_ERROR_VALUE_NONE = '{0} should not be None.' +_ERROR_VALUE_NONE_OR_EMPTY = '{0} should not be None or empty.' +_ERROR_VALUE_NEGATIVE = '{0} should not be negative.' +_ERROR_START_END_NEEDED_FOR_MD5 = \ + 'Both end_range and start_range need to be specified ' + \ + 'for getting content MD5.' +_ERROR_RANGE_TOO_LARGE_FOR_MD5 = \ + 'Getting content MD5 for a range greater than 4MB ' + \ + 'is not supported.' +_ERROR_MD5_MISMATCH = \ + 'MD5 mismatch. Expected value is \'{0}\', computed value is \'{1}\'.' +_ERROR_TOO_MANY_ACCESS_POLICIES = \ + 'Too many access policies provided. The server does not support setting more than 5 access policies on a single resource.' +_ERROR_OBJECT_INVALID = \ + '{0} does not define a complete interface. Value of {1} is either missing or invalid.' +_ERROR_UNSUPPORTED_ENCRYPTION_VERSION = \ + 'Encryption version is not supported.' +_ERROR_DECRYPTION_FAILURE = \ + 'Decryption failed' +_ERROR_ENCRYPTION_REQUIRED = \ + 'Encryption required but no key was provided.' +_ERROR_DECRYPTION_REQUIRED = \ + 'Decryption required but neither key nor resolver was provided.' + \ + ' If you do not want to decypt, please do not set the require encryption flag.' +_ERROR_INVALID_KID = \ + 'Provided or resolved key-encryption-key does not match the id of key used to encrypt.' +_ERROR_UNSUPPORTED_ENCRYPTION_ALGORITHM = \ + 'Specified encryption algorithm is not supported.' +_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION = 'The require_encryption flag is set, but encryption is not supported' + \ + ' for this method.' +_ERROR_UNKNOWN_KEY_WRAP_ALGORITHM = 'Unknown key wrap algorithm.' +_ERROR_DATA_NOT_ENCRYPTED = 'Encryption required, but received data does not contain appropriate metatadata.' + \ + 'Data was either not encrypted or metadata has been lost.' + + +def _dont_fail_on_exist(error): + ''' don't throw exception if the resource exists. + This is called by create_* APIs with fail_on_exist=False''' + if isinstance(error, AzureConflictHttpError): + return False + else: + raise error + + +def _dont_fail_not_exist(error): + ''' don't throw exception if the resource doesn't exist. + This is called by create_* APIs with fail_on_exist=False''' + if isinstance(error, AzureMissingResourceHttpError): + return False + else: + raise error + + +def _http_error_handler(http_error): + ''' Simple error handler for azure.''' + message = str(http_error) + error_code = None + + if 'x-ms-error-code' in http_error.respheader: + error_code = http_error.respheader['x-ms-error-code'] + message += ' ErrorCode: ' + error_code + + if http_error.respbody is not None: + message += '\n' + http_error.respbody.decode('utf-8-sig') + + ex = AzureHttpError(message, http_error.status) + ex.error_code = error_code + + raise ex + + +def _validate_type_bytes(param_name, param): + if not isinstance(param, bytes): + raise TypeError(_ERROR_VALUE_SHOULD_BE_BYTES.format(param_name)) + + +def _validate_type_bytes_or_stream(param_name, param): + if not (isinstance(param, bytes) or hasattr(param, 'read')): + raise TypeError(_ERROR_VALUE_SHOULD_BE_BYTES_OR_STREAM.format(param_name)) + + +def _validate_not_none(param_name, param): + if param is None: + raise ValueError(_ERROR_VALUE_NONE.format(param_name)) + + +def _validate_content_match(server_md5, computed_md5): + if server_md5 != computed_md5: + raise AzureException(_ERROR_MD5_MISMATCH.format(server_md5, computed_md5)) + + +def _validate_access_policies(identifiers): + if identifiers and len(identifiers) > 5: + raise AzureException(_ERROR_TOO_MANY_ACCESS_POLICIES) + + +def _validate_key_encryption_key_wrap(kek): + # Note that None is not callable and so will fail the second clause of each check. + if not hasattr(kek, 'wrap_key') or not callable(kek.wrap_key): + raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'wrap_key')) + if not hasattr(kek, 'get_kid') or not callable(kek.get_kid): + raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'get_kid')) + if not hasattr(kek, 'get_key_wrap_algorithm') or not callable(kek.get_key_wrap_algorithm): + raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'get_key_wrap_algorithm')) + + +def _validate_key_encryption_key_unwrap(kek): + if not hasattr(kek, 'get_kid') or not callable(kek.get_kid): + raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'get_kid')) + if not hasattr(kek, 'unwrap_key') or not callable(kek.unwrap_key): + raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'unwrap_key')) + + +def _validate_encryption_required(require_encryption, kek): + if require_encryption and (kek is None): + raise ValueError(_ERROR_ENCRYPTION_REQUIRED) + + +def _validate_decryption_required(require_encryption, kek, resolver): + if (require_encryption and (kek is None) and + (resolver is None)): + raise ValueError(_ERROR_DECRYPTION_REQUIRED) + + +def _validate_encryption_protocol_version(encryption_protocol): + if not (_ENCRYPTION_PROTOCOL_V1 == encryption_protocol): + raise ValueError(_ERROR_UNSUPPORTED_ENCRYPTION_VERSION) + + +def _validate_kek_id(kid, resolved_id): + if not (kid == resolved_id): + raise ValueError(_ERROR_INVALID_KID) + + +def _validate_encryption_unsupported(require_encryption, key_encryption_key): + if require_encryption or (key_encryption_key is not None): + raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_http/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_http/__init__.py new file mode 100644 index 000000000000..2990ec80abe0 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_http/__init__.py @@ -0,0 +1,74 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + + +class HTTPError(Exception): + ''' + Represents an HTTP Exception when response status code >= 300. + + :ivar int status: + the status code of the response + :ivar str message: + the message + :ivar list headers: + the returned headers, as a list of (name, value) pairs + :ivar bytes body: + the body of the response + ''' + + def __init__(self, status, message, respheader, respbody): + self.status = status + self.respheader = respheader + self.respbody = respbody + Exception.__init__(self, message) + + +class HTTPResponse(object): + ''' + Represents a response from an HTTP request. + + :ivar int status: + the status code of the response + :ivar str message: + the message + :ivar dict headers: + the returned headers + :ivar bytes body: + the body of the response + ''' + + def __init__(self, status, message, headers, body): + self.status = status + self.message = message + self.headers = headers + self.body = body + + +class HTTPRequest(object): + ''' + Represents an HTTP Request. + + :ivar str host: + the host name to connect to + :ivar str method: + the method to use to connect (string such as GET, POST, PUT, etc.) + :ivar str path: + the uri fragment + :ivar dict query: + query parameters + :ivar dict headers: + header values + :ivar bytes body: + the body of the request. + ''' + + def __init__(self): + self.host = '' + self.method = '' + self.path = '' + self.query = {} # list of (name, value) + self.headers = {} # list of (header name, header value) + self.body = '' diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_http/httpclient.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_http/httpclient.py new file mode 100644 index 000000000000..b5847660e296 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_http/httpclient.py @@ -0,0 +1,107 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +import logging +from . import HTTPResponse +from .._serialization import _get_data_bytes_or_stream_only +logger = logging.getLogger(__name__) + + +class _HTTPClient(object): + ''' + Takes the request and sends it to cloud service and returns the response. + ''' + + def __init__(self, protocol=None, session=None, timeout=None): + ''' + :param str protocol: + http or https. + :param requests.Session session: + session object created with requests library (or compatible). + :param int timeout: + timeout for the http request, in seconds. + ''' + self.protocol = protocol + self.session = session + self.timeout = timeout + + # By default, requests adds an Accept:*/* and Accept-Encoding to the session, + # which causes issues with some Azure REST APIs. Removing these here gives us + # the flexibility to add it back on a case by case basis. + if 'Accept' in self.session.headers: + del self.session.headers['Accept'] + + if 'Accept-Encoding' in self.session.headers: + del self.session.headers['Accept-Encoding'] + + self.proxies = None + + def set_proxy(self, host, port, user, password): + ''' + Sets the proxy server host and port for the HTTP CONNECT Tunnelling. + + Note that we set the proxies directly on the request later on rather than + using the session object as requests has a bug where session proxy is ignored + in favor of environment proxy. So, auth will not work unless it is passed + directly when making the request as this overrides both. + + :param str host: + Address of the proxy. Ex: '192.168.0.100' + :param int port: + Port of the proxy. Ex: 6000 + :param str user: + User for proxy authorization. + :param str password: + Password for proxy authorization. + ''' + if user and password: + proxy_string = '{}:{}@{}:{}'.format(user, password, host, port) + else: + proxy_string = '{}:{}'.format(host, port) + + self.proxies = {'http': 'http://{}'.format(proxy_string), + 'https': 'https://{}'.format(proxy_string)} + + def perform_request(self, request): + ''' + Sends an HTTPRequest to Azure Storage and returns an HTTPResponse. If + the response code indicates an error, raise an HTTPError. + + :param HTTPRequest request: + The request to serialize and send. + :return: An HTTPResponse containing the parsed HTTP response. + :rtype: :class:`~azure.storage.common._http.HTTPResponse` + ''' + # Verify the body is in bytes or either a file-like/stream object + if request.body: + request.body = _get_data_bytes_or_stream_only('request.body', request.body) + + # Construct the URI + uri = self.protocol.lower() + '://' + request.host + request.path + + # Send the request + response = self.session.request(request.method, + uri, + params=request.query, + headers=request.headers, + data=request.body or None, + timeout=self.timeout, + proxies=self.proxies) + + # Parse the response + status = int(response.status_code) + response_headers = {} + for key, name in response.headers.items(): + # Preserve the case of metadata + if key.lower().startswith('x-ms-meta-'): + response_headers[key] = name + else: + response_headers[key.lower()] = name + + wrap = HTTPResponse(status, response.reason, response_headers, response.content) + response.close() + + return wrap diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_serialization.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_serialization.py new file mode 100644 index 000000000000..af27ce5b0089 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_serialization.py @@ -0,0 +1,371 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import sys +import uuid +from datetime import date +from io import (BytesIO, IOBase, SEEK_SET, SEEK_END, UnsupportedOperation) +from os import fstat +from time import time +from wsgiref.handlers import format_date_time + +from dateutil.tz import tzutc + +if sys.version_info >= (3,): + from urllib.parse import quote as url_quote +else: + from urllib2 import quote as url_quote + +try: + from xml.etree import cElementTree as ETree +except ImportError: + from xml.etree import ElementTree as ETree + +from ._error import ( + _ERROR_VALUE_SHOULD_BE_BYTES, + _ERROR_VALUE_SHOULD_BE_BYTES_OR_STREAM, + _ERROR_VALUE_SHOULD_BE_SEEKABLE_STREAM +) +from .models import ( + _unicode_type, +) +from ._common_conversion import ( + _str, +) + + +def _to_utc_datetime(value): + # Azure expects the date value passed in to be UTC. + # Azure will always return values as UTC. + # If a date is passed in without timezone info, it is assumed to be UTC. + if value.tzinfo: + value = value.astimezone(tzutc()) + return value.strftime('%Y-%m-%dT%H:%M:%SZ') + + +def _update_request(request, x_ms_version, user_agent_string): + # Verify body + if request.body: + request.body = _get_data_bytes_or_stream_only('request.body', request.body) + length = _len_plus(request.body) + + # only scenario where this case is plausible is if the stream object is not seekable. + if length is None: + raise ValueError(_ERROR_VALUE_SHOULD_BE_SEEKABLE_STREAM) + + # if it is PUT, POST, MERGE, DELETE, need to add content-length to header. + if request.method in ['PUT', 'POST', 'MERGE', 'DELETE']: + request.headers['Content-Length'] = str(length) + + # append addtional headers based on the service + request.headers['x-ms-version'] = x_ms_version + request.headers['User-Agent'] = user_agent_string + request.headers['x-ms-client-request-id'] = str(uuid.uuid1()) + + # If the host has a path component (ex local storage), move it + path = request.host.split('/', 1) + if len(path) == 2: + request.host = path[0] + request.path = '/{}{}'.format(path[1], request.path) + + # Encode and optionally add local storage prefix to path + request.path = url_quote(request.path, '/()$=\',~') + + +def _add_metadata_headers(metadata, request): + if metadata: + if not request.headers: + request.headers = {} + for name, value in metadata.items(): + request.headers['x-ms-meta-' + name] = value + + +def _add_date_header(request): + current_time = format_date_time(time()) + request.headers['x-ms-date'] = current_time + + +def _get_data_bytes_only(param_name, param_value): + '''Validates the request body passed in and converts it to bytes + if our policy allows it.''' + if param_value is None: + return b'' + + if isinstance(param_value, bytes): + return param_value + + raise TypeError(_ERROR_VALUE_SHOULD_BE_BYTES.format(param_name)) + + +def _get_data_bytes_or_stream_only(param_name, param_value): + '''Validates the request body passed in is a stream/file-like or bytes + object.''' + if param_value is None: + return b'' + + if isinstance(param_value, bytes) or hasattr(param_value, 'read'): + return param_value + + raise TypeError(_ERROR_VALUE_SHOULD_BE_BYTES_OR_STREAM.format(param_name)) + + +def _get_request_body(request_body): + '''Converts an object into a request body. If it's None + we'll return an empty string, if it's one of our objects it'll + convert it to XML and return it. Otherwise we just use the object + directly''' + if request_body is None: + return b'' + + if isinstance(request_body, bytes) or isinstance(request_body, IOBase): + return request_body + + if isinstance(request_body, _unicode_type): + return request_body.encode('utf-8') + + request_body = str(request_body) + if isinstance(request_body, _unicode_type): + return request_body.encode('utf-8') + + return request_body + + +def _convert_signed_identifiers_to_xml(signed_identifiers): + if signed_identifiers is None: + return '' + + sis = ETree.Element('SignedIdentifiers') + for id, access_policy in signed_identifiers.items(): + # Root signed identifers element + si = ETree.SubElement(sis, 'SignedIdentifier') + + # Id element + ETree.SubElement(si, 'Id').text = id + + # Access policy element + policy = ETree.SubElement(si, 'AccessPolicy') + + if access_policy.start: + start = access_policy.start + if isinstance(access_policy.start, date): + start = _to_utc_datetime(start) + ETree.SubElement(policy, 'Start').text = start + + if access_policy.expiry: + expiry = access_policy.expiry + if isinstance(access_policy.expiry, date): + expiry = _to_utc_datetime(expiry) + ETree.SubElement(policy, 'Expiry').text = expiry + + if access_policy.permission: + ETree.SubElement(policy, 'Permission').text = _str(access_policy.permission) + + # Add xml declaration and serialize + try: + stream = BytesIO() + ETree.ElementTree(sis).write(stream, xml_declaration=True, encoding='utf-8', method='xml') + except: + raise + finally: + output = stream.getvalue() + stream.close() + + return output + + +def _convert_service_properties_to_xml(logging, hour_metrics, minute_metrics, + cors, target_version=None, delete_retention_policy=None, static_website=None): + ''' + + + + version-number + true|false + true|false + true|false + + true|false + number-of-days + + + + version-number + true|false + true|false + + true|false + number-of-days + + + + version-number + true|false + true|false + + true|false + number-of-days + + + + + comma-separated-list-of-allowed-origins + comma-separated-list-of-HTTP-verb + max-caching-age-in-seconds + comma-seperated-list-of-response-headers + comma-seperated-list-of-request-headers + + + + true|false + number-of-days + + + true|false + + + + + ''' + service_properties_element = ETree.Element('StorageServiceProperties') + + # Logging + if logging: + logging_element = ETree.SubElement(service_properties_element, 'Logging') + ETree.SubElement(logging_element, 'Version').text = logging.version + ETree.SubElement(logging_element, 'Delete').text = str(logging.delete) + ETree.SubElement(logging_element, 'Read').text = str(logging.read) + ETree.SubElement(logging_element, 'Write').text = str(logging.write) + + retention_element = ETree.SubElement(logging_element, 'RetentionPolicy') + _convert_retention_policy_to_xml(logging.retention_policy, retention_element) + + # HourMetrics + if hour_metrics: + hour_metrics_element = ETree.SubElement(service_properties_element, 'HourMetrics') + _convert_metrics_to_xml(hour_metrics, hour_metrics_element) + + # MinuteMetrics + if minute_metrics: + minute_metrics_element = ETree.SubElement(service_properties_element, 'MinuteMetrics') + _convert_metrics_to_xml(minute_metrics, minute_metrics_element) + + # CORS + # Make sure to still serialize empty list + if cors is not None: + cors_element = ETree.SubElement(service_properties_element, 'Cors') + for rule in cors: + cors_rule = ETree.SubElement(cors_element, 'CorsRule') + ETree.SubElement(cors_rule, 'AllowedOrigins').text = ",".join(rule.allowed_origins) + ETree.SubElement(cors_rule, 'AllowedMethods').text = ",".join(rule.allowed_methods) + ETree.SubElement(cors_rule, 'MaxAgeInSeconds').text = str(rule.max_age_in_seconds) + ETree.SubElement(cors_rule, 'ExposedHeaders').text = ",".join(rule.exposed_headers) + ETree.SubElement(cors_rule, 'AllowedHeaders').text = ",".join(rule.allowed_headers) + + # Target version + if target_version: + ETree.SubElement(service_properties_element, 'DefaultServiceVersion').text = target_version + + # DeleteRetentionPolicy + if delete_retention_policy: + policy_element = ETree.SubElement(service_properties_element, 'DeleteRetentionPolicy') + ETree.SubElement(policy_element, 'Enabled').text = str(delete_retention_policy.enabled) + + if delete_retention_policy.enabled: + ETree.SubElement(policy_element, 'Days').text = str(delete_retention_policy.days) + + # StaticWebsite + if static_website: + static_website_element = ETree.SubElement(service_properties_element, 'StaticWebsite') + ETree.SubElement(static_website_element, 'Enabled').text = str(static_website.enabled) + + if static_website.enabled: + + if static_website.index_document is not None: + ETree.SubElement(static_website_element, 'IndexDocument').text = str(static_website.index_document) + + if static_website.error_document_404_path is not None: + ETree.SubElement(static_website_element, 'ErrorDocument404Path').text = \ + str(static_website.error_document_404_path) + + # Add xml declaration and serialize + try: + stream = BytesIO() + ETree.ElementTree(service_properties_element).write(stream, xml_declaration=True, encoding='utf-8', + method='xml') + except: + raise + finally: + output = stream.getvalue() + stream.close() + + return output + + +def _convert_metrics_to_xml(metrics, root): + ''' + version-number + true|false + true|false + + true|false + number-of-days + + ''' + # Version + ETree.SubElement(root, 'Version').text = metrics.version + + # Enabled + ETree.SubElement(root, 'Enabled').text = str(metrics.enabled) + + # IncludeAPIs + if metrics.enabled and metrics.include_apis is not None: + ETree.SubElement(root, 'IncludeAPIs').text = str(metrics.include_apis) + + # RetentionPolicy + retention_element = ETree.SubElement(root, 'RetentionPolicy') + _convert_retention_policy_to_xml(metrics.retention_policy, retention_element) + + +def _convert_retention_policy_to_xml(retention_policy, root): + ''' + true|false + number-of-days + ''' + # Enabled + ETree.SubElement(root, 'Enabled').text = str(retention_policy.enabled) + + # Days + if retention_policy.enabled and retention_policy.days: + ETree.SubElement(root, 'Days').text = str(retention_policy.days) + + +def _len_plus(data): + length = None + # Check if object implements the __len__ method, covers most input cases such as bytearray. + try: + length = len(data) + except: + pass + + if not length: + # Check if the stream is a file-like stream object. + # If so, calculate the size using the file descriptor. + try: + fileno = data.fileno() + except (AttributeError, UnsupportedOperation): + pass + else: + return fstat(fileno).st_size + + # If the stream is seekable and tell() is implemented, calculate the stream size. + try: + current_position = data.tell() + data.seek(0, SEEK_END) + length = data.tell() - current_position + data.seek(current_position, SEEK_SET) + except (AttributeError, UnsupportedOperation): + pass + + return length diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/cloudstorageaccount.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/cloudstorageaccount.py new file mode 100644 index 000000000000..f3ac1aa7be70 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/cloudstorageaccount.py @@ -0,0 +1,200 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +# Note that we import BlobService/QueueService/FileService on demand +# because this module is imported by azure/storage/__init__ +# ie. we don't want 'import azure.storage' to trigger an automatic import +# of blob/queue/file packages. + +from ._error import _validate_not_none +from .models import ( + ResourceTypes, + Services, + AccountPermissions, +) +from .sharedaccesssignature import ( + SharedAccessSignature, +) + +''' +from azure.storage.common._error import _validate_not_none +from azure.storage.common.models import ( + ResourceTypes, + Services, + AccountPermissions, +) +from azure.storage.common.sharedaccesssignature import ( + SharedAccessSignature, +) +''' + + +class CloudStorageAccount(object): + """ + Provides a factory for creating the blob, queue, and file services + with a common account name and account key or sas token. Users can either + use the factory or can construct the appropriate service directly. + """ + + def __init__(self, account_name=None, account_key=None, sas_token=None, is_emulated=None): + ''' + :param str account_name: + The storage account name. This is used to authenticate requests + signed with an account key and to construct the storage endpoint. It + is required unless is_emulated is used. + :param str account_key: + The storage account key. This is used for shared key authentication. + :param str sas_token: + A shared access signature token to use to authenticate requests + instead of the account key. If account key and sas token are both + specified, account key will be used to sign. + :param bool is_emulated: + Whether to use the emulator. Defaults to False. If specified, will + override all other parameters. + ''' + self.account_name = account_name + self.account_key = account_key + self.sas_token = sas_token + self.is_emulated = is_emulated + + def create_block_blob_service(self): + ''' + Creates a BlockBlobService object with the settings specified in the + CloudStorageAccount. + + :return: A service object. + :rtype: :class:`~azure.storage.blob.blockblobservice.BlockBlobService` + ''' + try: + from azure.storage.blob.blockblobservice import BlockBlobService + return BlockBlobService(self.account_name, self.account_key, + sas_token=self.sas_token, + is_emulated=self.is_emulated) + except ImportError: + raise Exception('The package azure-storage-blob is required. ' + + 'Please install it using "pip install azure-storage-blob"') + + def create_page_blob_service(self): + ''' + Creates a PageBlobService object with the settings specified in the + CloudStorageAccount. + + :return: A service object. + :rtype: :class:`~azure.storage.blob.pageblobservice.PageBlobService` + ''' + try: + from azure.storage.blob.pageblobservice import PageBlobService + return PageBlobService(self.account_name, self.account_key, + sas_token=self.sas_token, + is_emulated=self.is_emulated) + except ImportError: + raise Exception('The package azure-storage-blob is required. ' + + 'Please install it using "pip install azure-storage-blob"') + + def create_append_blob_service(self): + ''' + Creates a AppendBlobService object with the settings specified in the + CloudStorageAccount. + + :return: A service object. + :rtype: :class:`~azure.storage.blob.appendblobservice.AppendBlobService` + ''' + try: + from azure.storage.blob.appendblobservice import AppendBlobService + return AppendBlobService(self.account_name, self.account_key, + sas_token=self.sas_token, + is_emulated=self.is_emulated) + except ImportError: + raise Exception('The package azure-storage-blob is required. ' + + 'Please install it using "pip install azure-storage-blob"') + + def create_queue_service(self): + ''' + Creates a QueueService object with the settings specified in the + CloudStorageAccount. + + :return: A service object. + :rtype: :class:`~azure.storage.queue.queueservice.QueueService` + ''' + try: + from azure.storage.queue.queueservice import QueueService + return QueueService(self.account_name, self.account_key, + sas_token=self.sas_token, + is_emulated=self.is_emulated) + except ImportError: + raise Exception('The package azure-storage-queue is required. ' + + 'Please install it using "pip install azure-storage-queue"') + + def create_file_service(self): + ''' + Creates a FileService object with the settings specified in the + CloudStorageAccount. + + :return: A service object. + :rtype: :class:`~azure.storage.file.fileservice.FileService` + ''' + try: + from azure.storage.file.fileservice import FileService + return FileService(self.account_name, self.account_key, + sas_token=self.sas_token) + except ImportError: + raise Exception('The package azure-storage-file is required. ' + + 'Please install it using "pip install azure-storage-file"') + + def generate_shared_access_signature(self, services, resource_types, + permission, expiry, start=None, + ip=None, protocol=None): + ''' + Generates a shared access signature for the account. + Use the returned signature with the sas_token parameter of the service + or to create a new account object. + + :param Services services: + Specifies the services accessible with the account SAS. You can + combine values to provide access to more than one service. + :param ResourceTypes resource_types: + Specifies the resource types that are accessible with the account + SAS. You can combine values to provide access to more than one + resource type. + :param AccountPermissions permission: + The permissions associated with the shared access signature. The + user is restricted to operations allowed by the permissions. + Required unless an id is given referencing a stored access policy + which contains this field. This field must be omitted if it has been + specified in an associated stored access policy. You can combine + values to provide more than one permission. + :param expiry: + The time at which the shared access signature becomes invalid. + Required unless an id is given referencing a stored access policy + which contains this field. This field must be omitted if it has + been specified in an associated stored access policy. Azure will always + convert values to UTC. If a date is passed in without timezone info, it + is assumed to be UTC. + :type expiry: datetime or str + :param start: + The time at which the shared access signature becomes valid. If + omitted, start time for this call is assumed to be the time when the + storage service receives the request. Azure will always convert values + to UTC. If a date is passed in without timezone info, it is assumed to + be UTC. + :type start: datetime or str + :param str ip: + Specifies an IP address or a range of IP addresses from which to accept requests. + If the IP address from which the request originates does not match the IP address + or address range specified on the SAS token, the request is not authenticated. + For example, specifying sip=168.1.5.65 or sip=168.1.5.60-168.1.5.70 on the SAS + restricts the request to those IP addresses. + :param str protocol: + Specifies the protocol permitted for a request made. Possible values are + both HTTPS and HTTP (https,http) or HTTPS only (https). The default value + is https,http. Note that HTTP only is not a permitted value. + ''' + _validate_not_none('self.account_name', self.account_name) + _validate_not_none('self.account_key', self.account_key) + + sas = SharedAccessSignature(self.account_name, self.account_key) + return sas.generate_account(services, resource_types, permission, + expiry, start=start, ip=ip, protocol=protocol) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/models.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/models.py new file mode 100644 index 000000000000..5ada54ce29dd --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/models.py @@ -0,0 +1,672 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import sys + +if sys.version_info < (3,): + from collections import Iterable + + _unicode_type = unicode +else: + from collections.abc import Iterable + + _unicode_type = str + +from ._error import ( + _validate_not_none +) + + +class _HeaderDict(dict): + def __getitem__(self, index): + return super(_HeaderDict, self).__getitem__(index.lower()) + + +class _list(list): + '''Used so that additional properties can be set on the return list''' + pass + + +class _dict(dict): + '''Used so that additional properties can be set on the return dictionary''' + pass + + +class _OperationContext(object): + ''' + Contains information that lasts the lifetime of an operation. This operation + may span multiple calls to the Azure service. + + :ivar bool location_lock: + Whether the location should be locked for this operation. + :ivar str location: + The location to lock to. + ''' + + def __init__(self, location_lock=False): + self.location_lock = location_lock + self.host_location = None + + +class ListGenerator(Iterable): + ''' + A generator object used to list storage resources. The generator will lazily + follow the continuation tokens returned by the service and stop when all + resources have been returned or max_results is reached. + + If max_results is specified and the account has more than that number of + resources, the generator will have a populated next_marker field once it + finishes. This marker can be used to create a new generator if more + results are desired. + ''' + + def __init__(self, resources, list_method, list_args, list_kwargs): + self.items = resources + self.next_marker = resources.next_marker + + self._list_method = list_method + self._list_args = list_args + self._list_kwargs = list_kwargs + + def __iter__(self): + # return results + for i in self.items: + yield i + + while True: + # if no more results on the service, return + if not self.next_marker: + break + + # update the marker args + self._list_kwargs['marker'] = self.next_marker + + # handle max results, if present + max_results = self._list_kwargs.get('max_results') + if max_results is not None: + max_results = max_results - len(self.items) + + # if we've reached max_results, return + # else, update the max_results arg + if max_results <= 0: + break + else: + self._list_kwargs['max_results'] = max_results + + # get the next segment + resources = self._list_method(*self._list_args, **self._list_kwargs) + self.items = resources + self.next_marker = resources.next_marker + + # return results + for i in self.items: + yield i + + +class RetryContext(object): + ''' + Contains the request and response information that can be used to determine + whether and how to retry. This context is stored across retries and may be + used to store other information relevant to the retry strategy. + + :ivar ~azure.storage.common._http.HTTPRequest request: + The request sent to the storage service. + :ivar ~azure.storage.common._http.HTTPResponse response: + The response returned by the storage service. + :ivar LocationMode location_mode: + The location the request was sent to. + :ivar Exception exception: + The exception that just occurred. The type could either be AzureException (for HTTP errors), + or other Exception types from lower layers, which are kept unwrapped for easier processing. + :ivar bool is_emulated: + Whether retry is targeting the emulator. The default value is False. + :ivar int body_position: + The initial position of the body stream. It is useful when retries happen and we need to rewind the stream. + ''' + + def __init__(self): + self.request = None + self.response = None + self.location_mode = None + self.exception = None + self.is_emulated = False + self.body_position = None + + +class LocationMode(object): + ''' + Specifies the location the request should be sent to. This mode only applies + for RA-GRS accounts which allow secondary read access. All other account types + must use PRIMARY. + ''' + + PRIMARY = 'primary' + ''' Requests should be sent to the primary location. ''' + + SECONDARY = 'secondary' + ''' Requests should be sent to the secondary location, if possible. ''' + + +class RetentionPolicy(object): + ''' + By default, Storage Analytics will not delete any logging or metrics data. Blobs + will continue to be written until the shared 20TB limit is + reached. Once the 20TB limit is reached, Storage Analytics will stop writing + new data and will not resume until free space is available. This 20TB limit + is independent of the total limit for your storage account. + + There are two ways to delete Storage Analytics data: by manually making deletion + requests or by setting a data retention policy. Manual requests to delete Storage + Analytics data are billable, but delete requests resulting from a retention policy + are not billable. + ''' + + def __init__(self, enabled=False, days=None): + ''' + :param bool enabled: + Indicates whether a retention policy is enabled for the + storage service. If disabled, logging and metrics data will be retained + infinitely by the service unless explicitly deleted. + :param int days: + Required if enabled is true. Indicates the number of + days that metrics or logging data should be retained. All data older + than this value will be deleted. The minimum value you can specify is 1; + the largest value is 365 (one year). + ''' + _validate_not_none("enabled", enabled) + if enabled: + _validate_not_none("days", days) + + self.enabled = enabled + self.days = days + + +class Logging(object): + ''' + Storage Analytics logs detailed information about successful and failed requests + to a storage service. This information can be used to monitor individual requests + and to diagnose issues with a storage service. Requests are logged on a best-effort + basis. + + All logs are stored in block blobs in a container named $logs, which is + automatically created when Storage Analytics is enabled for a storage account. + The $logs container is located in the blob namespace of the storage account. + This container cannot be deleted once Storage Analytics has been enabled, though + its contents can be deleted. + + For more information, see https://msdn.microsoft.com/en-us/library/azure/hh343262.aspx + ''' + + def __init__(self, delete=False, read=False, write=False, + retention_policy=None): + ''' + :param bool delete: + Indicates whether all delete requests should be logged. + :param bool read: + Indicates whether all read requests should be logged. + :param bool write: + Indicates whether all write requests should be logged. + :param RetentionPolicy retention_policy: + The retention policy for the metrics. + ''' + _validate_not_none("read", read) + _validate_not_none("write", write) + _validate_not_none("delete", delete) + + self.version = u'1.0' + self.delete = delete + self.read = read + self.write = write + self.retention_policy = retention_policy if retention_policy else RetentionPolicy() + + +class Metrics(object): + ''' + Metrics include aggregated transaction statistics and capacity data about requests + to a storage service. Transactions are reported at both the API operation level + as well as at the storage service level, and capacity is reported at the storage + service level. Metrics data can be used to analyze storage service usage, diagnose + issues with requests made against the storage service, and to improve the + performance of applications that use a service. + + For more information, see https://msdn.microsoft.com/en-us/library/azure/hh343258.aspx + ''' + + def __init__(self, enabled=False, include_apis=None, + retention_policy=None): + ''' + :param bool enabled: + Indicates whether metrics are enabled for + the service. + :param bool include_apis: + Required if enabled is True. Indicates whether metrics + should generate summary statistics for called API operations. + :param RetentionPolicy retention_policy: + The retention policy for the metrics. + ''' + _validate_not_none("enabled", enabled) + if enabled: + _validate_not_none("include_apis", include_apis) + + self.version = u'1.0' + self.enabled = enabled + self.include_apis = include_apis + self.retention_policy = retention_policy if retention_policy else RetentionPolicy() + + +class CorsRule(object): + ''' + CORS is an HTTP feature that enables a web application running under one domain + to access resources in another domain. Web browsers implement a security + restriction known as same-origin policy that prevents a web page from calling + APIs in a different domain; CORS provides a secure way to allow one domain + (the origin domain) to call APIs in another domain. + + For more information, see https://msdn.microsoft.com/en-us/library/azure/dn535601.aspx + ''' + + def __init__(self, allowed_origins, allowed_methods, max_age_in_seconds=0, + exposed_headers=None, allowed_headers=None): + ''' + :param allowed_origins: + A list of origin domains that will be allowed via CORS, or "*" to allow + all domains. The list of must contain at least one entry. Limited to 64 + origin domains. Each allowed origin can have up to 256 characters. + :type allowed_origins: list(str) + :param allowed_methods: + A list of HTTP methods that are allowed to be executed by the origin. + The list of must contain at least one entry. For Azure Storage, + permitted methods are DELETE, GET, HEAD, MERGE, POST, OPTIONS or PUT. + :type allowed_methods: list(str) + :param int max_age_in_seconds: + The number of seconds that the client/browser should cache a + preflight response. + :param exposed_headers: + Defaults to an empty list. A list of response headers to expose to CORS + clients. Limited to 64 defined headers and two prefixed headers. Each + header can be up to 256 characters. + :type exposed_headers: list(str) + :param allowed_headers: + Defaults to an empty list. A list of headers allowed to be part of + the cross-origin request. Limited to 64 defined headers and 2 prefixed + headers. Each header can be up to 256 characters. + :type allowed_headers: list(str) + ''' + _validate_not_none("allowed_origins", allowed_origins) + _validate_not_none("allowed_methods", allowed_methods) + _validate_not_none("max_age_in_seconds", max_age_in_seconds) + + self.allowed_origins = allowed_origins if allowed_origins else list() + self.allowed_methods = allowed_methods if allowed_methods else list() + self.max_age_in_seconds = max_age_in_seconds + self.exposed_headers = exposed_headers if exposed_headers else list() + self.allowed_headers = allowed_headers if allowed_headers else list() + + +class DeleteRetentionPolicy(object): + ''' + To set DeleteRetentionPolicy, you must call Set Blob Service Properties using version 2017-07-29 or later. + This class groups the settings related to delete retention policy. + ''' + + def __init__(self, enabled=False, days=None): + ''' + :param bool enabled: + Required. Indicates whether a deleted blob or snapshot is retained or immediately removed by delete operation. + :param int days: + Required only if Enabled is true. Indicates the number of days that deleted blob be retained. + All data older than this value will be permanently deleted. + The minimum value you can specify is 1; the largest value is 365. + ''' + _validate_not_none("enabled", enabled) + if enabled: + _validate_not_none("days", days) + + self.enabled = enabled + self.days = days + + +class StaticWebsite(object): + ''' + Class representing the service properties pertaining to static websites. + To set StaticWebsite, you must call Set Blob Service Properties using version 2018-03-28 or later. + ''' + + def __init__(self, enabled=False, index_document=None, error_document_404_path=None): + ''' + :param bool enabled: + Required. True if static websites should be enabled on the blob service for the corresponding Storage Account. + :param str index_document: + Represents the name of the index document. This is commonly "index.html". + :param str error_document_404_path: + Represents the path to the error document that should be shown when an error 404 is issued, + in other words, when a browser requests a page that does not exist. + ''' + _validate_not_none("enabled", enabled) + + self.enabled = enabled + self.index_document = index_document + self.error_document_404_path = error_document_404_path + + +class ServiceProperties(object): + ''' + Returned by get_*_service_properties functions. Contains the properties of a + storage service, including Analytics and CORS rules. + + Azure Storage Analytics performs logging and provides metrics data for a storage + account. You can use this data to trace requests, analyze usage trends, and + diagnose issues with your storage account. To use Storage Analytics, you must + enable it individually for each service you want to monitor. + + The aggregated data is stored in a well-known blob (for logging) and in well-known + tables (for metrics), which may be accessed using the Blob service and Table + service APIs. + + For an in-depth guide on using Storage Analytics and other tools to identify, + diagnose, and troubleshoot Azure Storage-related issues, see + http://azure.microsoft.com/documentation/articles/storage-monitoring-diagnosing-troubleshooting/ + + For more information on CORS, see https://msdn.microsoft.com/en-us/library/azure/dn535601.aspx + ''' + + pass + + +class ServiceStats(object): + ''' + Returned by get_*_service_stats functions. Contains statistics related to + replication for the given service. It is only available when read-access + geo-redundant replication is enabled for the storage account. + + :ivar GeoReplication geo_replication: + An object containing statistics related to replication for the given service. + ''' + pass + + +class GeoReplication(object): + ''' + Contains statistics related to replication for the given service. + + :ivar str status: + The status of the secondary location. Possible values are: + live: Indicates that the secondary location is active and operational. + bootstrap: Indicates initial synchronization from the primary location + to the secondary location is in progress. This typically occurs + when replication is first enabled. + unavailable: Indicates that the secondary location is temporarily + unavailable. + :ivar date last_sync_time: + A GMT date value, to the second. All primary writes preceding this value + are guaranteed to be available for read operations at the secondary. + Primary writes after this point in time may or may not be available for + reads. The value may be empty if LastSyncTime is not available. This can + happen if the replication status is bootstrap or unavailable. Although + geo-replication is continuously enabled, the LastSyncTime result may + reflect a cached value from the service that is refreshed every few minutes. + ''' + pass + + +class AccessPolicy(object): + ''' + Access Policy class used by the set and get acl methods in each service. + + A stored access policy can specify the start time, expiry time, and + permissions for the Shared Access Signatures with which it's associated. + Depending on how you want to control access to your resource, you can + specify all of these parameters within the stored access policy, and omit + them from the URL for the Shared Access Signature. Doing so permits you to + modify the associated signature's behavior at any time, as well as to revoke + it. Or you can specify one or more of the access policy parameters within + the stored access policy, and the others on the URL. Finally, you can + specify all of the parameters on the URL. In this case, you can use the + stored access policy to revoke the signature, but not to modify its behavior. + + Together the Shared Access Signature and the stored access policy must + include all fields required to authenticate the signature. If any required + fields are missing, the request will fail. Likewise, if a field is specified + both in the Shared Access Signature URL and in the stored access policy, the + request will fail with status code 400 (Bad Request). + ''' + + def __init__(self, permission=None, expiry=None, start=None): + ''' + :param str permission: + The permissions associated with the shared access signature. The + user is restricted to operations allowed by the permissions. + Required unless an id is given referencing a stored access policy + which contains this field. This field must be omitted if it has been + specified in an associated stored access policy. + :param expiry: + The time at which the shared access signature becomes invalid. + Required unless an id is given referencing a stored access policy + which contains this field. This field must be omitted if it has + been specified in an associated stored access policy. Azure will always + convert values to UTC. If a date is passed in without timezone info, it + is assumed to be UTC. + :type expiry: datetime or str + :param start: + The time at which the shared access signature becomes valid. If + omitted, start time for this call is assumed to be the time when the + storage service receives the request. Azure will always convert values + to UTC. If a date is passed in without timezone info, it is assumed to + be UTC. + :type start: datetime or str + ''' + self.start = start + self.expiry = expiry + self.permission = permission + + +class Protocol(object): + ''' + Specifies the protocol permitted for a SAS token. Note that HTTP only is + not allowed. + ''' + + HTTPS = 'https' + ''' Allow HTTPS requests only. ''' + + HTTPS_HTTP = 'https,http' + ''' Allow HTTP and HTTPS requests. ''' + + +class ResourceTypes(object): + ''' + Specifies the resource types that are accessible with the account SAS. + + :ivar ResourceTypes ResourceTypes.CONTAINER: + Access to container-level APIs (e.g., Create/Delete Container, + Create/Delete Queue, Create/Delete Share, + List Blobs/Files and Directories) + :ivar ResourceTypes ResourceTypes.OBJECT: + Access to object-level APIs for blobs, queue messages, and + files(e.g. Put Blob, Query Entity, Get Messages, Create File, etc.) + :ivar ResourceTypes ResourceTypes.SERVICE: + Access to service-level APIs (e.g., Get/Set Service Properties, + Get Service Stats, List Containers/Queues/Shares) + ''' + + def __init__(self, service=False, container=False, object=False, _str=None): + ''' + :param bool service: + Access to service-level APIs (e.g., Get/Set Service Properties, + Get Service Stats, List Containers/Queues/Shares) + :param bool container: + Access to container-level APIs (e.g., Create/Delete Container, + Create/Delete Queue, Create/Delete Share, + List Blobs/Files and Directories) + :param bool object: + Access to object-level APIs for blobs, queue messages, and + files(e.g. Put Blob, Query Entity, Get Messages, Create File, etc.) + :param str _str: + A string representing the resource types. + ''' + if not _str: + _str = '' + self.service = service or ('s' in _str) + self.container = container or ('c' in _str) + self.object = object or ('o' in _str) + + def __or__(self, other): + return ResourceTypes(_str=str(self) + str(other)) + + def __add__(self, other): + return ResourceTypes(_str=str(self) + str(other)) + + def __str__(self): + return (('s' if self.service else '') + + ('c' if self.container else '') + + ('o' if self.object else '')) + + +ResourceTypes.SERVICE = ResourceTypes(service=True) +ResourceTypes.CONTAINER = ResourceTypes(container=True) +ResourceTypes.OBJECT = ResourceTypes(object=True) + + +class Services(object): + ''' + Specifies the services accessible with the account SAS. + + :ivar Services Services.BLOB: The blob service. + :ivar Services Services.FILE: The file service + :ivar Services Services.QUEUE: The queue service. + :ivar Services Services.TABLE: The table service. + ''' + + def __init__(self, blob=False, queue=False, file=False, table=False, _str=None): + ''' + :param bool blob: + Access to any blob service, for example, the `.BlockBlobService` + :param bool queue: + Access to the `.QueueService` + :param bool file: + Access to the `.FileService` + :param bool table: + Access to the TableService + :param str _str: + A string representing the services. + ''' + if not _str: + _str = '' + self.blob = blob or ('b' in _str) + self.queue = queue or ('q' in _str) + self.file = file or ('f' in _str) + self.table = table or ('t' in _str) + + def __or__(self, other): + return Services(_str=str(self) + str(other)) + + def __add__(self, other): + return Services(_str=str(self) + str(other)) + + def __str__(self): + return (('b' if self.blob else '') + + ('q' if self.queue else '') + + ('t' if self.table else '') + + ('f' if self.file else '')) + + +Services.BLOB = Services(blob=True) +Services.QUEUE = Services(queue=True) +Services.TABLE = Services(table=True) +Services.FILE = Services(file=True) + + +class AccountPermissions(object): + ''' + :class:`~ResourceTypes` class to be used with generate_shared_access_signature + method and for the AccessPolicies used with set_*_acl. There are two types of + SAS which may be used to grant resource access. One is to grant access to a + specific resource (resource-specific). Another is to grant access to the + entire service for a specific account and allow certain operations based on + perms found here. + + :ivar AccountPermissions AccountPermissions.ADD: + Valid for the following Object resource types only: queue messages and append blobs. + :ivar AccountPermissions AccountPermissions.CREATE: + Valid for the following Object resource types only: blobs and files. Users + can create new blobs or files, but may not overwrite existing blobs or files. + :ivar AccountPermissions AccountPermissions.DELETE: + Valid for Container and Object resource types, except for queue messages. + :ivar AccountPermissions AccountPermissions.LIST: + Valid for Service and Container resource types only. + :ivar AccountPermissions AccountPermissions.PROCESS: + Valid for the following Object resource type only: queue messages. + :ivar AccountPermissions AccountPermissions.READ: + Valid for all signed resources types (Service, Container, and Object). + Permits read permissions to the specified resource type. + :ivar AccountPermissions AccountPermissions.UPDATE: + Valid for the following Object resource types only: queue messages. + :ivar AccountPermissions AccountPermissions.WRITE: + Valid for all signed resources types (Service, Container, and Object). + Permits write permissions to the specified resource type. + ''' + + def __init__(self, read=False, write=False, delete=False, list=False, + add=False, create=False, update=False, process=False, _str=None): + ''' + :param bool read: + Valid for all signed resources types (Service, Container, and Object). + Permits read permissions to the specified resource type. + :param bool write: + Valid for all signed resources types (Service, Container, and Object). + Permits write permissions to the specified resource type. + :param bool delete: + Valid for Container and Object resource types, except for queue messages. + :param bool list: + Valid for Service and Container resource types only. + :param bool add: + Valid for the following Object resource types only: queue messages, and append blobs. + :param bool create: + Valid for the following Object resource types only: blobs and files. + Users can create new blobs or files, but may not overwrite existing + blobs or files. + :param bool update: + Valid for the following Object resource types only: queue messages. + :param bool process: + Valid for the following Object resource type only: queue messages. + :param str _str: + A string representing the permissions. + ''' + if not _str: + _str = '' + self.read = read or ('r' in _str) + self.write = write or ('w' in _str) + self.delete = delete or ('d' in _str) + self.list = list or ('l' in _str) + self.add = add or ('a' in _str) + self.create = create or ('c' in _str) + self.update = update or ('u' in _str) + self.process = process or ('p' in _str) + + def __or__(self, other): + return AccountPermissions(_str=str(self) + str(other)) + + def __add__(self, other): + return AccountPermissions(_str=str(self) + str(other)) + + def __str__(self): + return (('r' if self.read else '') + + ('w' if self.write else '') + + ('d' if self.delete else '') + + ('l' if self.list else '') + + ('a' if self.add else '') + + ('c' if self.create else '') + + ('u' if self.update else '') + + ('p' if self.process else '')) + + +AccountPermissions.READ = AccountPermissions(read=True) +AccountPermissions.WRITE = AccountPermissions(write=True) +AccountPermissions.DELETE = AccountPermissions(delete=True) +AccountPermissions.LIST = AccountPermissions(list=True) +AccountPermissions.ADD = AccountPermissions(add=True) +AccountPermissions.CREATE = AccountPermissions(create=True) +AccountPermissions.UPDATE = AccountPermissions(update=True) +AccountPermissions.PROCESS = AccountPermissions(process=True) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/retry.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/retry.py new file mode 100644 index 000000000000..85764430259d --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/retry.py @@ -0,0 +1,306 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from abc import ABCMeta +from math import pow +import random +from io import (SEEK_SET, UnsupportedOperation) + +from .models import LocationMode +from ._constants import ( + DEV_ACCOUNT_NAME, + DEV_ACCOUNT_SECONDARY_NAME +) + + +class _Retry(object): + ''' + The base class for Exponential and Linear retries containing shared code. + ''' + __metaclass__ = ABCMeta + + def __init__(self, max_attempts, retry_to_secondary): + ''' + Constructs a base retry object. + + :param int max_attempts: + The maximum number of retry attempts. + :param bool retry_to_secondary: + Whether the request should be retried to secondary, if able. This should + only be enabled of RA-GRS accounts are used and potentially stale data + can be handled. + ''' + self.max_attempts = max_attempts + self.retry_to_secondary = retry_to_secondary + + def _should_retry(self, context): + ''' + A function which determines whether or not to retry. + + :param ~azure.storage.models.RetryContext context: + The retry context. This contains the request, response, and other data + which can be used to determine whether or not to retry. + :return: + A boolean indicating whether or not to retry the request. + :rtype: bool + ''' + # If max attempts are reached, do not retry. + if context.count >= self.max_attempts: + return False + + status = None + if context.response and context.response.status: + status = context.response.status + + if status is None: + ''' + If status is None, retry as this request triggered an exception. For + example, network issues would trigger this. + ''' + return True + elif 200 <= status < 300: + ''' + This method is called after a successful response, meaning we failed + during the response body download or parsing. So, success codes should + be retried. + ''' + return True + elif 300 <= status < 500: + ''' + An exception occured, but in most cases it was expected. Examples could + include a 309 Conflict or 412 Precondition Failed. + ''' + if status == 404 and context.location_mode == LocationMode.SECONDARY: + # Response code 404 should be retried if secondary was used. + return True + if status == 408: + # Response code 408 is a timeout and should be retried. + return True + return False + elif status >= 500: + ''' + Response codes above 500 with the exception of 501 Not Implemented and + 505 Version Not Supported indicate a server issue and should be retried. + ''' + if status == 501 or status == 505: + return False + return True + else: + # If something else happened, it's unexpected. Retry. + return True + + def _set_next_host_location(self, context): + ''' + A function which sets the next host location on the request, if applicable. + + :param ~azure.storage.models.RetryContext context: + The retry context containing the previous host location and the request + to evaluate and possibly modify. + ''' + if len(context.request.host_locations) > 1: + # If there's more than one possible location, retry to the alternative + if context.location_mode == LocationMode.PRIMARY: + context.location_mode = LocationMode.SECONDARY + + # if targeting the emulator (with path style), change path instead of host + if context.is_emulated: + # replace the first instance of primary account name with the secondary account name + context.request.path = context.request.path.replace(DEV_ACCOUNT_NAME, DEV_ACCOUNT_SECONDARY_NAME, 1) + else: + context.request.host = context.request.host_locations.get(context.location_mode) + else: + context.location_mode = LocationMode.PRIMARY + + # if targeting the emulator (with path style), change path instead of host + if context.is_emulated: + # replace the first instance of secondary account name with the primary account name + context.request.path = context.request.path.replace(DEV_ACCOUNT_SECONDARY_NAME, DEV_ACCOUNT_NAME, 1) + else: + context.request.host = context.request.host_locations.get(context.location_mode) + + def _retry(self, context, backoff): + ''' + A function which determines whether and how to retry. + + :param ~azure.storage.models.RetryContext context: + The retry context. This contains the request, response, and other data + which can be used to determine whether or not to retry. + :param function() backoff: + A function which returns the backoff time if a retry is to be performed. + :return: + An integer indicating how long to wait before retrying the request, + or None to indicate no retry should be performed. + :rtype: int or None + ''' + # If the context does not contain a count parameter, this request has not + # been retried yet. Add the count parameter to track the number of retries. + if not hasattr(context, 'count'): + context.count = 0 + + # Determine whether to retry, and if so increment the count, modify the + # request as desired, and return the backoff. + if self._should_retry(context): + backoff_interval = backoff(context) + context.count += 1 + + # If retry to secondary is enabled, attempt to change the host if the + # request allows it + if self.retry_to_secondary: + self._set_next_host_location(context) + + # rewind the request body if it is a stream + if hasattr(context.request.body, 'read'): + # no position was saved, then retry would not work + if context.body_position is None: + return None + else: + try: + # attempt to rewind the body to the initial position + context.request.body.seek(context.body_position, SEEK_SET) + except UnsupportedOperation: + # if body is not seekable, then retry would not work + return None + + return backoff_interval + + return None + + +class ExponentialRetry(_Retry): + ''' + Exponential retry. + ''' + + def __init__(self, initial_backoff=15, increment_base=3, max_attempts=3, + retry_to_secondary=False, random_jitter_range=3): + ''' + Constructs an Exponential retry object. The initial_backoff is used for + the first retry. Subsequent retries are retried after initial_backoff + + increment_power^retry_count seconds. For example, by default the first retry + occurs after 15 seconds, the second after (15+3^1) = 18 seconds, and the + third after (15+3^2) = 24 seconds. + + :param int initial_backoff: + The initial backoff interval, in seconds, for the first retry. + :param int increment_base: + The base, in seconds, to increment the initial_backoff by after the + first retry. + :param int max_attempts: + The maximum number of retry attempts. + :param bool retry_to_secondary: + Whether the request should be retried to secondary, if able. This should + only be enabled of RA-GRS accounts are used and potentially stale data + can be handled. + :param int random_jitter_range: + A number in seconds which indicates a range to jitter/randomize for the back-off interval. + For example, a random_jitter_range of 3 results in the back-off interval x to vary between x+3 and x-3. + ''' + self.initial_backoff = initial_backoff + self.increment_base = increment_base + self.random_jitter_range = random_jitter_range + super(ExponentialRetry, self).__init__(max_attempts, retry_to_secondary) + + ''' + A function which determines whether and how to retry. + + :param ~azure.storage.models.RetryContext context: + The retry context. This contains the request, response, and other data + which can be used to determine whether or not to retry. + :return: + An integer indicating how long to wait before retrying the request, + or None to indicate no retry should be performed. + :rtype: int or None + ''' + + def retry(self, context): + return self._retry(context, self._backoff) + + ''' + Calculates how long to sleep before retrying. + + :return: + An integer indicating how long to wait before retrying the request, + or None to indicate no retry should be performed. + :rtype: int or None + ''' + + def _backoff(self, context): + random_generator = random.Random() + backoff = self.initial_backoff + (0 if context.count == 0 else pow(self.increment_base, context.count)) + random_range_start = backoff - self.random_jitter_range if backoff > self.random_jitter_range else 0 + random_range_end = backoff + self.random_jitter_range + return random_generator.uniform(random_range_start, random_range_end) + + +class LinearRetry(_Retry): + ''' + Linear retry. + ''' + + def __init__(self, backoff=15, max_attempts=3, retry_to_secondary=False, random_jitter_range=3): + ''' + Constructs a Linear retry object. + + :param int backoff: + The backoff interval, in seconds, between retries. + :param int max_attempts: + The maximum number of retry attempts. + :param bool retry_to_secondary: + Whether the request should be retried to secondary, if able. This should + only be enabled of RA-GRS accounts are used and potentially stale data + can be handled. + :param int random_jitter_range: + A number in seconds which indicates a range to jitter/randomize for the back-off interval. + For example, a random_jitter_range of 3 results in the back-off interval x to vary between x+3 and x-3. + ''' + self.backoff = backoff + self.max_attempts = max_attempts + self.random_jitter_range = random_jitter_range + super(LinearRetry, self).__init__(max_attempts, retry_to_secondary) + + ''' + A function which determines whether and how to retry. + + :param ~azure.storage.models.RetryContext context: + The retry context. This contains the request, response, and other data + which can be used to determine whether or not to retry. + :return: + An integer indicating how long to wait before retrying the request, + or None to indicate no retry should be performed. + :rtype: int or None + ''' + + def retry(self, context): + return self._retry(context, self._backoff) + + ''' + Calculates how long to sleep before retrying. + + :return: + An integer indicating how long to wait before retrying the request, + or None to indicate no retry should be performed. + :rtype: int or None + ''' + + def _backoff(self, context): + random_generator = random.Random() + # the backoff interval normally does not change, however there is the possibility + # that it was modified by accessing the property directly after initializing the object + self.random_range_start = self.backoff - self.random_jitter_range if self.backoff > self.random_jitter_range else 0 + self.random_range_end = self.backoff + self.random_jitter_range + return random_generator.uniform(self.random_range_start, self.random_range_end) + + +def no_retry(context): + ''' + Specifies never to retry. + + :param ~azure.storage.models.RetryContext context: + The retry context. + :return: + Always returns None to indicate never to retry. + :rtype: None + ''' + return None diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/sharedaccesssignature.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/sharedaccesssignature.py new file mode 100644 index 000000000000..c23201a85bcf --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/sharedaccesssignature.py @@ -0,0 +1,217 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from datetime import date + +from ._common_conversion import ( + _sign_string, + _to_str, +) +from ._constants import DEFAULT_X_MS_VERSION +from ._serialization import ( + url_quote, + _to_utc_datetime, +) + + +class SharedAccessSignature(object): + ''' + Provides a factory for creating account access + signature tokens with an account name and account key. Users can either + use the factory or can construct the appropriate service and use the + generate_*_shared_access_signature method directly. + ''' + + def __init__(self, account_name, account_key, x_ms_version=DEFAULT_X_MS_VERSION): + ''' + :param str account_name: + The storage account name used to generate the shared access signatures. + :param str account_key: + The access key to generate the shares access signatures. + :param str x_ms_version: + The service version used to generate the shared access signatures. + ''' + self.account_name = account_name + self.account_key = account_key + self.x_ms_version = x_ms_version + + def generate_account(self, services, resource_types, permission, expiry, start=None, + ip=None, protocol=None): + ''' + Generates a shared access signature for the account. + Use the returned signature with the sas_token parameter of the service + or to create a new account object. + + :param Services services: + Specifies the services accessible with the account SAS. You can + combine values to provide access to more than one service. + :param ResourceTypes resource_types: + Specifies the resource types that are accessible with the account + SAS. You can combine values to provide access to more than one + resource type. + :param AccountPermissions permission: + The permissions associated with the shared access signature. The + user is restricted to operations allowed by the permissions. + Required unless an id is given referencing a stored access policy + which contains this field. This field must be omitted if it has been + specified in an associated stored access policy. You can combine + values to provide more than one permission. + :param expiry: + The time at which the shared access signature becomes invalid. + Required unless an id is given referencing a stored access policy + which contains this field. This field must be omitted if it has + been specified in an associated stored access policy. Azure will always + convert values to UTC. If a date is passed in without timezone info, it + is assumed to be UTC. + :type expiry: datetime or str + :param start: + The time at which the shared access signature becomes valid. If + omitted, start time for this call is assumed to be the time when the + storage service receives the request. Azure will always convert values + to UTC. If a date is passed in without timezone info, it is assumed to + be UTC. + :type start: datetime or str + :param str ip: + Specifies an IP address or a range of IP addresses from which to accept requests. + If the IP address from which the request originates does not match the IP address + or address range specified on the SAS token, the request is not authenticated. + For example, specifying sip=168.1.5.65 or sip=168.1.5.60-168.1.5.70 on the SAS + restricts the request to those IP addresses. + :param str protocol: + Specifies the protocol permitted for a request made. The default value + is https,http. See :class:`~azure.storage.common.models.Protocol` for possible values. + ''' + sas = _SharedAccessHelper() + sas.add_base(permission, expiry, start, ip, protocol, self.x_ms_version) + sas.add_account(services, resource_types) + sas.add_account_signature(self.account_name, self.account_key) + + return sas.get_token() + + +class _QueryStringConstants(object): + SIGNED_SIGNATURE = 'sig' + SIGNED_PERMISSION = 'sp' + SIGNED_START = 'st' + SIGNED_EXPIRY = 'se' + SIGNED_RESOURCE = 'sr' + SIGNED_IDENTIFIER = 'si' + SIGNED_IP = 'sip' + SIGNED_PROTOCOL = 'spr' + SIGNED_VERSION = 'sv' + SIGNED_CACHE_CONTROL = 'rscc' + SIGNED_CONTENT_DISPOSITION = 'rscd' + SIGNED_CONTENT_ENCODING = 'rsce' + SIGNED_CONTENT_LANGUAGE = 'rscl' + SIGNED_CONTENT_TYPE = 'rsct' + START_PK = 'spk' + START_RK = 'srk' + END_PK = 'epk' + END_RK = 'erk' + SIGNED_RESOURCE_TYPES = 'srt' + SIGNED_SERVICES = 'ss' + + +class _SharedAccessHelper(object): + def __init__(self): + self.query_dict = {} + + def _add_query(self, name, val): + if val: + self.query_dict[name] = _to_str(val) + + def add_base(self, permission, expiry, start, ip, protocol, x_ms_version): + if isinstance(start, date): + start = _to_utc_datetime(start) + + if isinstance(expiry, date): + expiry = _to_utc_datetime(expiry) + + self._add_query(_QueryStringConstants.SIGNED_START, start) + self._add_query(_QueryStringConstants.SIGNED_EXPIRY, expiry) + self._add_query(_QueryStringConstants.SIGNED_PERMISSION, permission) + self._add_query(_QueryStringConstants.SIGNED_IP, ip) + self._add_query(_QueryStringConstants.SIGNED_PROTOCOL, protocol) + self._add_query(_QueryStringConstants.SIGNED_VERSION, x_ms_version) + + def add_resource(self, resource): + self._add_query(_QueryStringConstants.SIGNED_RESOURCE, resource) + + def add_id(self, id): + self._add_query(_QueryStringConstants.SIGNED_IDENTIFIER, id) + + def add_account(self, services, resource_types): + self._add_query(_QueryStringConstants.SIGNED_SERVICES, services) + self._add_query(_QueryStringConstants.SIGNED_RESOURCE_TYPES, resource_types) + + def add_override_response_headers(self, cache_control, + content_disposition, + content_encoding, + content_language, + content_type): + self._add_query(_QueryStringConstants.SIGNED_CACHE_CONTROL, cache_control) + self._add_query(_QueryStringConstants.SIGNED_CONTENT_DISPOSITION, content_disposition) + self._add_query(_QueryStringConstants.SIGNED_CONTENT_ENCODING, content_encoding) + self._add_query(_QueryStringConstants.SIGNED_CONTENT_LANGUAGE, content_language) + self._add_query(_QueryStringConstants.SIGNED_CONTENT_TYPE, content_type) + + def add_resource_signature(self, account_name, account_key, service, path): + def get_value_to_append(query): + return_value = self.query_dict.get(query) or '' + return return_value + '\n' + + if path[0] != '/': + path = '/' + path + + canonicalized_resource = '/' + service + '/' + account_name + path + '\n' + + # Form the string to sign from shared_access_policy and canonicalized + # resource. The order of values is important. + string_to_sign = \ + (get_value_to_append(_QueryStringConstants.SIGNED_PERMISSION) + + get_value_to_append(_QueryStringConstants.SIGNED_START) + + get_value_to_append(_QueryStringConstants.SIGNED_EXPIRY) + + canonicalized_resource + + get_value_to_append(_QueryStringConstants.SIGNED_IDENTIFIER) + + get_value_to_append(_QueryStringConstants.SIGNED_IP) + + get_value_to_append(_QueryStringConstants.SIGNED_PROTOCOL) + + get_value_to_append(_QueryStringConstants.SIGNED_VERSION)) + + if service == 'blob' or service == 'file': + string_to_sign += \ + (get_value_to_append(_QueryStringConstants.SIGNED_CACHE_CONTROL) + + get_value_to_append(_QueryStringConstants.SIGNED_CONTENT_DISPOSITION) + + get_value_to_append(_QueryStringConstants.SIGNED_CONTENT_ENCODING) + + get_value_to_append(_QueryStringConstants.SIGNED_CONTENT_LANGUAGE) + + get_value_to_append(_QueryStringConstants.SIGNED_CONTENT_TYPE)) + + # remove the trailing newline + if string_to_sign[-1] == '\n': + string_to_sign = string_to_sign[:-1] + + self._add_query(_QueryStringConstants.SIGNED_SIGNATURE, + _sign_string(account_key, string_to_sign)) + + def add_account_signature(self, account_name, account_key): + def get_value_to_append(query): + return_value = self.query_dict.get(query) or '' + return return_value + '\n' + + string_to_sign = \ + (account_name + '\n' + + get_value_to_append(_QueryStringConstants.SIGNED_PERMISSION) + + get_value_to_append(_QueryStringConstants.SIGNED_SERVICES) + + get_value_to_append(_QueryStringConstants.SIGNED_RESOURCE_TYPES) + + get_value_to_append(_QueryStringConstants.SIGNED_START) + + get_value_to_append(_QueryStringConstants.SIGNED_EXPIRY) + + get_value_to_append(_QueryStringConstants.SIGNED_IP) + + get_value_to_append(_QueryStringConstants.SIGNED_PROTOCOL) + + get_value_to_append(_QueryStringConstants.SIGNED_VERSION)) + + self._add_query(_QueryStringConstants.SIGNED_SIGNATURE, + _sign_string(account_key, string_to_sign)) + + def get_token(self): + return '&'.join(['{0}={1}'.format(n, url_quote(v)) for n, v in self.query_dict.items() if v is not None]) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/storageclient.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/storageclient.py new file mode 100644 index 000000000000..859a729df466 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/storageclient.py @@ -0,0 +1,391 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +import sys +from abc import ABCMeta +import logging + +logger = logging.getLogger(__name__) +from time import sleep + +import requests +from azure.common import ( + AzureException, + AzureHttpError, +) + +from ._constants import ( + DEFAULT_SOCKET_TIMEOUT, + DEFAULT_X_MS_VERSION, + DEFAULT_USER_AGENT_STRING, + USER_AGENT_STRING_PREFIX, + USER_AGENT_STRING_SUFFIX, +) +from ._error import ( + _ERROR_DECRYPTION_FAILURE, + _http_error_handler, +) +from ._http import HTTPError +from ._http.httpclient import _HTTPClient +from ._serialization import ( + _update_request, + _add_date_header, +) +from .models import ( + RetryContext, + LocationMode, + _OperationContext, +) +from .retry import ExponentialRetry +from io import UnsupportedOperation + + +class StorageClient(object): + ''' + This is the base class for service objects. Service objects are used to do + all requests to Storage. This class cannot be instantiated directly. + + :ivar str account_name: + The storage account name. This is used to authenticate requests + signed with an account key and to construct the storage endpoint. It + is required unless a connection string is given, or if a custom + domain is used with anonymous authentication. + :ivar str account_key: + The storage account key. This is used for shared key authentication. + If neither account key or sas token is specified, anonymous access + will be used. + :ivar str sas_token: + A shared access signature token to use to authenticate requests + instead of the account key. If account key and sas token are both + specified, account key will be used to sign. If neither are + specified, anonymous access will be used. + :ivar str primary_endpoint: + The endpoint to send storage requests to. + :ivar str secondary_endpoint: + The secondary endpoint to read storage data from. This will only be a + valid endpoint if the storage account used is RA-GRS and thus allows + reading from secondary. + :ivar function(context) retry: + A function which determines whether to retry. Takes as a parameter a + :class:`~azure.storage.common.models.RetryContext` object. Returns the number + of seconds to wait before retrying the request, or None to indicate not + to retry. + :ivar ~azure.storage.common.models.LocationMode location_mode: + The host location to use to make requests. Defaults to LocationMode.PRIMARY. + Note that this setting only applies to RA-GRS accounts as other account + types do not allow reading from secondary. If the location_mode is set to + LocationMode.SECONDARY, read requests will be sent to the secondary endpoint. + Write requests will continue to be sent to primary. + :ivar str protocol: + The protocol to use for requests. Defaults to https. + :ivar requests.Session request_session: + The session object to use for http requests. + :ivar function(request) request_callback: + A function called immediately before each request is sent. This function + takes as a parameter the request object and returns nothing. It may be + used to added custom headers or log request data. + :ivar function() response_callback: + A function called immediately after each response is received. This + function takes as a parameter the response object and returns nothing. + It may be used to log response data. + :ivar function() retry_callback: + A function called immediately after retry evaluation is performed. This + function takes as a parameter the retry context object and returns nothing. + It may be used to detect retries and log context information. + ''' + + __metaclass__ = ABCMeta + + def __init__(self, connection_params): + ''' + :param obj connection_params: The parameters to use to construct the client. + ''' + self.account_name = connection_params.account_name + self.account_key = connection_params.account_key + self.sas_token = connection_params.sas_token + self.token_credential = connection_params.token_credential + self.is_emulated = connection_params.is_emulated + + self.primary_endpoint = connection_params.primary_endpoint + self.secondary_endpoint = connection_params.secondary_endpoint + + protocol = connection_params.protocol + request_session = connection_params.request_session or requests.Session() + socket_timeout = connection_params.socket_timeout or DEFAULT_SOCKET_TIMEOUT + self._httpclient = _HTTPClient( + protocol=protocol, + session=request_session, + timeout=socket_timeout, + ) + + self.retry = ExponentialRetry().retry + self.location_mode = LocationMode.PRIMARY + + self.request_callback = None + self.response_callback = None + self.retry_callback = None + self._X_MS_VERSION = DEFAULT_X_MS_VERSION + self._USER_AGENT_STRING = DEFAULT_USER_AGENT_STRING + + def _update_user_agent_string(self, service_package_version): + self._USER_AGENT_STRING = '{}{} {}'.format(USER_AGENT_STRING_PREFIX, + service_package_version, + USER_AGENT_STRING_SUFFIX) + + @property + def socket_timeout(self): + return self._httpclient.timeout + + @socket_timeout.setter + def socket_timeout(self, value): + self._httpclient.timeout = value + + @property + def protocol(self): + return self._httpclient.protocol + + @protocol.setter + def protocol(self, value): + self._httpclient.protocol = value + + @property + def request_session(self): + return self._httpclient.session + + @request_session.setter + def request_session(self, value): + self._httpclient.session = value + + def set_proxy(self, host, port, user=None, password=None): + ''' + Sets the proxy server host and port for the HTTP CONNECT Tunnelling. + + :param str host: Address of the proxy. Ex: '192.168.0.100' + :param int port: Port of the proxy. Ex: 6000 + :param str user: User for proxy authorization. + :param str password: Password for proxy authorization. + ''' + self._httpclient.set_proxy(host, port, user, password) + + def _get_host_locations(self, primary=True, secondary=False): + locations = {} + if primary: + locations[LocationMode.PRIMARY] = self.primary_endpoint + if secondary: + locations[LocationMode.SECONDARY] = self.secondary_endpoint + return locations + + def _apply_host(self, request, operation_context, retry_context): + if operation_context.location_lock and operation_context.host_location: + # If this is a location locked operation and the location is set, + # override the request location and host_location. + request.host_locations = operation_context.host_location + request.host = list(operation_context.host_location.values())[0] + retry_context.location_mode = list(operation_context.host_location.keys())[0] + elif len(request.host_locations) == 1: + # If only one location is allowed, use that location. + request.host = list(request.host_locations.values())[0] + retry_context.location_mode = list(request.host_locations.keys())[0] + else: + # If multiple locations are possible, choose based on the location mode. + request.host = request.host_locations.get(self.location_mode) + retry_context.location_mode = self.location_mode + + @staticmethod + def extract_date_and_request_id(retry_context): + if getattr(retry_context, 'response', None) is None: + return "" + resp = retry_context.response + + if 'date' in resp.headers and 'x-ms-request-id' in resp.headers: + return str.format("Server-Timestamp={0}, Server-Request-ID={1}", + resp.headers['date'], resp.headers['x-ms-request-id']) + elif 'date' in resp.headers: + return str.format("Server-Timestamp={0}", resp.headers['date']) + elif 'x-ms-request-id' in resp.headers: + return str.format("Server-Request-ID={0}", resp.headers['x-ms-request-id']) + else: + return "" + + def _perform_request(self, request, parser=None, parser_args=None, operation_context=None, expected_errors=None): + ''' + Sends the request and return response. Catches HTTPError and hands it + to error handler + ''' + operation_context = operation_context or _OperationContext() + retry_context = RetryContext() + retry_context.is_emulated = self.is_emulated + + # if request body is a stream, we need to remember its current position in case retries happen + if hasattr(request.body, 'read'): + try: + retry_context.body_position = request.body.tell() + except (AttributeError, UnsupportedOperation): + # if body position cannot be obtained, then retries will not work + pass + + # Apply the appropriate host based on the location mode + self._apply_host(request, operation_context, retry_context) + + # Apply common settings to the request + _update_request(request, self._X_MS_VERSION, self._USER_AGENT_STRING) + client_request_id_prefix = str.format("Client-Request-ID={0}", request.headers['x-ms-client-request-id']) + + while True: + try: + try: + # Execute the request callback + if self.request_callback: + self.request_callback(request) + + # Add date and auth after the callback so date doesn't get too old and + # authentication is still correct if signed headers are added in the request + # callback. This also ensures retry policies with long back offs + # will work as it resets the time sensitive headers. + _add_date_header(request) + + try: + # request can be signed individually + self.authentication.sign_request(request) + except AttributeError: + # session can also be signed + self.request_session = self.authentication.signed_session(self.request_session) + + # Set the request context + retry_context.request = request + + # Log the request before it goes out + logger.info("%s Outgoing request: Method=%s, Path=%s, Query=%s, Headers=%s.", + client_request_id_prefix, + request.method, + request.path, + request.query, + str(request.headers).replace('\n', '')) + + # Perform the request + response = self._httpclient.perform_request(request) + + # Execute the response callback + if self.response_callback: + self.response_callback(response) + + # Set the response context + retry_context.response = response + + # Log the response when it comes back + logger.info("%s Receiving Response: " + "%s, HTTP Status Code=%s, Message=%s, Headers=%s.", + client_request_id_prefix, + self.extract_date_and_request_id(retry_context), + response.status, + response.message, + str(response.headers).replace('\n', '')) + + # Parse and wrap HTTP errors in AzureHttpError which inherits from AzureException + if response.status >= 300: + # This exception will be caught by the general error handler + # and raised as an azure http exception + _http_error_handler( + HTTPError(response.status, response.message, response.headers, response.body)) + + # Parse the response + if parser: + if parser_args: + args = [response] + args.extend(parser_args) + return parser(*args) + else: + return parser(response) + else: + return + except AzureException as ex: + retry_context.exception = ex + raise ex + except Exception as ex: + retry_context.exception = ex + if sys.version_info >= (3,): + # Automatic chaining in Python 3 means we keep the trace + raise AzureException(ex.args[0]) + else: + # There isn't a good solution in 2 for keeping the stack trace + # in general, or that will not result in an error in 3 + # However, we can keep the previous error type and message + # TODO: In the future we will log the trace + msg = "" + if len(ex.args) > 0: + msg = ex.args[0] + raise AzureException('{}: {}'.format(ex.__class__.__name__, msg)) + + except AzureException as ex: + # only parse the strings used for logging if logging is at least enabled for CRITICAL + if logger.isEnabledFor(logging.CRITICAL): + exception_str_in_one_line = str(ex).replace('\n', '') + status_code = retry_context.response.status if retry_context.response is not None else 'Unknown' + timestamp_and_request_id = self.extract_date_and_request_id(retry_context) + + # if the http error was expected, we should short-circuit + if isinstance(ex, AzureHttpError) and expected_errors is not None and ex.error_code in expected_errors: + logger.info("%s Received expected http error: " + "%s, HTTP status code=%s, Exception=%s.", + client_request_id_prefix, + timestamp_and_request_id, + status_code, + exception_str_in_one_line) + raise ex + + logger.info("%s Operation failed: checking if the operation should be retried. " + "Current retry count=%s, %s, HTTP status code=%s, Exception=%s.", + client_request_id_prefix, + retry_context.count if hasattr(retry_context, 'count') else 0, + timestamp_and_request_id, + status_code, + exception_str_in_one_line) + + # Decryption failures (invalid objects, invalid algorithms, data unencrypted in strict mode, etc) + # will not be resolved with retries. + if str(ex) == _ERROR_DECRYPTION_FAILURE: + logger.error("%s Encountered decryption failure: this cannot be retried. " + "%s, HTTP status code=%s, Exception=%s.", + client_request_id_prefix, + timestamp_and_request_id, + status_code, + exception_str_in_one_line) + raise ex + + # Determine whether a retry should be performed and if so, how + # long to wait before performing retry. + retry_interval = self.retry(retry_context) + if retry_interval is not None: + # Execute the callback + if self.retry_callback: + self.retry_callback(retry_context) + + logger.info( + "%s Retry policy is allowing a retry: Retry count=%s, Interval=%s.", + client_request_id_prefix, + retry_context.count, + retry_interval) + + # Sleep for the desired retry interval + sleep(retry_interval) + else: + logger.error("%s Retry policy did not allow for a retry: " + "%s, HTTP status code=%s, Exception=%s.", + client_request_id_prefix, + timestamp_and_request_id, + status_code, + exception_str_in_one_line) + raise ex + finally: + # If this is a location locked operation and the location is not set, + # this is the first request of that operation. Set the location to + # be used for subsequent requests in the operation. + if operation_context.location_lock and not operation_context.host_location: + # note: to cover the emulator scenario, the host_location is grabbed + # from request.host_locations(which includes the dev account name) + # instead of request.host(which at this point no longer includes the dev account name) + operation_context.host_location = { + retry_context.location_mode: request.host_locations[retry_context.location_mode]} diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/tokencredential.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/tokencredential.py new file mode 100644 index 000000000000..4d724ef06ad1 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/tokencredential.py @@ -0,0 +1,48 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +import requests + + +class TokenCredential(object): + """ + Represents a token credential that is used to authorize HTTPS requests. + The token can be updated by the user. + + :ivar str token: + The authorization token. It can be set by the user at any point in a thread-safe way. + """ + + def __init__(self, initial_value=None): + """ + :param initial_value: initial value for the token. + """ + self.token = initial_value + + def signed_session(self, session=None): + """ + Sign requests session with the token. This method is called every time a request is going on the wire. + The user is responsible for updating the token with the preferred tool/SDK. + In general there are two options: + - override this method to update the token in a preferred way and set Authorization header on session + - not override this method, and have a timer that triggers periodically to update the token on this class + + The second option is recommended as it tends to be more performance-friendly. + + :param session: The session to configure for authentication + :type session: requests.Session + :rtype: requests.Session + """ + session = session or requests.Session() + session.headers['Authorization'] = "Bearer {}".format(self.token) + + return session + + def token(self, new_value): + """ + :param new_value: new value to be set as the token. + """ + self.token = new_value \ No newline at end of file diff --git a/sdk/eventhub/azure-eventhubs/setup.py b/sdk/eventhub/azure-eventhubs/setup.py index f3cc586d2bd2..15b2ec62da43 100644 --- a/sdk/eventhub/azure-eventhubs/setup.py +++ b/sdk/eventhub/azure-eventhubs/setup.py @@ -78,7 +78,9 @@ 'uamqp~=1.2.0', 'msrestazure>=0.4.32,<2.0.0', 'azure-common~=1.1', - 'azure-storage-blob~=1.3', + 'cryptography', + 'python-dateutil', + 'requests' # 'azure-core>=0.0.1', # will add back here and remove from dev_requirements.txt after azure core is released ], extras_require={