From 0b1a84dc691d5683f73df496d84bdf29ba17497f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcus=20Sch=C3=A4fer?= Date: Tue, 23 Jun 2015 15:30:23 +0200 Subject: [PATCH] Added Azure files support to the SDK Starting with share listing/creation and deletion --- azure/__init__.py | 4 +- azure/files/__init__.py | 134 ++++++++++++++++++++++++ azure/files/filesservice.py | 197 ++++++++++++++++++++++++++++++++++++ setup.py | 1 + tests/test_filesservice.py | 103 +++++++++++++++++++ 5 files changed, 438 insertions(+), 1 deletion(-) create mode 100644 azure/files/__init__.py create mode 100644 azure/files/filesservice.py create mode 100644 tests/test_filesservice.py diff --git a/azure/__init__.py b/azure/__init__.py index 71a546ed2def..919c2b82d3a1 100644 --- a/azure/__init__.py +++ b/azure/__init__.py @@ -43,10 +43,11 @@ # constants __author__ = 'Microsoft Corp. ' -__version__ = '0.11.1' +__version__ = '0.11.2' # Live ServiceClient URLs BLOB_SERVICE_HOST_BASE = '.blob.core.windows.net' +FILES_SERVICE_HOST_BASE = '.file.core.windows.net' QUEUE_SERVICE_HOST_BASE = '.queue.core.windows.net' TABLE_SERVICE_HOST_BASE = '.table.core.windows.net' SERVICE_BUS_HOST_BASE = '.servicebus.windows.net' @@ -54,6 +55,7 @@ # Development ServiceClient URLs DEV_BLOB_HOST = '127.0.0.1:10000' +DEV_FILES_HOST = '127.0.0.1:10003' # guessing on the port value, please advise DEV_QUEUE_HOST = '127.0.0.1:10001' DEV_TABLE_HOST = '127.0.0.1:10002' diff --git a/azure/files/__init__.py b/azure/files/__init__.py new file mode 100644 index 000000000000..87b890a1384d --- /dev/null +++ b/azure/files/__init__.py @@ -0,0 +1,134 @@ +#------------------------------------------------------------------------- +# Copyright (c) 2015 SUSE Linux GmbH. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#-------------------------------------------------------------------------- +import sys +import threading +import types + +from datetime import datetime +from dateutil import parser +from dateutil.tz import tzutc +from time import sleep +from azure import (WindowsAzureData, + WindowsAzureError, + METADATA_NS, + url_quote, + xml_escape, + _create_entry, + _decode_base64_to_text, + _decode_base64_to_bytes, + _encode_base64, + _general_error_handler, + _list_of, + _parse_response_for_dict, + _sign_string, + _unicode_type, + _ERROR_CANNOT_SERIALIZE_VALUE_TO_ENTITY, + _etree_entity_feed_namespaces, + _make_etree_ns_attr_name, + _get_etree_tag_name_without_ns, + _get_etree_text, + ETree, + _ETreeXmlToObject, + ) + +# x-ms-version for files service. +X_MS_VERSION = '2014-02-14' + +def _update_storage_header(request): + ''' add additional headers for storage request. ''' + if request.body: + assert isinstance(request.body, bytes) + + # if it is PUT, POST, MERGE, DELETE, need to add content-length to header. + if request.method in ['PUT', 'POST', 'MERGE', 'DELETE']: + request.headers.append(('Content-Length', str(len(request.body)))) + + # append addtional headers base on the service + request.headers.append(('x-ms-version', X_MS_VERSION)) + + # append x-ms-meta name, values to header + for name, value in request.headers: + if 'x-ms-meta-name-values' in name and value: + for meta_name, meta_value in value.items(): + request.headers.append(('x-ms-meta-' + meta_name, meta_value)) + request.headers.remove((name, value)) + break + return request + +def _update_storage_files_header(request, authentication): + ''' add/update additional headers for storage files request. ''' + + request = _update_storage_header(request) + current_time = datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT') + request.headers.append(('x-ms-date', current_time)) + request.headers.append( + ('Content-Type', 'application/octet-stream Charset=UTF-8')) + authentication.sign_request(request) + + return request.headers + + +class EnumResultsBase(object): + + ''' base class for EnumResults. ''' + + def __init__(self): + self.prefix = u'' + self.marker = u'' + self.max_results = 0 + self.next_marker = u'' + + +class ShareEnumResults(EnumResultsBase): + + ''' Files share list. ''' + + def __init__(self): + EnumResultsBase.__init__(self) + self.shares = _list_of(Share) + + def __iter__(self): + return iter(self.shares) + + def __len__(self): + return len(self.shares) + + def __getitem__(self, index): + return self.shares[index] + + +class Properties(WindowsAzureData): + + ''' Files ahre's properties class. ''' + + def __init__(self): + self.last_modified = u'' + self.etag = u'' + + +class Share(WindowsAzureData): + + ''' Share container class. ''' + + def __init__(self): + self.name = u'' + self.url = u'' + self.properties = Properties() + self.metadata = {} + + + +# make these available just from files. +from azure.files.filesservice import FilesService diff --git a/azure/files/filesservice.py b/azure/files/filesservice.py new file mode 100644 index 000000000000..f4ad66f85faa --- /dev/null +++ b/azure/files/filesservice.py @@ -0,0 +1,197 @@ +#------------------------------------------------------------------------- +# Copyright (c) 2015 SUSE Linux GmbH. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#-------------------------------------------------------------------------- +from azure import ( + WindowsAzureError, + FILES_SERVICE_HOST_BASE, + DEFAULT_HTTP_TIMEOUT, + DEV_FILES_HOST, + _ERROR_VALUE_NEGATIVE, + _ERROR_PAGE_BLOB_SIZE_ALIGNMENT, + _convert_class_to_xml, + _dont_fail_not_exist, + _dont_fail_on_exist, + _encode_base64, + _get_request_body, + _get_request_body_bytes_only, + _int_or_none, + _parse_response_for_dict, + _parse_response_for_dict_filter, + _parse_response_for_dict_prefix, + _str, + _str_or_none, + _update_request_uri_query_local_storage, + _validate_type_bytes, + _validate_not_none, + _ETreeXmlToObject, +) + +from azure.storage.storageclient import _StorageClient + +from azure.http import HTTPRequest + +from azure.storage import ( + StorageSASAuthentication, + StorageSharedKeyAuthentication, + StorageNoAuthentication +) + +from azure.files import ( + Share, + ShareEnumResults, + _update_storage_files_header +) + +class FilesService(_StorageClient): + + ''' + This is the main class managing Files resources. + ''' + + def __init__( + self, account_name=None, account_key=None, protocol='https', + host_base=FILES_SERVICE_HOST_BASE, dev_host=DEV_FILES_HOST, + timeout=DEFAULT_HTTP_TIMEOUT, sas_token=None + ): + ''' + account_name: + your storage account name, required for all operations. + account_key: + your storage account key, required for all operations. + protocol: + Optional. Protocol. Defaults to https. + host_base: + Optional. Live host base url. Defaults to Azure url. Override this + for on-premise. + dev_host: + Optional. Dev host url. Defaults to localhost. + timeout: + Optional. Timeout for the http request, in seconds. + sas_token: + Optional. Token to use to authenticate with shared access signature. + ''' + super(FilesService, self).__init__( + account_name, account_key, protocol, + host_base, dev_host, timeout, sas_token + ) + + if self.account_key: + self.authentication = StorageSharedKeyAuthentication( + self.account_name, + self.account_key, + ) + elif self.sas_token: + self.authentication = StorageSASAuthentication(self.sas_token) + else: + self.authentication = StorageNoAuthentication() + + def list_shares( + self, prefix=None, marker=None, maxresults=None, include=None + ): + ''' + The List Shares operation returns a list of the shares under the + specified account. + + prefix: + Optional. Filters the results to return only shares whose names + begin with the specified prefix. + marker: + Optional. A string value that identifies the portion of the list to + be returned with the next list operation. + maxresults: + Optional. Specifies the maximum number of shares to return. + include: + Optional. Include this parameter to specify that the share's + metadata be returned as part of the response body. set this + parameter to string 'metadata' to get shares's metadata. + ''' + request = HTTPRequest() + request.method = 'GET' + request.host = self._get_host() + request.path = '/?comp=list' + request.query = [ + ('prefix', _str_or_none(prefix)), + ('marker', _str_or_none(marker)), + ('maxresults', _int_or_none(maxresults)), + ('include', _str_or_none(include)) + ] + request.path, request.query = _update_request_uri_query_local_storage( + request, self.use_local_storage + ) + request.headers = _update_storage_files_header( + request, self.authentication + ) + response = self._perform_request(request) + + return _ETreeXmlToObject.parse_enum_results_list( + response, ShareEnumResults, "Shares", Share) + + def create_share(self, share_name, x_ms_meta_name_values=None): + ''' + The Create Share operation creates a new share under the specified + account. If the share with the same name already exists, the operation + fails. The share resource includes metadata and properties for that + share. It does not include a list of the files contained by the share. + + share_name: + Name of the share to create + x_ms_meta_name_values: + Optional. A dict with name_value pairs to associate with the + share as metadata. Example:{'Category':'test'} + ''' + _validate_not_none('share_name', share_name) + request = HTTPRequest() + request.method = 'PUT' + request.host = self._get_host() + request.path = '/' + _str(share_name) + '?restype=share' + request.headers = [ + ('x-ms-meta-name-values', x_ms_meta_name_values) + ] + request.path, request.query = _update_request_uri_query_local_storage( + request, self.use_local_storage + ) + request.headers = _update_storage_files_header( + request, self.authentication + ) + self._perform_request(request) + return True + + def delete_share(self, share_name, x_ms_meta_name_values=None): + ''' + The Delete Share operation marks the specified share for deletion. + The share and any files contained within it are later deleted + during garbage collection. + + share_name: + Name of the share to create + x_ms_meta_name_values: + Optional. A dict with name_value pairs to associate with the + share as metadata. Example:{'Category':'test'} + ''' + _validate_not_none('share_name', share_name) + request = HTTPRequest() + request.method = 'DELETE' + request.host = self._get_host() + request.path = '/' + _str(share_name) + '?restype=share' + request.headers = [ + ('x-ms-meta-name-values', x_ms_meta_name_values) + ] + request.path, request.query = _update_request_uri_query_local_storage( + request, self.use_local_storage + ) + request.headers = _update_storage_files_header( + request, self.authentication + ) + self._perform_request(request) + return True diff --git a/setup.py b/setup.py index 06d21e6186bb..70da0e958b3e 100644 --- a/setup.py +++ b/setup.py @@ -51,6 +51,7 @@ 'Programming Language :: Python :: 3.4', 'License :: OSI Approved :: Apache Software License'], packages=['azure', + 'azure.files', 'azure.http', 'azure.servicebus', 'azure.storage', diff --git a/tests/test_filesservice.py b/tests/test_filesservice.py new file mode 100644 index 000000000000..4b1ed20404d1 --- /dev/null +++ b/tests/test_filesservice.py @@ -0,0 +1,103 @@ +# coding: utf-8 +#------------------------------------------------------------------------- +# Copyright (c) 2015 SUSE Linux GmbH. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#-------------------------------------------------------------------------- +import base64 +import datetime +import os +import random +import sys +import time +import unittest +if sys.version_info < (3,): + from httplib import HTTPConnection +else: + from http.client import HTTPConnection + +from azure.files.filesservice import FilesService + +from azure.storage.storageclient import ( + AZURE_STORAGE_ACCESS_KEY, + AZURE_STORAGE_ACCOUNT, + EMULATED, + DEV_ACCOUNT_NAME, + DEV_ACCOUNT_KEY, + ) + +from util import ( + AzureTestCase, + credentials, + getUniqueName, + set_service_options, + ) + + +#------------------------------------------------------------------------------ + + +class FilesServiceTest(AzureTestCase): + + def setUp(self): + self.fs = FilesService( + credentials.getStorageServicesName(), + credentials.getStorageServicesKey() + ) + + self.sharename = 'testshare' + self.timeout = 1 + self.trycount = 60 + + remote_storage_service_name = credentials.getRemoteStorageServicesName() + remote_storage_service_key = credentials.getRemoteStorageServicesKey() + if not remote_storage_service_key or not remote_storage_service_name: + print("Remote Storage Account not configured. Add " \ + "'remotestorageserviceskey' and 'remotestorageservicesname'" \ + " to windowsazurecredentials.json to test functionality " \ + "involving multiple storage accounts.") + + def tearDown(self): + return super(FilesServiceTest, self).tearDown() + + def test_create_and_list_share(self): + try_count = 0 + while try_count < self.trycount: + try: + created = self.fs.create_share(self.sharename) + self.assertTrue(created) + try_count = self.trycount + except: + time.sleep(self.timeout) + try_count = try_count + 1 + share_name = None + try_count = 0 + while try_count < self.trycount: + try: + for share in self.fs.list_shares(): + if share.name == self.sharename: + share_name = share.name + try_count = self.trycount + break + except: + time.sleep(self.timeout) + try_count = try_count + 1 + self.assertEqual(share_name, self.sharename) + + def test_delete_share(self): + deleted = self.fs.delete_share(self.sharename) + self.assertTrue(deleted) + + +#------------------------------------------------------------------------------ +if __name__ == '__main__': + unittest.main()