diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_consistent_hash_ring.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_consistent_hash_ring.py deleted file mode 100644 index 89f5a10652c9..000000000000 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/_consistent_hash_ring.py +++ /dev/null @@ -1,129 +0,0 @@ -# The MIT License (MIT) -# Copyright (c) 2014 Microsoft Corporation - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -"""Internal class for consistent hash ring implementation in the Azure Cosmos database service. -""" - -from struct import unpack -import six -from six.moves import xrange - -from . import _partition - - -class ConsistentHashRing(object): - """The ConsistentHashRing class implements a consistent hash ring using the - hash generator specified. - """ - - def __init__(self, collection_links, partitions_per_node, hash_generator): - """ - :param list collection_links: - The links of collections participating in partitioning. - :param int partitions_per_node: - The partitions per node. - :param HashGenerator hash_generator: - The hash generator to be used for hashing algorithm. - """ - if collection_links is None: - raise ValueError("collection_links is None.") - - if partitions_per_node <= 0: - raise ValueError("The partitions per node must greater than 0.") - - if hash_generator is None: - raise ValueError("hash_generator is None.") - - self.collection_links = collection_links - self.hash_generator = hash_generator - - self.partitions = self._ConstructPartitions(self.collection_links, partitions_per_node) - - def GetCollectionNode(self, partition_key): - """Gets the SelfLink/ID based link of the collection node that maps to - the partition key based on the hashing algorithm used for finding the - node in the ring. - - :param str partition_key: - The partition key to be used for finding the node in the ring. - :return: The name of the collection mapped to that partition. - :rtype: str - """ - if partition_key is None: - raise ValueError("partition_key is None or empty.") - - partition_number = self._FindPartition(self._GetBytes(partition_key)) - return self.partitions[partition_number].GetNode() - - def _ConstructPartitions(self, collection_links, partitions_per_node): - """Constructs the partitions in the consistent ring by assigning them to - collection nodes using the hashing algorithm and then finally sorting - the partitions based on the hash value. - """ - collections_node_count = len(collection_links) - partitions = [_partition.Partition() for _ in xrange(0, partitions_per_node * collections_node_count)] - - index = 0 - for collection_node in collection_links: - hash_value = self.hash_generator.ComputeHash(self._GetBytes(collection_node)) - for _ in xrange(0, partitions_per_node): - partitions[index] = _partition.Partition(hash_value, collection_node) - index += 1 - hash_value = self.hash_generator.ComputeHash(hash_value) - - partitions.sort() - return partitions - - def _FindPartition(self, key): - """Finds the partition from the byte array representation of the partition key. - """ - hash_value = self.hash_generator.ComputeHash(key) - return self._LowerBoundSearch(self.partitions, hash_value) - - def _GetSerializedPartitionList(self): - """Gets the serialized version of the ConsistentRing. - - Added this helper for the test code. - """ - partition_list = list() - - for part in self.partitions: - partition_list.append((part.node, unpack("> 16 - h = (h * 0x85EBCA6B) & 0xFFFFFFFF - h ^= h >> 13 - h = (h * 0xC2B2AE35) & 0xFFFFFFFF - h ^= h >> 16 - return h - - length = len(key) - nblocks = int(length / 4) - - h1 = seed - - c1 = 0xCC9E2D51 - c2 = 0x1B873593 - - # body - for block_start in xrange(0, nblocks * 4, 4): - k1 = ( - key[block_start + 3] << 24 - | key[block_start + 2] << 16 - | key[block_start + 1] << 8 - | key[block_start + 0] - ) - - k1 = c1 * k1 & 0xFFFFFFFF - k1 = (k1 << 15 | k1 >> 17) & 0xFFFFFFFF # inlined ROTL32 - k1 = (c2 * k1) & 0xFFFFFFFF - - h1 ^= k1 - h1 = (h1 << 13 | h1 >> 19) & 0xFFFFFFFF # inlined _ROTL32 - h1 = (h1 * 5 + 0xE6546B64) & 0xFFFFFFFF - - # tail - tail_index = nblocks * 4 - k1 = 0 - tail_size = length & 3 - - if tail_size >= 3: - k1 ^= key[tail_index + 2] << 16 - if tail_size >= 2: - k1 ^= key[tail_index + 1] << 8 - if tail_size >= 1: - k1 ^= key[tail_index + 0] - - if tail_size != 0: - k1 = (k1 * c1) & 0xFFFFFFFF - k1 = (k1 << 15 | k1 >> 17) & 0xFFFFFFFF # _ROTL32 - k1 = (k1 * c2) & 0xFFFFFFFF - h1 ^= k1 - - return fmix(h1 ^ length) diff --git a/sdk/cosmos/azure-cosmos/test/test_config.py b/sdk/cosmos/azure-cosmos/test/test_config.py index 1f0cc69cd8b6..32be96cc5d42 100644 --- a/sdk/cosmos/azure-cosmos/test/test_config.py +++ b/sdk/cosmos/azure-cosmos/test/test_config.py @@ -39,7 +39,7 @@ class _test_config(object): #[SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Cosmos DB Emulator Key")] masterKey = os.getenv('ACCOUNT_KEY', 'C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==') - host = os.getenv('ACCOUNT_HOST', 'https://localhost:443/') + host = os.getenv('ACCOUNT_HOST', 'https://localhost:8081/') connection_str = os.getenv('ACCOUNT_CONNECTION_STR', 'AccountEndpoint={};AccountKey={};'.format(host, masterKey)) connectionPolicy = documents.ConnectionPolicy() diff --git a/sdk/cosmos/azure-cosmos/test/test_crud.py b/sdk/cosmos/azure-cosmos/test/test_crud.py index d9a650a9dfbf..d8e4480381a3 100644 --- a/sdk/cosmos/azure-cosmos/test/test_crud.py +++ b/sdk/cosmos/azure-cosmos/test/test_crud.py @@ -44,11 +44,9 @@ from azure.core import MatchConditions from azure.core.exceptions import AzureError, ServiceResponseError from azure.core.pipeline.transport import RequestsTransport, RequestsTransportResponse -from azure.cosmos import _consistent_hash_ring import azure.cosmos.documents as documents import azure.cosmos.exceptions as exceptions from azure.cosmos.http_constants import HttpHeaders, StatusCodes, SubStatusCodes -import azure.cosmos._murmur_hash as _murmur_hash import test_config import azure.cosmos._base as base import azure.cosmos.cosmos_client as cosmos_client @@ -917,59 +915,6 @@ def test_document_crud(self): replaced_document['id'], replaced_document['id']) - def test_murmur_hash(self): - str = 'afdgdd' - bytes = bytearray(str, encoding='utf-8') - - hash_value = _murmur_hash.MurmurHash._ComputeHash(bytes) - self.assertEqual(1099701186, hash_value) - - num = 374.0 - bytes = bytearray(pack('d', num)) - - hash_value = _murmur_hash.MurmurHash._ComputeHash(bytes) - self.assertEqual(3717946798, hash_value) - - self._validate_bytes("", 0x1B873593, bytearray(b'\xEE\xA8\xA2\x67'), 1738713326) - self._validate_bytes("1", 0xE82562E4, bytearray(b'\xD0\x92\x24\xED'), 3978597072) - self._validate_bytes("00", 0xB4C39035, bytearray(b'\xFA\x09\x64\x1B'), 459540986) - self._validate_bytes("eyetooth", 0x8161BD86, bytearray(b'\x98\x62\x1C\x6F'), 1864131224) - self._validate_bytes("acid", 0x4DFFEAD7, bytearray(b'\x36\x92\xC0\xB9'), 3116405302) - self._validate_bytes("elevation", 0x1A9E1828, bytearray(b'\xA9\xB6\x40\xDF'), 3745560233) - self._validate_bytes("dent", 0xE73C4579, bytearray(b'\xD4\x59\xE1\xD3'), 3554761172) - self._validate_bytes("homeland", 0xB3DA72CA, bytearray(b'\x06\x4D\x72\xBB'), 3144830214) - self._validate_bytes("glamor", 0x8078A01B, bytearray(b'\x89\x89\xA2\xA7'), 2812447113) - self._validate_bytes("flags", 0x4D16CD6C, bytearray(b'\x52\x87\x66\x02'), 40273746) - self._validate_bytes("democracy", 0x19B4FABD, bytearray(b'\xE4\x55\xD6\xB0'), 2966836708) - self._validate_bytes("bumble", 0xE653280E, bytearray(b'\xFE\xD7\xC3\x0C'), 214161406) - self._validate_bytes("catch", 0xB2F1555F, bytearray(b'\x98\x4B\xB6\xCD'), 3451276184) - self._validate_bytes("omnomnomnivore", 0x7F8F82B0, bytearray(b'\x38\xC4\xCD\xFF'), 4291675192) - self._validate_bytes("The quick brown fox jumps over the lazy dog", 0x4C2DB001, bytearray(b'\x6D\xAB\x8D\xC9'), - 3381504877) - - def _validate_bytes(self, str, seed, expected_hash_bytes, expected_value): - hash_value = _murmur_hash.MurmurHash._ComputeHash(bytearray(str, encoding='utf-8'), seed) - bytes = bytearray(pack('I', hash_value)) - self.assertEqual(expected_value, hash_value) - self.assertEqual(expected_hash_bytes, bytes) - - def test_get_bytes(self): - actual_bytes = _consistent_hash_ring.ConsistentHashRing._GetBytes("documentdb") - expected_bytes = bytearray(b'\x64\x6F\x63\x75\x6D\x65\x6E\x74\x64\x62') - self.assertEqual(expected_bytes, actual_bytes) - - actual_bytes = _consistent_hash_ring.ConsistentHashRing._GetBytes("azure") - expected_bytes = bytearray(b'\x61\x7A\x75\x72\x65') - self.assertEqual(expected_bytes, actual_bytes) - - actual_bytes = _consistent_hash_ring.ConsistentHashRing._GetBytes("json") - expected_bytes = bytearray(b'\x6A\x73\x6F\x6E') - self.assertEqual(expected_bytes, actual_bytes) - - actual_bytes = _consistent_hash_ring.ConsistentHashRing._GetBytes("nosql") - expected_bytes = bytearray(b'\x6E\x6F\x73\x71\x6C') - self.assertEqual(expected_bytes, actual_bytes) - def test_document_upsert(self): # create database created_db = self.databaseForTest