[Python] support BucketByteBuffer and BucketUUID

python/iceberg/api/transforms/bucket.py

@@ -28,6 +28,7 @@
                            Operation)
 from ..types.types import (IntegerType,
                            TypeID)
+from ...api.types.conversions import Conversions
 
 
 class Bucket(Transform):
@@ -69,7 +70,7 @@ def __key(self):
         return Bucket.__class__, self.n
 
     def __repr__(self):
-        return "Bucket(n)" % self.n
+        return "Bucket[%s]" % self.n
 
     def __str__(self):
         return "bucket[%s]" % self.n
@@ -167,14 +168,24 @@ def can_transform(self, type_var):
 
 class BucketByteBuffer(Bucket):
     def __init__(self, n):
-        # super(BucketByteBuffer, self).__init__(n)
-        raise NotImplementedError()
+        super(BucketByteBuffer, self).__init__(n)
+
+    def hash(self, value):
+        return Bucket.MURMUR3.hash(value)
+
+    def can_transform(self, type_var):
+        return type_var.type_id in [TypeID.BINARY, TypeID.FIXED]
 
 
 class BucketUUID(Bucket):
     def __init__(self, n):
-        # super(BucketUUID, self).__init__(n)
-        raise NotImplementedError()
+        super(BucketUUID, self).__init__(n)
+
+    def hash(self, value):
+        return Bucket.MURMUR3.hash(Conversions.to_byte_buffer(TypeID.UUID, value))
+
+    def can_transform(self, type_var):
+        return type_var.type_id == TypeID.UUID
 
 
 def to_bytes(n, length, byteorder='big'):

python/tests/api/test_partition_spec.py

@@ -30,9 +30,9 @@
 from tests.api.test_helpers import TestHelpers
 
 
-class TestConversions(unittest.TestCase):
+class TestPartitionSpec(unittest.TestCase):
 
-    def test_transforms(self):
+    def test_partition_spec(self):
         schema = Schema(NestedField.required(1, "i", IntegerType.get()),
                         NestedField.required(2, "l", LongType.get()),
                         NestedField.required(3, "d", DateType.get()),
@@ -60,10 +60,9 @@ def test_transforms(self):
                  PartitionSpec.builder_for(schema).bucket("ts", 128).build(),
                  PartitionSpec.builder_for(schema).bucket("dec", 128).build(),
                  PartitionSpec.builder_for(schema).bucket("s", 128).build(),
-                 # todo support them
-                 # PartitionSpec.builder_for(schema).bucket("u", 128).build(),
-                 # PartitionSpec.builder_for(schema).bucket("f", 128).build(),
-                 # PartitionSpec.builder_for(schema).bucket("b", 128).build(),
+                 PartitionSpec.builder_for(schema).bucket("u", 128).build(),
+                 PartitionSpec.builder_for(schema).bucket("f", 128).build(),
+                 PartitionSpec.builder_for(schema).bucket("b", 128).build(),
                  PartitionSpec.builder_for(schema).year("d").build(),
                  PartitionSpec.builder_for(schema).month("d").build(),
                  PartitionSpec.builder_for(schema).day("d").build(),
@@ -75,9 +74,8 @@ def test_transforms(self):
                  PartitionSpec.builder_for(schema).truncate("l", 10).build(),
                  PartitionSpec.builder_for(schema).truncate("dec", 10).build(),
                  PartitionSpec.builder_for(schema).truncate("s", 10).build(),
-                 # todo support them
-                 # PartitionSpec.builder_for(schema).add_without_field_id(6, "dec_unsupported", "unsupported").build(),
-                 # PartitionSpec.builder_for(schema).add(6, 1111, "dec_unsupported", "unsupported").build(),
+                 PartitionSpec.builder_for(schema).add_without_field_id(6, "dec_unsupported", "unsupported").build(),
+                 PartitionSpec.builder_for(schema).add(6, 1111, "dec_unsupported", "unsupported").build(),
                  ]
 
         for spec in specs:

python/tests/api/transforms/test_bucket.py

@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import decimal
+import unittest
+import uuid
+
+from iceberg.api.transforms import Transforms
+from iceberg.api.types import (BinaryType,
+                               DateType,
+                               DecimalType,
+                               FixedType,
+                               IntegerType,
+                               LongType,
+                               StringType,
+                               TimestampType,
+                               TimeType,
+                               UUIDType)
+
+
+class TestBucket(unittest.TestCase):
+
+    def test_bucket_hash(self):
+        buckets = [
+            [Transforms.bucket(IntegerType.get(), 100), 34, 2017239379],
+            [Transforms.bucket(LongType.get(), 100), 34, 2017239379],
+            [Transforms.bucket(DateType.get(), 100), 17486, -653330422],
+            [Transforms.bucket(TimeType.get(), 100), 81068000000, -662762989],
+            [Transforms.bucket(TimestampType.without_timezone(), 100), 1510871468000000, -2047944441],
+            [Transforms.bucket(DecimalType.of(9, 2), 100), decimal.Decimal("14.20"), -500754589],
+            [Transforms.bucket(StringType.get(), 100), "iceberg", 1210000089],
+            [Transforms.bucket(UUIDType.get(), 100), uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7"), 1488055340],
+            [Transforms.bucket(FixedType.of_length(3), 128), b'foo', -156908512],
+            [Transforms.bucket(BinaryType.get(), 128), b'\x00\x01\x02\x03', -188683207]
+        ]
+
+        for bucket in buckets:
+            self.assertEqual(bucket[2], bucket[0].hash(bucket[1]))