Use HDF C impl

Author: martindurant

numcodecs/_fletcher.c

@@ -0,0 +1,43 @@
+#include <stdint.h>
+#include <stddef.h>
+
+// https://github.com/Unidata/netcdf-c/blob/8eb71290eb9360dcfd4955ba94759ba8d02c40a9/plugins/H5checksum.c
+
+
+uint32_t H5_checksum_fletcher32(const void *_data, size_t _len)
+{
+    const uint8_t *data = (const uint8_t *)_data;  /* Pointer to the data to be summed */
+    size_t len = _len / 2;      /* Length in 16-bit words */
+    uint32_t sum1 = 0, sum2 = 0;
+
+
+    /* Compute checksum for pairs of bytes */
+    /* (the magic "360" value is is the largest number of sums that can be
+     *  performed without numeric overflow)
+     */
+    while (len) {
+        size_t tlen = len > 360 ? 360 : len;
+        len -= tlen;
+        do {
+            sum1 += (uint32_t)(((uint16_t)data[0]) << 8) | ((uint16_t)data[1]);
+            data += 2;
+            sum2 += sum1;
+        } while (--tlen);
+        sum1 = (sum1 & 0xffff) + (sum1 >> 16);
+        sum2 = (sum2 & 0xffff) + (sum2 >> 16);
+    }
+
+    /* Check for odd # of bytes */
+    if(_len % 2) {
+        sum1 += (uint32_t)(((uint16_t)*data) << 8);
+        sum2 += sum1;
+        sum1 = (sum1 & 0xffff) + (sum1 >> 16);
+        sum2 = (sum2 & 0xffff) + (sum2 >> 16);
+    } /* end if */
+
+    /* Second reduction step to reduce sums to 16 bits */
+    sum1 = (sum1 & 0xffff) + (sum1 >> 16);
+    sum2 = (sum2 & 0xffff) + (sum2 >> 16);
+
+    return (sum2 << 16) | sum1;
+} /* end H5_checksum_fletcher32() */

numcodecs/fletcher32.pyx

@@ -11,19 +11,8 @@ from numcodecs.compat import ensure_contiguous_ndarray
 
 from libc.stdint cimport uint8_t, uint16_t, uint32_t
 
-cpdef uint32_t fletcher32(const uint16_t[::1] data):
-    cdef:
-        uint32_t sum1 = 0
-        uint32_t sum2 = 0
-        int index
-        int size = data.shape[0]
-
-    for index in range(0, size):
-        sum1 = (sum1 + data[index]) % 0xffff
-        sum2 = (sum2 + sum1) % 0xffff
-
-    return (sum2 << 16) | sum1
-
+cdef extern from "_fletcher.c":
+    uint32_t H5_checksum_fletcher32(const void *_data, size_t _len)
 
 class Fletcher32(Codec):
     """The fletcher checksum with 16-bit words and 32-bit output
@@ -37,28 +26,20 @@ class Fletcher32(Codec):
     codec_id = "fletcher32"
 
     def encode(self, buf):
-        buf = ensure_contiguous_ndarray(buf).ravel()
-        if len(buf) % 2:
-            # rare, odd size of bytes data only
-            arr = np.frombuffer(buf.tobytes() + b"\x00", dtype="uint16")
-            val = fletcher32(arr)
-        else:
-            val = fletcher32(buf.view('uint16'))
+        buf = ensure_contiguous_ndarray(buf).ravel().view('uint8')
+        cdef const uint8_t[::1] b_ptr = buf
+        val = H5_checksum_fletcher32(&b_ptr[0], buf.nbytes)
         return buf.tobytes() + struct.pack("<I", val)
 
     def decode(self, buf, out=None):
         b = ensure_contiguous_ndarray(buf).view('uint8')
-        if len(buf) % 2:
-            # rare, odd size of bytes data only
-            arr = np.frombuffer(b.tobytes() + b"\x00", dtype="uint16")
-            val = fletcher32(arr)
-        else:
-            val = fletcher32(b[:-4].view('uint16'))
-        found = b[-4:].view('uint32')[0]
+        cdef const uint8_t[::1] b_ptr = b
+        val = H5_checksum_fletcher32(&b_ptr[0], b.nbytes - 4)
+        found = b[-4:].view("<u4")[0]
         if val != found:
             raise ValueError(
-                f"The fletcher32 checksum of the data ({found}) did not"
-                f" match the expected checksum ({val}).\n"
+                f"The fletcher32 checksum of the data ({val}) did not"
+                f" match the expected checksum ({found}).\n"
                 "This could be a sign that the data has been corrupted."
             )
         if out:

numcodecs/tests/test_fletcher32.py

@@ -1,24 +1,15 @@
 import numpy as np
 import pytest
 
-from numcodecs.fletcher32 import Fletcher32, fletcher32
-
-
-@pytest.mark.parametrize("inval,outval", [
-    [b"abcdef", 1448095018],
-    [b"abcdefgh", 3957429649]
-])
-def test_vectors(inval, outval):
-    arr = np.array(list(inval), dtype="uint8").view('uint16')
-    assert fletcher32(arr) == outval
+from numcodecs.fletcher32 import Fletcher32
 
 
 @pytest.mark.parametrize(
     "dtype",
     ["uint8", "int32", "float32"]
 )
 def test_with_data(dtype):
-    data = np.empty(100, dtype=dtype)
+    data = np.arange(100, dtype=dtype)
     f = Fletcher32()
     arr = np.frombuffer(f.decode(f.encode(data)), dtype=dtype)
     assert (arr == data).all()
@@ -34,3 +25,18 @@ def test_error():
         f.decode(enc2)
     assert "fletcher32 checksum" in str(e.value)
 
+
+def test_known():
+    data = (
+        b'\xf04\xfe\x1a\x03\xb2\xb1?^\x99j\xf3\xd6f\xef?\xbbm\x04n'
+        b'\x9a\xdf\xeb?x\x9eIL\xdeW\xc8?A\xef\x88\xa8&\xad\xef?'
+        b'\xf2\xc6a\x01a\xb8\xe8?#&\x96\xabY\xf2\xe7?\xe2Pw\xba\xd0w\xea?'
+        b'\x80\xc5\xf8M@0\x9a?\x98H+\xb4\x03\xfa\xc6?\xb9P\x1e1'
+    )
+    data3 = Fletcher32().decode(data)
+    outarr = np.frombuffer(data3, dtype="<f8")
+    expected = [
+        0.0691225, 0.98130367, 0.87104532, 0.19018153, 0.9898866,
+        0.77250719, 0.74833377, 0.8271259, 0.02557469, 0.17950484
+    ]
+    assert np.allclose(outarr, expected)