check for hash collisions

Author: jreback

pandas/indexes/multi.py

@@ -2181,7 +2181,9 @@ def equals(self, other):
             return True
 
         if not isinstance(other, Index):
-            return False
+            if not isinstance(other, tuple):
+                return False
+            other = Index([other])
 
         if not isinstance(other, MultiIndex):
             return array_equivalent(self._values,

pandas/src/hashtable_class_helper.pxi.in

@@ -859,24 +859,50 @@ cdef class MultiIndexHashTable(HashTable):
                                        sizeof(size_t) + # vals
                                        sizeof(uint32_t)) # flags
 
+    def _check_for_collisions(self, int64_t[:] locs, object mi):
+        # validate that the locs map to the actual values
+        # provided in the mi
+        # we can only check if we *don't* have any missing values
+        # :<
+        cdef:
+            ndarray[int64_t] alocs
+
+        alocs = np.asarray(locs)
+        if (alocs!=-1).all():
+
+            result = self.mi.take(locs)
+            if not result.equals(mi):
+                raise ValueError("hash collision alert")
+
     def __contains__(self, object key):
         cdef:
             khiter_t k
             uint64_t value
 
         value = self.mi._hashed_indexing_key(key)
         k = kh_get_uint64(self.table, value)
-        return k != self.table.n_buckets
+        if k != self.table.n_buckets:
+            loc = self.table.vals[k]
+            locs = np.array([loc], dtype=np.int64)
+            self._check_for_collisions(locs, key)
+            return True
+
+        return False
 
     cpdef get_item(self, object key):
         cdef:
             khiter_t k
             uint64_t value
+            int64_t[:] locs
+            Py_ssize_t loc
 
         value = self.mi._hashed_indexing_key(key)
         k = kh_get_uint64(self.table, value)
         if k != self.table.n_buckets:
-            return self.table.vals[k]
+            loc = self.table.vals[k]
+            locs = np.array([loc], dtype=np.int64)
+            self._check_for_collisions(locs, key)
+            return loc
         else:
             raise KeyError(key)
 
@@ -927,6 +953,7 @@ cdef class MultiIndexHashTable(HashTable):
                 else:
                     locs[i] = -1
 
+        self._check_for_collisions(locs, mi)
         return np.asarray(locs)
 
     def unique(self, object mi):