release GIL on median

release GIL on is_lexsorted / fix memory leak
release GIL on nancorr

Author: jreback

pandas/_libs/algos.pxd

@@ -1,39 +1,13 @@
 from util cimport numeric
 from numpy cimport float64_t, double_t
 
-cpdef numeric kth_smallest(numeric[:] a, Py_ssize_t k)
+cpdef numeric kth_smallest(numeric[:] a, Py_ssize_t k) nogil
 
-cdef inline Py_ssize_t swap(numeric *a, numeric *b) nogil except -1:
+cdef inline Py_ssize_t swap(numeric *a, numeric *b) nogil:
     cdef numeric t
 
     # cython doesn't allow pointer dereference so use array syntax
     t = a[0]
     a[0] = b[0]
     b[0] = t
     return 0
-
-
-cdef inline kth_smallest_c(float64_t* a, Py_ssize_t k, Py_ssize_t n):
-    cdef:
-        Py_ssize_t i, j, l, m
-        double_t x, t
-
-    l = 0
-    m = n -1
-    while (l<m):
-        x = a[k]
-        i = l
-        j = m
-
-        while 1:
-            while a[i] < x: i += 1
-            while x < a[j]: j -= 1
-            if i <= j:
-                swap(&a[i], &a[j])
-                i += 1; j -= 1
-
-            if i > j: break
-
-        if j < k: l = i
-        if k < i: m = j
-    return a[k]

pandas/_libs/algos.pyx

@@ -105,6 +105,7 @@ def is_lexsorted(list list_of_arrays):
         Py_ssize_t n, nlevels
         int64_t k, cur, pre
         ndarray arr
+        bint result = 1
 
     nlevels = len(list_of_arrays)
     n = len(list_of_arrays[0])
@@ -115,18 +116,20 @@ def is_lexsorted(list list_of_arrays):
         vecs[i] = <int64_t*> arr.data
 
     # Assume uniqueness??
-    for i from 1 <= i < n:
-        for k from 0 <= k < nlevels:
-            cur = vecs[k][i]
-            pre = vecs[k][i -1]
-            if cur == pre:
-                continue
-            elif cur > pre:
-                break
-            else:
-                return False
+    with nogil:
+        for i from 1 <= i < n:
+            for k from 0 <= k < nlevels:
+                cur = vecs[k][i]
+                pre = vecs[k][i -1]
+                if cur == pre:
+                    continue
+                elif cur > pre:
+                    break
+                else:
+                    result = 0
+                    break
     free(vecs)
-    return True
+    return result
 
 
 @cython.boundscheck(False)
@@ -177,10 +180,11 @@ def groupsort_indexer(ndarray[int64_t] index, Py_ssize_t ngroups):
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-cpdef numeric kth_smallest(numeric[:] a, Py_ssize_t k):
+cpdef numeric kth_smallest(numeric[:] a, Py_ssize_t k) nogil:
     cdef:
-        Py_ssize_t i, j, l, m, n = a.size
+        Py_ssize_t i, j, l, m, n = a.shape[0]
         numeric x
+
     with nogil:
         l = 0
         m = n - 1
@@ -201,7 +205,7 @@ cpdef numeric kth_smallest(numeric[:] a, Py_ssize_t k):
 
             if j < k: l = i
             if k < i: m = j
-        return a[k]
+    return a[k]
 
 
 cpdef numeric median(numeric[:] arr):
@@ -238,17 +242,18 @@ def max_subseq(ndarray[double_t] arr):
     S = m
     T = 0
 
-    for i in range(1, n):
-        # S = max { S + A[i], A[i] )
-        if (S > 0):
-            S = S + arr[i]
-        else:
-            S = arr[i]
-            T = i
-        if S > m:
-            s = T
-            e = i
-            m = S
+    with nogil:
+        for i in range(1, n):
+            # S = max { S + A[i], A[i] )
+            if (S > 0):
+                S = S + arr[i]
+            else:
+                S = arr[i]
+                T = i
+            if S > m:
+                s = T
+                e = i
+                m = S
 
     return (s, e, m)
 
@@ -268,9 +273,10 @@ def min_subseq(ndarray[double_t] arr):
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def nancorr(ndarray[float64_t, ndim=2] mat, cov=False, minp=None):
+def nancorr(ndarray[float64_t, ndim=2] mat, bint cov=0, minp=None):
     cdef:
         Py_ssize_t i, j, xi, yi, N, K
+        bint minpv
         ndarray[float64_t, ndim=2] result
         ndarray[uint8_t, ndim=2] mask
         int64_t nobs = 0
@@ -279,46 +285,49 @@ def nancorr(ndarray[float64_t, ndim=2] mat, cov=False, minp=None):
     N, K = (<object> mat).shape
 
     if minp is None:
-        minp = 1
+        minpv = 1
+    else:
+        minpv = <int>minp
 
     result = np.empty((K, K), dtype=np.float64)
     mask = np.isfinite(mat).view(np.uint8)
 
-    for xi in range(K):
-        for yi in range(xi + 1):
-            nobs = sumxx = sumyy = sumx = sumy = 0
-            for i in range(N):
-                if mask[i, xi] and mask[i, yi]:
-                    vx = mat[i, xi]
-                    vy = mat[i, yi]
-                    nobs += 1
-                    sumx += vx
-                    sumy += vy
-
-            if nobs < minp:
-                result[xi, yi] = result[yi, xi] = np.NaN
-            else:
-                meanx = sumx / nobs
-                meany = sumy / nobs
-
-                # now the cov numerator
-                sumx = 0
-
+    with nogil:
+        for xi in range(K):
+            for yi in range(xi + 1):
+                nobs = sumxx = sumyy = sumx = sumy = 0
                 for i in range(N):
                     if mask[i, xi] and mask[i, yi]:
-                        vx = mat[i, xi] - meanx
-                        vy = mat[i, yi] - meany
+                        vx = mat[i, xi]
+                        vy = mat[i, yi]
+                        nobs += 1
+                        sumx += vx
+                        sumy += vy
+
+                if nobs < minpv:
+                    result[xi, yi] = result[yi, xi] = NaN
+                else:
+                    meanx = sumx / nobs
+                    meany = sumy / nobs
 
-                        sumx += vx * vy
-                        sumxx += vx * vx
-                        sumyy += vy * vy
+                    # now the cov numerator
+                    sumx = 0
 
-                divisor = (nobs - 1.0) if cov else sqrt(sumxx * sumyy)
+                    for i in range(N):
+                        if mask[i, xi] and mask[i, yi]:
+                            vx = mat[i, xi] - meanx
+                            vy = mat[i, yi] - meany
 
-                if divisor != 0:
-                    result[xi, yi] = result[yi, xi] = sumx / divisor
-                else:
-                    result[xi, yi] = result[yi, xi] = np.NaN
+                            sumx += vx * vy
+                            sumxx += vx * vx
+                            sumyy += vy * vy
+
+                    divisor = (nobs - 1.0) if cov else sqrt(sumxx * sumyy)
+
+                    if divisor != 0:
+                        result[xi, yi] = result[yi, xi] = sumx / divisor
+                    else:
+                        result[xi, yi] = result[yi, xi] = NaN
 
     return result
 
@@ -351,7 +360,7 @@ def nancorr_spearman(ndarray[float64_t, ndim=2] mat, Py_ssize_t minp=1):
                     nobs += 1
 
             if nobs < minp:
-                result[xi, yi] = result[yi, xi] = np.NaN
+                result[xi, yi] = result[yi, xi] = NaN
             else:
                 maskedx = np.empty(nobs, dtype=np.float64)
                 maskedy = np.empty(nobs, dtype=np.float64)
@@ -382,7 +391,7 @@ def nancorr_spearman(ndarray[float64_t, ndim=2] mat, Py_ssize_t minp=1):
                 if divisor != 0:
                     result[xi, yi] = result[yi, xi] = sumx / divisor
                 else:
-                    result[xi, yi] = result[yi, xi] = np.NaN
+                    result[xi, yi] = result[yi, xi] = NaN
 
     return result
 

pandas/_libs/groupby.pyx

@@ -16,7 +16,7 @@ from numpy cimport (int8_t, int16_t, int32_t, int64_t, uint8_t, uint16_t,
 from libc.stdlib cimport malloc, free
 
 from util cimport numeric, get_nat
-from algos cimport kth_smallest, kth_smallest_c
+from algos cimport swap
 from algos import take_2d_axis1_float64_float64, groupsort_indexer
 
 cdef int64_t iNaT = get_nat()
@@ -219,7 +219,7 @@ def group_last_bin_object(ndarray[object, ndim=2] out,
                 out[i, j] = resx[i, j]
 
 
-cdef inline float64_t _median_linear(float64_t* a, int n):
+cdef inline float64_t _median_linear(float64_t* a, int n) nogil:
     cdef int i, j, na_count = 0
     cdef float64_t result
     cdef float64_t* tmp
@@ -259,5 +259,31 @@ cdef inline float64_t _median_linear(float64_t* a, int n):
     return result
 
 
+cdef inline float64_t kth_smallest_c(float64_t* a, Py_ssize_t k, Py_ssize_t n) nogil:
+    cdef:
+        Py_ssize_t i, j, l, m
+        double_t x, t
+
+    l = 0
+    m = n -1
+    while (l<m):
+        x = a[k]
+        i = l
+        j = m
+
+        while 1:
+            while a[i] < x: i += 1
+            while x < a[j]: j -= 1
+            if i <= j:
+                swap(&a[i], &a[j])
+                i += 1; j -= 1
+
+            if i > j: break
+
+        if j < k: l = i
+        if k < i: m = j
+    return a[k]
+
+
 # generated from template
 include "groupby_helper.pxi"

pandas/_libs/groupby_helper.pxi.in

@@ -681,6 +681,8 @@ def group_cummax_{{name}}(ndarray[{{dest_type2}}, ndim=2] out,
 #----------------------------------------------------------------------
 
 
+@cython.boundscheck(False)
+@cython.wraparound(False)
 def group_median_float64(ndarray[float64_t, ndim=2] out,
                          ndarray[int64_t] counts,
                          ndarray[float64_t, ndim=2] values,
@@ -704,13 +706,15 @@ def group_median_float64(ndarray[float64_t, ndim=2] out,
 
     take_2d_axis1_float64_float64(values.T, indexer, out=data)
 
-    for i in range(K):
-        # exclude NA group
-        ptr += _counts[0]
-        for j in range(ngroups):
-            size = _counts[j + 1]
-            out[j, i] = _median_linear(ptr, size)
-            ptr += size
+    with nogil:
+
+        for i in range(K):
+            # exclude NA group
+            ptr += _counts[0]
+            for j in range(ngroups):
+                size = _counts[j + 1]
+                out[j, i] = _median_linear(ptr, size)
+                ptr += size
 
 
 @cython.boundscheck(False)

pandas/tests/test_algos.py

@@ -10,7 +10,8 @@
 import pandas as pd
 
 from pandas import compat
-from pandas._libs import algos as libalgos, hashtable
+from pandas._libs import (groupby as libgroupby, algos as libalgos,
+                          hashtable)
 from pandas._libs.hashtable import unique_label_indices
 from pandas.compat import lrange
 import pandas.core.algorithms as algos
@@ -889,7 +890,7 @@ def test_group_var_constant(self):
 class TestGroupVarFloat64(tm.TestCase, GroupVarTestMixin):
     __test__ = True
 
-    algo = algos.algos.group_var_float64
+    algo = libgroupby.group_var_float64
     dtype = np.float64
     rtol = 1e-5
 
@@ -912,7 +913,7 @@ def test_group_var_large_inputs(self):
 class TestGroupVarFloat32(tm.TestCase, GroupVarTestMixin):
     __test__ = True
 
-    algo = algos.algos.group_var_float32
+    algo = libgroupby.group_var_float32
     dtype = np.float32
     rtol = 1e-2