implement libmissing; untangles _libs dependencies (#18357)

Author: jbrockmendel

pandas/_libs/algos.pyx

@@ -32,8 +32,7 @@ from libc.math cimport sqrt, fabs
 # this is our util.pxd
 from util cimport numeric, get_nat
 
-cimport lib
-from pandas._libs import lib
+import missing
 
 cdef int64_t iNaT = get_nat()
 

pandas/_libs/algos_rank_helper.pxi.in

@@ -83,7 +83,7 @@ def rank_1d_{{dtype}}(object in_arr, ties_method='average', ascending=True,
         nan_value = {{neg_nan_value}}
 
     {{if dtype == 'object'}}
-    mask = lib.isnaobj(values)
+    mask = missing.isnaobj(values)
     {{elif dtype == 'float64'}}
     mask = np.isnan(values)
     {{elif dtype == 'int64'}}
@@ -259,7 +259,7 @@ def rank_2d_{{dtype}}(object in_arr, axis=0, ties_method='average',
         nan_value = {{neg_nan_value}}
 
     {{if dtype == 'object'}}
-    mask = lib.isnaobj2d(values)
+    mask = missing.isnaobj2d(values)
     {{elif dtype == 'float64'}}
     mask = np.isnan(values)
     {{elif dtype == 'int64'}}

pandas/_libs/hashtable.pyx

@@ -42,7 +42,7 @@ cdef extern from "numpy/npy_math.h":
 cimport cython
 cimport numpy as cnp
 
-from pandas._libs.lib import checknull
+from missing cimport checknull
 
 cnp.import_array()
 cnp.import_ufunc()

pandas/_libs/hashtable_class_helper.pxi.in

@@ -4,7 +4,7 @@ Template for each `dtype` helper function for hashtable
 WARNING: DO NOT edit .pxi FILE directly, .pxi is generated from .pxi.in
 """
 
-from lib cimport is_null_datetimelike
+from missing cimport is_null_datetimelike
 
 
 #----------------------------------------------------------------------

pandas/_libs/lib.pxd

@@ -1,4 +1,3 @@
 # prototypes for sharing
 
-cdef bint is_null_datetimelike(v)
 cpdef bint is_period(val)

pandas/_libs/lib.pyx

@@ -53,14 +53,14 @@ PyDateTime_IMPORT
 
 from tslibs.np_datetime cimport get_timedelta64_value, get_datetime64_value
 
-from tslib cimport _check_all_nulls
 from tslib import NaT, Timestamp, Timedelta, array_to_datetime
 from interval import Interval
+from missing cimport checknull
 
 cdef int64_t NPY_NAT = util.get_nat()
 
 cimport util
-from util cimport is_array, _checknull, _checknan
+from util cimport is_array, _checknull
 
 from libc.math cimport sqrt, fabs
 
@@ -112,54 +112,6 @@ def memory_usage_of_objects(ndarray[object, ndim=1] arr):
 
 
 # ----------------------------------------------------------------------
-# isnull / notnull related
-
-cdef double INF = <double> np.inf
-cdef double NEGINF = -INF
-
-
-cpdef bint checknull(object val):
-    if util.is_float_object(val) or util.is_complex_object(val):
-        return val != val  # and val != INF and val != NEGINF
-    elif util.is_datetime64_object(val):
-        return get_datetime64_value(val) == NPY_NAT
-    elif val is NaT:
-        return True
-    elif util.is_timedelta64_object(val):
-        return get_timedelta64_value(val) == NPY_NAT
-    elif is_array(val):
-        return False
-    else:
-        return _checknull(val)
-
-
-cpdef bint checknull_old(object val):
-    if util.is_float_object(val) or util.is_complex_object(val):
-        return val != val or val == INF or val == NEGINF
-    elif util.is_datetime64_object(val):
-        return get_datetime64_value(val) == NPY_NAT
-    elif val is NaT:
-        return True
-    elif util.is_timedelta64_object(val):
-        return get_timedelta64_value(val) == NPY_NAT
-    elif is_array(val):
-        return False
-    else:
-        return _checknull(val)
-
-
-cpdef bint isposinf_scalar(object val):
-    if util.is_float_object(val) and val == INF:
-        return True
-    else:
-        return False
-
-
-cpdef bint isneginf_scalar(object val):
-    if util.is_float_object(val) and val == NEGINF:
-        return True
-    else:
-        return False
 
 
 cpdef bint isscalar(object val):
@@ -212,78 +164,6 @@ def item_from_zerodim(object val):
     return util.unbox_if_zerodim(val)
 
 
-@cython.wraparound(False)
-@cython.boundscheck(False)
-def isnaobj(ndarray arr):
-    cdef Py_ssize_t i, n
-    cdef object val
-    cdef ndarray[uint8_t] result
-
-    assert arr.ndim == 1, "'arr' must be 1-D."
-
-    n = len(arr)
-    result = np.empty(n, dtype=np.uint8)
-    for i from 0 <= i < n:
-        val = arr[i]
-        result[i] = _check_all_nulls(val)
-    return result.view(np.bool_)
-
-
-@cython.wraparound(False)
-@cython.boundscheck(False)
-def isnaobj_old(ndarray arr):
-    cdef Py_ssize_t i, n
-    cdef object val
-    cdef ndarray[uint8_t] result
-
-    assert arr.ndim == 1, "'arr' must be 1-D."
-
-    n = len(arr)
-    result = np.zeros(n, dtype=np.uint8)
-    for i from 0 <= i < n:
-        val = arr[i]
-        result[i] = val is NaT or util._checknull_old(val)
-    return result.view(np.bool_)
-
-
-@cython.wraparound(False)
-@cython.boundscheck(False)
-def isnaobj2d(ndarray arr):
-    cdef Py_ssize_t i, j, n, m
-    cdef object val
-    cdef ndarray[uint8_t, ndim=2] result
-
-    assert arr.ndim == 2, "'arr' must be 2-D."
-
-    n, m = (<object> arr).shape
-    result = np.zeros((n, m), dtype=np.uint8)
-    for i from 0 <= i < n:
-        for j from 0 <= j < m:
-            val = arr[i, j]
-            if checknull(val):
-                result[i, j] = 1
-    return result.view(np.bool_)
-
-
-@cython.wraparound(False)
-@cython.boundscheck(False)
-def isnaobj2d_old(ndarray arr):
-    cdef Py_ssize_t i, j, n, m
-    cdef object val
-    cdef ndarray[uint8_t, ndim=2] result
-
-    assert arr.ndim == 2, "'arr' must be 2-D."
-
-    n, m = (<object> arr).shape
-    result = np.zeros((n, m), dtype=np.uint8)
-    for i from 0 <= i < n:
-        for j from 0 <= j < m:
-            val = arr[i, j]
-            if checknull_old(val):
-                result[i, j] = 1
-    return result.view(np.bool_)
-
-
 @cython.wraparound(False)
 @cython.boundscheck(False)
 cpdef ndarray[object] list_to_object_array(list obj):

pandas/_libs/missing.pxd

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+# cython: profile=False
+
+cdef bint is_null_datetimelike(object val)
+cpdef bint checknull(object val)
+cpdef bint checknull_old(object val)