diff --git a/.github/workflows/tests-nightly.yml b/.github/workflows/tests-nightly.yml index 3f2c7c0b..d4ce1914 100644 --- a/.github/workflows/tests-nightly.yml +++ b/.github/workflows/tests-nightly.yml @@ -37,7 +37,6 @@ jobs: - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 env: - JULIA_DEBUG: PythonCall JULIA_NUM_THREADS: '2' - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v5 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 10194356..aa78d53d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,7 +54,6 @@ jobs: - name: Run tests uses: julia-actions/julia-runtest@v1 env: - JULIA_DEBUG: PythonCall JULIA_NUM_THREADS: '2' PYTHON: python JULIA_PYTHONCALL_EXE: ${{ matrix.pythonexe }} diff --git a/CHANGELOG.md b/CHANGELOG.md index f615e692..bb159d80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ * The vast majority of these changes are breaking, see the [v1 Migration Guide](@ref) for how to upgrade. * Changes to core functionality: * Comparisons like `==(::Py, ::Py)`, `<(::Py, ::Number)`, `isless(::Number, ::Py)` now return `Bool` instead of `Py`. +* Changes to conversion: + * `pyconvert` rules are now scoped by target type instead of prioritized; rules are ordered by Python type specificity and creation order. + * `pyconvert_add_rule` now has the signature `pyconvert_add_rule(func::Function, t::String, ::Type{T}, ::Type{S}=T)`; the optional scope `S` controls when a rule is considered during conversion. * Changes to `PythonCall.GC` (now more like `Base.GC`): * `enable(true)` replaces `enable()`. * `enable(false)` replaces `disable()`. diff --git a/CondaPkg.toml b/CondaPkg.toml index e5416bc9..34133a30 100644 --- a/CondaPkg.toml +++ b/CondaPkg.toml @@ -15,5 +15,6 @@ version = ">=3.10,<3.14" [dev.deps] matplotlib = "" numpy = "" +pandas = "" pyside6 = "" python = "<3.14" diff --git a/docs/src/conversion-to-julia.md b/docs/src/conversion-to-julia.md index 5c65a2aa..8168ba66 100644 --- a/docs/src/conversion-to-julia.md +++ b/docs/src/conversion-to-julia.md @@ -2,62 +2,75 @@ ## [Conversion Rules](@id py2jl-conversion) -The following table specifies the conversion rules used whenever converting a Python object to a Julia object. If the initial Python type matches the "From" column and the desired type `T` intersects with the "To" column, then that conversion is attempted. Conversions are tried in priority order, then in specificity order. +The following table specifies the conversion rules used whenever converting a Python object to a Julia object. If the initial Python type matches the "From" column and the desired type `T` intersects with the "To" column, then that conversion is attempted. Rules are ordered by Python type specificity (strict subclassing only) and then by creation order. A rule only applies when the requested target type is a subtype of its scope; the "Scope" column lists the type that must match. From Julia, one can convert Python objects to a desired type using `pyconvert(T, x)` for example. From Python, the arguments to a Julia function will be converted according to these rules with `T=Any`. -| From | To | -| :----------------------------------------------------------------------------------------------------------- | :---------------------------------------------------------- | -| **Top priority (wrapped values).** | | -| `juliacall.Jl` | `Any` | -| **Very high priority (arrays).** | | -| Objects satisfying the buffer or array interface (inc. `bytes`, `bytearray`, `array.array`, `numpy.ndarray`) | `PyArray` | -| **High priority (canonical conversions).** | | -| `None` | `Nothing` | -| `bool` | `Bool` | -| `numbers.Integral` (inc. `int`) | `Integer` (prefers `Int`, or `BigInt` on overflow) | -| `float` | `Float64` | -| `complex` | `Complex{Float64}` | -| `range` | `StepRange` | -| `str` | `String` | -| `tuple` | `Tuple` | -| `collections.abc.Mapping` (inc. `dict`) | `PyDict` | -| `collections.abc.Sequence` (inc. `list`) | `PyList` | -| `collections.abc.Set` (inc. `set`, `frozenset`) | `PySet` | -| `io.IOBase` (includes open files) | `PyIO` | -| `BaseException` | `PyException` | -| `datetime.date`/`datetime.time`/`datetime.datetime` | `Date`/`Time`/`DateTime` | -| `datetime.timedelta` | `Microsecond` (or `Millisecond` or `Second` on overflow) | -| `numpy.intXX`/`numpy.uintXX`/`numpy.floatXX` | `IntXX`/`UIntXX`/`FloatXX` | -| `numpy.datetime64` | `NumpyDates.DateTime64` | -| `numpy.timedelta64` | `NumpyDates.TimeDelta64` | -| **Standard priority (other reasonable conversions).** | | -| `None` | `Missing` | -| `bytes` | `Vector{UInt8}`, `Vector{Int8}`, `String` | -| `str` | `String`, `Symbol`, `Char`, `Vector{UInt8}`, `Vector{Int8}` | -| `range` | `UnitRange` | -| `collections.abc.Mapping` | `Dict` | -| `collections.abc.Iterable` | `Vector`, `Set`, `Tuple`, `NamedTuple`, `Pair` | -| `datetime.timedelta` | `Dates.CompoundPeriod` | -| `numbers.Integral` | `Integer`, `Rational`, `Real`, `Number` | -| `numbers.Real` | `AbstractFloat`, `Number`, `Missing`/`Nothing` (if NaN) | -| `numbers.Complex` | `Complex`, `Number` | -| `ctypes.c_int` and other integers | `Integer`, `Rational`, `Real`, `Number` | -| `ctypes.c_float`/`ctypes.c_double` | `Cfloat`/`Cdouble`, `AbstractFloat`, `Real`, `Number` | -| `ctypes.c_voidp` | `Ptr{Cvoid}`, `Ptr` | -| `ctypes.c_char_p` | `Cstring`, `Ptr{Cchar}`, `Ptr` | -| `ctypes.c_wchar_p` | `Cwstring`, `Ptr{Cwchar}`, `Ptr` | -| `numpy.bool_`/`numpy.intXX`/`numpy.uintXX`/`numpy.floatXX` | `Bool`, `Integer`, `Rational`, `Real`, `Number` | -| `numpy.datetime64` | `NumpyDates.InlineDateTime64`, `Dates.DateTime` | -| `numpy.timedelta64` | `NumpyDates.InlineTimeDelta64`, `Dates.Period` | -| Objects satisfying the buffer or array interface | `Array`, `AbstractArray` | -| **Low priority (fallback to `Py`).** | | -| Anything | `Py` | -| **Bottom priority (must be explicitly specified by excluding `Py`).** | | -| Objects satisfying the buffer interface | `PyBuffer` | -| Anything | `PyRef` | +| From | To | Scope | +| :----------------------------------------------------------------------------------------------------------- | :--------------------------------------------- | :---- | +| **Default-scope rules (apply even when converting to `Any`).** | | | +| `juliacall.Jl` | `Any` | `Any` | +| Objects satisfying the buffer or array interface (inc. `bytes`, `bytearray`, `array.array`, `numpy.ndarray`) | `PyArray` | `Any` | +| `None` | `Nothing` | `Any` | +| `bool` | `Bool` | `Any` | +| `numbers.Integral` (inc. `int`) | `Integer` (prefers `Int`, or `BigInt` on overflow) | `Any` | +| `numbers.Rational` | `Rational{<:Integer}` | `Any` | +| `float` | `Float64` | `Any` | +| `complex` | `Complex{Float64}` | `Any` | +| `range` | `StepRange{<:Integer,<:Integer}` | `Any` | +| `str` | `String` | `Any` | +| `bytes` | `Base.CodeUnits{UInt8,String}` | `Any` | +| `tuple` | `NamedTuple` | `Any` | +| `tuple` | `Tuple` | `Any` | +| `datetime.datetime` | `DateTime` | `Any` | +| `datetime.date` | `Date` | `Any` | +| `datetime.time` | `Time` | `Any` | +| `datetime.timedelta` | `Microsecond` | `Any` | +| `numpy.bool_`/`numpy.intXX`/`numpy.uintXX`/`numpy.floatXX`/`numpy.complexXX` | matching Julia scalar | `Any` | +| `numpy.datetime64` | `NumpyDates.DateTime64` | `Any` | +| `numpy.timedelta64` | `NumpyDates.TimeDelta64` | `Any` | +| `pandas._libs.missing.NAType` | `Missing` | `Any` | +| `juliacall.JlBase` | `Any` | `Any` | +| `builtins.object` | `Py` | `Any` | +| **Scoped conversions (only when the requested target matches the scope).** | | | +| `None` | `Missing` | `Missing` | +| `bool` | `Number` | `Number` | +| `float` | `Number` | `Number` | +| `float` (NaN) | `Nothing` / `Missing` | `Nothing` / `Missing` | +| `complex` | `Number` | `Number` | +| `numbers.Integral` | `Number` | `Number` | +| `numbers.Rational` | `Number` | `Number` | +| `str` | `Symbol` | `Symbol` | +| `str` | `Char` | `Char` | +| `bytes` | `Vector{UInt8}` | `Vector{UInt8}` | +| `range` | `UnitRange{<:Integer}` | `UnitRange{<:Integer}` | +| `collections.abc.Iterable` | `Vector` / `Tuple` / `Set` / `NamedTuple` / `Pair` | respective targets | +| `collections.abc.Sequence` | `Vector` / `Tuple` | respective targets | +| `collections.abc.Set` | `Set` | `Set` | +| `collections.abc.Mapping` | `Dict` | `Dict` | +| `datetime.timedelta` | `Millisecond` / `Second` / `Nanosecond` | same as target | +| `numpy.datetime64` | `NumpyDates.InlineDateTime64` | `NumpyDates.InlineDateTime64` | +| `numpy.datetime64` | `NumpyDates.DatesInstant` | `NumpyDates.DatesInstant` | +| `numpy.datetime64` | `Missing` / `Nothing` | same as target | +| `numpy.timedelta64` | `NumpyDates.InlineTimeDelta64` | `NumpyDates.InlineTimeDelta64` | +| `numpy.timedelta64` | `NumpyDates.DatesPeriod` | `NumpyDates.DatesPeriod` | +| `numpy.timedelta64` | `Missing` / `Nothing` | same as target | +| NumPy scalars (`numpy.bool_`, `numpy.intXX`, `numpy.uintXX`, `numpy.floatXX`, `numpy.complexXX`) | `Int` / `UInt` / `Integer` / `Real` / `Complex{Float64}` / `Complex` / `Number` | scope matches target | +| ctypes simple values (`c_int`, `c_double`, `c_void_p`, `c_char_p`, etc.) | matching C type; widening numeric targets (`Int` / `UInt` / `Integer` / `Real` / `Number`); pointers (`Ptr`, `Cstring`, `Cwstring`) | scope matches target | +| `pandas._libs.missing.NAType` | `Nothing` | `Nothing` | +| **Wrapper conversions (opt-in scopes).** | | | +| `collections.abc.Iterable` | `PyIterable` | `PyIterable` | +| `collections.abc.Sequence` | `PyList` | `PyList` | +| `collections.abc.Set` | `PySet` | `PySet` | +| `collections.abc.Mapping` | `PyDict` | `PyDict` | +| `io.IOBase` / `_io._IOBase` | `PyIO` | `PyIO` | +| `pandas.core.frame.DataFrame` | `PyPandasDataFrame` | `PyPandasDataFrame` | +| `pandas.core.arrays.base.ExtensionArray` | `PyList` | `PyList` | +| Objects satisfying the buffer or array interface | `Array` / `AbstractArray` | same as target | +| **Explicit wrapper opt-out.** | | | +| Anything | `PyRef` | `PyRef` | See [here](@ref python-wrappers) for an explanation of the `Py*` wrapper types (`PyList`, `PyIO`, etc). diff --git a/src/API/exports.jl b/src/API/exports.jl index 0454e0e1..9a529c53 100644 --- a/src/API/exports.jl +++ b/src/API/exports.jl @@ -103,11 +103,6 @@ export pyxor export @pyconvert export pyconvert export pyconvert_add_rule -export PYCONVERT_PRIORITY_ARRAY -export PYCONVERT_PRIORITY_CANONICAL -export PYCONVERT_PRIORITY_FALLBACK -export PYCONVERT_PRIORITY_NORMAL -export PYCONVERT_PRIORITY_WRAP export pyconvert_return export pyconvert_unconverted diff --git a/src/API/types.jl b/src/API/types.jl index 14acece8..79252afb 100644 --- a/src/API/types.jl +++ b/src/API/types.jl @@ -1,13 +1,3 @@ -# Convert - -@enum PyConvertPriority begin - PYCONVERT_PRIORITY_WRAP = 400 - PYCONVERT_PRIORITY_ARRAY = 300 - PYCONVERT_PRIORITY_CANONICAL = 200 - PYCONVERT_PRIORITY_NORMAL = 0 - PYCONVERT_PRIORITY_FALLBACK = -100 -end - # Core """ diff --git a/src/Convert/Convert.jl b/src/Convert/Convert.jl index fc506fc6..7987a923 100644 --- a/src/Convert/Convert.jl +++ b/src/Convert/Convert.jl @@ -18,8 +18,7 @@ import ..PythonCall: pyconvert_add_rule, pyconvert_return, pyconvert_unconverted, - pyconvert, - PyConvertPriority + pyconvert export pyconvert_isunconverted, pyconvert_result, @@ -36,10 +35,7 @@ include("numpy.jl") include("pandas.jl") function __init__() - init_pyconvert() - init_ctypes() - init_numpy() - init_pandas() + init_pyconvert_extratypes() end end diff --git a/src/Convert/ctypes.jl b/src/Convert/ctypes.jl index d18bbc74..efbce51e 100644 --- a/src/Convert/ctypes.jl +++ b/src/Convert/ctypes.jl @@ -34,31 +34,3 @@ const CTYPES_SIMPLE_TYPES = [ ("void_p", Ptr{Cvoid}), ] -function init_ctypes() - for (t, T) in CTYPES_SIMPLE_TYPES - isptr = endswith(t, "_p") - isreal = !isptr - isnumber = isreal - isfloat = t in ("float", "double") - isint = isreal && !isfloat - isuint = isint && (startswith(t, "u") || t == "size_t") - - name = "ctypes:c_$t" - rule = pyconvert_rule_ctypessimplevalue{T,false}() - saferule = pyconvert_rule_ctypessimplevalue{T,true}() - - t == "char_p" && pyconvert_add_rule(name, Cstring, saferule) - t == "wchar_p" && pyconvert_add_rule(name, Cwstring, saferule) - pyconvert_add_rule(name, T, saferule) - isuint && pyconvert_add_rule(name, UInt, sizeof(T) ≤ sizeof(UInt) ? saferule : rule) - isuint && pyconvert_add_rule(name, Int, sizeof(T) < sizeof(Int) ? saferule : rule) - isint && - !isuint && - pyconvert_add_rule(name, Int, sizeof(T) ≤ sizeof(Int) ? saferule : rule) - isint && pyconvert_add_rule(name, Integer, rule) - isfloat && pyconvert_add_rule(name, Float64, saferule) - isreal && pyconvert_add_rule(name, Real, rule) - isnumber && pyconvert_add_rule(name, Number, rule) - isptr && pyconvert_add_rule(name, Ptr, saferule) - end -end diff --git a/src/Convert/numpy.jl b/src/Convert/numpy.jl index 73383d54..276d0849 100644 --- a/src/Convert/numpy.jl +++ b/src/Convert/numpy.jl @@ -97,64 +97,3 @@ const NUMPY_SIMPLE_TYPES = [ ("complex128", ComplexF64), ] -function init_numpy() - # simple numeric scalar types - for (t, T) in NUMPY_SIMPLE_TYPES - isbool = occursin("bool", t) - isint = occursin("int", t) || isbool - isuint = occursin("uint", t) || isbool - isfloat = occursin("float", t) - iscomplex = occursin("complex", t) - isreal = isint || isfloat - isnumber = isreal || iscomplex - - name = "numpy:$t" - rule = pyconvert_rule_numpysimplevalue{T,false}() - saferule = pyconvert_rule_numpysimplevalue{T,true}() - - pyconvert_add_rule(name, T, saferule, PYCONVERT_PRIORITY_ARRAY) - isuint && pyconvert_add_rule(name, UInt, sizeof(T) ≤ sizeof(UInt) ? saferule : rule) - isuint && pyconvert_add_rule(name, Int, sizeof(T) < sizeof(Int) ? saferule : rule) - isint && - !isuint && - pyconvert_add_rule(name, Int, sizeof(T) ≤ sizeof(Int) ? saferule : rule) - isint && pyconvert_add_rule(name, Integer, rule) - isfloat && pyconvert_add_rule(name, Float64, saferule) - isreal && pyconvert_add_rule(name, Real, rule) - iscomplex && pyconvert_add_rule(name, ComplexF64, saferule) - iscomplex && pyconvert_add_rule(name, Complex, rule) - isnumber && pyconvert_add_rule(name, Number, rule) - end - - # datetime64 - pyconvert_add_rule( - "numpy:datetime64", - DateTime64, - pyconvert_rule_datetime64, - PYCONVERT_PRIORITY_ARRAY, - ) - pyconvert_add_rule("numpy:datetime64", InlineDateTime64, pyconvert_rule_datetime64) - pyconvert_add_rule( - "numpy:datetime64", - NumpyDates.DatesInstant, - pyconvert_rule_datetime64, - ) - pyconvert_add_rule("numpy:datetime64", Missing, pyconvert_rule_datetime64) - pyconvert_add_rule("numpy:datetime64", Nothing, pyconvert_rule_datetime64) - - # timedelta64 - pyconvert_add_rule( - "numpy:timedelta64", - TimeDelta64, - pyconvert_rule_timedelta64, - PYCONVERT_PRIORITY_ARRAY, - ) - pyconvert_add_rule("numpy:timedelta64", InlineTimeDelta64, pyconvert_rule_timedelta64) - pyconvert_add_rule( - "numpy:timedelta64", - NumpyDates.DatesPeriod, - pyconvert_rule_timedelta64, - ) - pyconvert_add_rule("numpy:timedelta64", Missing, pyconvert_rule_timedelta64) - pyconvert_add_rule("numpy:timedelta64", Nothing, pyconvert_rule_timedelta64) -end diff --git a/src/Convert/pandas.jl b/src/Convert/pandas.jl index 65786f1c..bf670cc4 100644 --- a/src/Convert/pandas.jl +++ b/src/Convert/pandas.jl @@ -1,12 +1,3 @@ pyconvert_rule_pandas_na(::Type{Nothing}, x::Py) = pyconvert_return(nothing) pyconvert_rule_pandas_na(::Type{Missing}, x::Py) = pyconvert_return(missing) -function init_pandas() - pyconvert_add_rule( - "pandas._libs.missing:NAType", - Missing, - pyconvert_rule_pandas_na, - PYCONVERT_PRIORITY_CANONICAL, - ) - pyconvert_add_rule("pandas._libs.missing:NAType", Nothing, pyconvert_rule_pandas_na) -end diff --git a/src/Convert/pyconvert.jl b/src/Convert/pyconvert.jl index 2b03eb72..4853e61c 100644 --- a/src/Convert/pyconvert.jl +++ b/src/Convert/pyconvert.jl @@ -1,14 +1,17 @@ struct PyConvertRule + tname::String type::Type + scope::Type func::Function - priority::PyConvertPriority + order::Int end const PYCONVERT_RULES = Dict{String,Vector{PyConvertRule}}() +const PYCONVERT_RULE_ORDER = Ref{Int}(0) const PYCONVERT_EXTRATYPES = Py[] """ - pyconvert_add_rule(tname::String, T::Type, func::Function, priority::PyConvertPriority=PYCONVERT_PRIORITY_NORMAL) + pyconvert_add_rule(func::Function, tname::String, ::Type{T}, ::Type{S}=T) where {T,S} Add a new conversion rule for `pyconvert`. @@ -19,14 +22,15 @@ Add a new conversion rule for `pyconvert`. Python objects of this type. - `T` is a Julia type, such that this rule only applies when the target type intersects with `T`. +- `S` is a Julia type, such that this rule only applies when the target type is a subtype + of `S` (or a union whose components include a subtype of `S`). - `func` is the function implementing the rule. -- `priority` determines whether to prioritise this rule above others. When `pyconvert(R, x)` is called, all rules such that `typeintersect(T, R) != Union{}` -and `pyisinstance(x, t)` are considered. These rules are sorted first by priority, -then by the specificity of `t` (e.g. `bool` is more specific than `int` is more specific -than `object`) then by the order they were added. The rules are tried in turn until one -succeeds. +and `pyisinstance(x, t)` are considered. It also requires that `R <: S`, or if `R` is a +union then at least one component satisfies this property. These rules are sorted first +by the specificity of `t` (strict subclassing only) then by the order they were added. +The rules are tried in turn until one succeeds. ### Implementing `func` @@ -40,32 +44,17 @@ It must return one of: The target type `S` is never a union or the empty type, i.e. it is always a data type or union-all. -### Priority - -Most rules should have priority `PYCONVERT_PRIORITY_NORMAL` (the default) which is for any -reasonable conversion rule. - -Use priority `PYCONVERT_PRIORITY_CANONICAL` for **canonical** conversion rules. Immutable -objects may be canonically converted to their corresponding Julia type, such as `int` to -`Integer`. Mutable objects **must** be converted to a wrapper type, such that the original -Python object can be retrieved. For example a `list` is canonically converted to `PyList` -and not to a `Vector`. There should not be more than one canonical conversion rule for a -given Python type. - -Other priorities are reserved for internal use. """ -function pyconvert_add_rule( - pytypename::String, - type::Type, - func::Function, - priority::PyConvertPriority = PYCONVERT_PRIORITY_NORMAL, -) - @nospecialize type func +function pyconvert_add_rule(func::Function, pytypename::String, type::Type, scope::Type = type) + @nospecialize type func scope + type <: scope || error("pyconvert rule must satisfy T <: S") + order = (PYCONVERT_RULE_ORDER[] += 1) push!( get!(Vector{PyConvertRule}, PYCONVERT_RULES, pytypename), - PyConvertRule(type, func, priority), + PyConvertRule(pytypename, type, scope, func, order), ) empty!.(values(PYCONVERT_RULES_CACHE)) + empty!(PYCONVERT_PREFERRED_TYPE) return end @@ -148,109 +137,73 @@ function pyconvert_typename(t::Py) return "$m:$n" end -function _pyconvert_get_rules(pytype::Py) - pyisin(x, ys) = any(pyis(x, y) for y in ys) - - # get the MROs of all base types we are considering - omro = collect(pytype.__mro__) - basetypes = Py[pytype] - basemros = Vector{Py}[omro] - for xtype in PYCONVERT_EXTRATYPES - # find the topmost supertype of - xbase = PyNULL - for base in omro - if pyissubclass(base, xtype) - xbase = base - end +pyconvert_is_special_tname(tname::String) = + tname in ("", "", "", "") + +function _pyconvert_collect_supertypes(pytype::Py) + seen = Set{C.PyPtr}() + queue = Py[pytype] + types = Py[] + while !isempty(queue) + t = pop!(queue) + ptr = getptr(t) + ptr ∈ seen && continue + push!(seen, ptr) + push!(types, t) + if pyhasattr(t, "__bases__") + append!(queue, (Py(b) for b in t.__bases__)) end - if !pyisnull(xbase) - push!(basetypes, xtype) - xmro = collect(xtype.__mro__) - pyisin(xbase, xmro) || pushfirst!(xmro, xbase) - push!(basemros, xmro) - end - end - for xbase in basetypes[2:end] - push!(basemros, [xbase]) end + return types +end - # merge the MROs - # this is a port of the merge() function at the bottom of: - # https://www.python.org/download/releases/2.3/mro/ - mro = Py[] - while !isempty(basemros) - # find the first head not contained in any tail - ok = false - b = PyNULL - for bmro in basemros - b = bmro[1] - if all(bmro -> !pyisin(b, bmro[2:end]), basemros) - ok = true - break - end - end - ok || error( - "Fatal inheritance error: could not merge MROs (mro=$mro, basemros=$basemros)", - ) - # add it to the list - push!(mro, b) - # remove it from consideration - for bmro in basemros - filter!(t -> !pyis(t, b), bmro) - end - # remove empty lists - filter!(x -> !isempty(x), basemros) - end - # check the original MRO is preserved - omro_ = filter(t -> pyisin(t, omro), mro) - @assert length(omro) == length(omro_) - @assert all(pyis(x, y) for (x, y) in zip(omro, omro_)) - - # get the names of the types in the MRO of pytype - xmro = [String[pyconvert_typename(t)] for t in mro] - - # add special names corresponding to certain interfaces - # these get inserted just above the topmost type satisfying the interface - for (t, x) in reverse(collect(zip(mro, xmro))) - if pyhasattr(t, "__array_struct__") - push!(x, "") - break - end - end - for (t, x) in reverse(collect(zip(mro, xmro))) - if pyhasattr(t, "__array_interface__") - push!(x, "") - break - end +function _pyconvert_get_rules(pytype::Py) + typemap = Dict{String,Py}() + tnames = Set{String}() + + function add_type!(t::Py) + tname = pyconvert_typename(t) + haskey(typemap, tname) || (typemap[tname] = t) + push!(tnames, tname) + return nothing end - for (t, x) in reverse(collect(zip(mro, xmro))) - if pyhasattr(t, "__array__") - push!(x, "") - break - end + + for t in _pyconvert_collect_supertypes(pytype) + add_type!(t) + pyhasattr(t, "__array_struct__") && push!(tnames, "") + pyhasattr(t, "__array_interface__") && push!(tnames, "") + pyhasattr(t, "__array__") && push!(tnames, "") + (C.PyType_CheckBuffer(t) != 0) && push!(tnames, "") end - for (t, x) in reverse(collect(zip(mro, xmro))) - if C.PyType_CheckBuffer(t) - push!(x, "") - break + + for xtype in PYCONVERT_EXTRATYPES + pyissubclass(pytype, xtype) || continue + for t in _pyconvert_collect_supertypes(xtype) + add_type!(t) end end - # flatten to get the MRO as a list of strings - mro = String[x for xs in xmro for x in xs] - - # get corresponding rules - rules = PyConvertRule[ - rule for tname in mro for - rule in get!(Vector{PyConvertRule}, PYCONVERT_RULES, tname) + rules = PyConvertRuleInfo[ + PyConvertRuleInfo( + tname, + get(typemap, tname, PyNULL), + rule.type, + rule.scope, + rule.func, + rule.order, + ) for tname in tnames for rule in get!(Vector{PyConvertRule}, PYCONVERT_RULES, tname) ] - # order the rules by priority, then by original order - order = sort(axes(rules, 1), by = i -> (rules[i].priority, -i), rev = true) - rules = rules[order] + return rules, typemap +end - @debug "pyconvert" pytype mro = join(mro, " ") - return rules +struct PyConvertRuleInfo + tname::String + pytype::Py + type::Type + scope::Type + func::Function + order::Int end const PYCONVERT_PREFERRED_TYPE = Dict{Py,Type}() @@ -260,43 +213,26 @@ pyconvert_preferred_type(pytype::Py) = if pyissubclass(pytype, pybuiltins.int) Union{Int,BigInt} else - _pyconvert_get_rules(pytype)[1].type + pyconvert_get_rules_info(Any, pytype)[1].type end end function pyconvert_get_rules(type::Type, pytype::Py) @nospecialize type - # this could be cached - rules = _pyconvert_get_rules(pytype) - - # intersect rules with type - rules = PyConvertRule[ - PyConvertRule(typeintersect(rule.type, type), rule.func, rule.priority) for - rule in rules - ] - - # explode out unions - rules = [ - PyConvertRule(type, rule.func, rule.priority) for rule in rules for - type in Utils.explode_union(rule.type) - ] - - # filter out empty rules - rules = [rule for rule in rules if rule.type != Union{}] - - # filter out repeated rules - rules = [ - rule for (i, rule) in enumerate(rules) if !any( - (rule.func === rules[j].func) && ((rule.type) <: (rules[j].type)) for - j = 1:(i-1) - ) - ] + rules = pyconvert_get_rules_info(type, pytype) @debug "pyconvert" type rules return Function[pyconvert_fix(rule.type, rule.func) for rule in rules] end +function pyconvert_get_rules_info(type::Type, pytype::Py) + @nospecialize type + rules, typemap = _pyconvert_get_rules(pytype) + rules = _pyconvert_filter_rules(type, rules) + return _pyconvert_order_rules(rules, typemap) +end + pyconvert_fix(::Type{T}, func) where {T} = x -> func(T, x) const PYCONVERT_RULES_CACHE = Dict{Type,Dict{C.PyPtr,Vector{Function}}}() @@ -304,6 +240,85 @@ const PYCONVERT_RULES_CACHE = Dict{Type,Dict{C.PyPtr,Vector{Function}}}() @generated pyconvert_rules_cache(::Type{T}) where {T} = get!(Dict{C.PyPtr,Vector{Function}}, PYCONVERT_RULES_CACHE, T) +function _pyconvert_type_in_scope(::Type{R}, ::Type{S}) where {R,S} + if R isa Union + any(_pyconvert_type_in_scope(T, S) for T in Utils.explode_union(R)) + else + R <: S + end +end + +function _pyconvert_filter_rules(type::Type, rules::Vector{PyConvertRuleInfo}) + @nospecialize type + filtered = PyConvertRuleInfo[] + for rule in rules + T = typeintersect(rule.type, type) + T == Union{} && continue + _pyconvert_type_in_scope(type, rule.scope) || continue + for U in Utils.explode_union(T) + U == Union{} && continue + push!(filtered, PyConvertRuleInfo(rule.tname, rule.pytype, U, rule.scope, rule.func, rule.order)) + end + end + + filtered = [ + rule for (i, rule) in enumerate(filtered) if !any( + (rule.func === filtered[j].func) && (rule.type <: filtered[j].type) for j = 1:(i - 1) + ) + ] + return filtered +end + +function _pyconvert_tname_subclass(t1::String, t2::String, typemap::Dict{String,Py}) + t1 == t2 && return false + py1 = get(typemap, t1, PyNULL) + py2 = get(typemap, t2, PyNULL) + + if t2 == "" + return (t1 != "") && !pyisnull(py1) && (C.PyType_CheckBuffer(py1) != 0) + elseif t2 == "" + return (t1 != "") && !pyisnull(py1) && pyhasattr(py1, "__array__") + elseif t2 == "" + return (t1 != "") && !pyisnull(py1) && pyhasattr(py1, "__array_interface__") + elseif t2 == "" + return (t1 != "") && !pyisnull(py1) && pyhasattr(py1, "__array_struct__") + elseif pyisnull(py1) || pyisnull(py2) + return false + else + return pyissubclass(py1, py2) + end +end + +function _pyconvert_order_rules(rules::Vector{PyConvertRuleInfo}, typemap::Dict{String,Py}) + incoming = Dict{Int,Vector{Int}}() + for i in eachindex(rules) + incoming[i] = Int[] + end + for (i, ri) in pairs(rules), (j, rj) in pairs(rules) + (i == j) && continue + if _pyconvert_tname_subclass(ri.tname, rj.tname, typemap) + push!(incoming[j], i) + end + end + + ordered = PyConvertRuleInfo[] + remaining = collect(keys(incoming)) + while !isempty(remaining) + available = [i for i in remaining if isempty(incoming[i])] + isempty(available) && error("pyconvert rule ordering cycle detected") + sort!(available, by = i -> rules[i].order) + append!(ordered, rules[available]) + for i in available + delete!(incoming, i) + end + filter!(i -> haskey(incoming, i), remaining) + for deps in values(incoming) + filter!(i -> i ∉ available, deps) + end + end + return ordered +end + function pyconvert_rule_fast(::Type{T}, x::Py) where {T} if T isa Union a = pyconvert_rule_fast(T.a, x)::pyconvert_returntype(T.a) @@ -407,7 +422,8 @@ pyconvertarg(::Type{T}, x, name) where {T} = @autopy x @pyconvert T x_ begin pythrow() end -function init_pyconvert() +function init_pyconvert_extratypes() + isempty(PYCONVERT_EXTRATYPES) || return push!(PYCONVERT_EXTRATYPES, pyimport("io" => "IOBase")) push!( PYCONVERT_EXTRATYPES, @@ -417,100 +433,4 @@ function init_pyconvert() PYCONVERT_EXTRATYPES, pyimport("collections.abc" => ("Iterable", "Sequence", "Set", "Mapping"))..., ) - - priority = PYCONVERT_PRIORITY_CANONICAL - pyconvert_add_rule("builtins:NoneType", Nothing, pyconvert_rule_none, priority) - pyconvert_add_rule("builtins:bool", Bool, pyconvert_rule_bool, priority) - pyconvert_add_rule("builtins:float", Float64, pyconvert_rule_float, priority) - pyconvert_add_rule( - "builtins:complex", - Complex{Float64}, - pyconvert_rule_complex, - priority, - ) - pyconvert_add_rule("numbers:Integral", Integer, pyconvert_rule_int, priority) - pyconvert_add_rule("builtins:str", String, pyconvert_rule_str, priority) - pyconvert_add_rule( - "builtins:bytes", - Base.CodeUnits{UInt8,String}, - pyconvert_rule_bytes, - priority, - ) - pyconvert_add_rule( - "builtins:range", - StepRange{<:Integer,<:Integer}, - pyconvert_rule_range, - priority, - ) - pyconvert_add_rule( - "numbers:Rational", - Rational{<:Integer}, - pyconvert_rule_fraction, - priority, - ) - pyconvert_add_rule("builtins:tuple", NamedTuple, pyconvert_rule_iterable, priority) - pyconvert_add_rule("builtins:tuple", Tuple, pyconvert_rule_iterable, priority) - pyconvert_add_rule("datetime:datetime", DateTime, pyconvert_rule_datetime, priority) - pyconvert_add_rule("datetime:date", Date, pyconvert_rule_date, priority) - pyconvert_add_rule("datetime:time", Time, pyconvert_rule_time, priority) - pyconvert_add_rule( - "datetime:timedelta", - Microsecond, - pyconvert_rule_timedelta, - priority, - ) - pyconvert_add_rule( - "builtins:BaseException", - PyException, - pyconvert_rule_exception, - priority, - ) - - priority = PYCONVERT_PRIORITY_NORMAL - pyconvert_add_rule("builtins:NoneType", Missing, pyconvert_rule_none, priority) - pyconvert_add_rule("builtins:bool", Number, pyconvert_rule_bool, priority) - pyconvert_add_rule("numbers:Real", Number, pyconvert_rule_float, priority) - pyconvert_add_rule("builtins:float", Nothing, pyconvert_rule_float, priority) - pyconvert_add_rule("builtins:float", Missing, pyconvert_rule_float, priority) - pyconvert_add_rule("numbers:Complex", Number, pyconvert_rule_complex, priority) - pyconvert_add_rule("numbers:Integral", Number, pyconvert_rule_int, priority) - pyconvert_add_rule("builtins:str", Symbol, pyconvert_rule_str, priority) - pyconvert_add_rule("builtins:str", Char, pyconvert_rule_str, priority) - pyconvert_add_rule("builtins:bytes", Vector{UInt8}, pyconvert_rule_bytes, priority) - pyconvert_add_rule( - "builtins:range", - UnitRange{<:Integer}, - pyconvert_rule_range, - priority, - ) - pyconvert_add_rule("numbers:Rational", Number, pyconvert_rule_fraction, priority) - pyconvert_add_rule( - "collections.abc:Iterable", - Vector, - pyconvert_rule_iterable, - priority, - ) - pyconvert_add_rule("collections.abc:Iterable", Tuple, pyconvert_rule_iterable, priority) - pyconvert_add_rule("collections.abc:Iterable", Pair, pyconvert_rule_iterable, priority) - pyconvert_add_rule("collections.abc:Iterable", Set, pyconvert_rule_iterable, priority) - pyconvert_add_rule( - "collections.abc:Sequence", - Vector, - pyconvert_rule_iterable, - priority, - ) - pyconvert_add_rule("collections.abc:Sequence", Tuple, pyconvert_rule_iterable, priority) - pyconvert_add_rule("collections.abc:Set", Set, pyconvert_rule_iterable, priority) - pyconvert_add_rule("collections.abc:Mapping", Dict, pyconvert_rule_mapping, priority) - pyconvert_add_rule( - "datetime:timedelta", - Millisecond, - pyconvert_rule_timedelta, - priority, - ) - pyconvert_add_rule("datetime:timedelta", Second, pyconvert_rule_timedelta, priority) - pyconvert_add_rule("datetime:timedelta", Nanosecond, pyconvert_rule_timedelta, priority) - - priority = PYCONVERT_PRIORITY_FALLBACK - pyconvert_add_rule("builtins:object", Py, pyconvert_rule_object, priority) end diff --git a/src/JlWrap/base.jl b/src/JlWrap/base.jl index 2497f4b0..fce60a12 100644 --- a/src/JlWrap/base.jl +++ b/src/JlWrap/base.jl @@ -23,10 +23,6 @@ pyjlvalue(x) = @autopy x _pyjl_getvalue(x_) function init_base() setptr!(pyjlbasetype, incref(Cjl.PyJuliaBase_Type[])) pyjuliacallmodule.JlBase = pyjlbasetype - - # conversion rule - priority = PYCONVERT_PRIORITY_WRAP - pyconvert_add_rule("juliacall:JlBase", Any, pyconvert_rule_jlvalue, priority) end pyconvert_rule_jlvalue(::Type{T}, x::Py) where {T} = diff --git a/src/PythonCall.jl b/src/PythonCall.jl index e8c7597f..7e0e0ca1 100644 --- a/src/PythonCall.jl +++ b/src/PythonCall.jl @@ -2,6 +2,8 @@ module PythonCall const ROOT_DIR = dirname(@__DIR__) +using Dates: Date, DateTime, Microsecond, Millisecond, Nanosecond, Second, Time + include("API/API.jl") include("Utils/Utils.jl") include("NumpyDates/NumpyDates.jl") @@ -31,4 +33,182 @@ for k in [ @eval using .JlWrap: $k end +function __init__() + # Core pyconvert rules + pyconvert_add_rule(Convert.pyconvert_rule_none, "builtins:NoneType", Nothing, Any) + pyconvert_add_rule(Convert.pyconvert_rule_bool, "builtins:bool", Bool, Any) + pyconvert_add_rule(Convert.pyconvert_rule_float, "builtins:float", Float64, Any) + pyconvert_add_rule(Convert.pyconvert_rule_complex, "builtins:complex", Complex{Float64}, Any) + pyconvert_add_rule(Convert.pyconvert_rule_int, "numbers:Integral", Integer, Any) + pyconvert_add_rule(Convert.pyconvert_rule_str, "builtins:str", String, Any) + pyconvert_add_rule(Convert.pyconvert_rule_bytes, "builtins:bytes", Base.CodeUnits{UInt8,String}, Any) + pyconvert_add_rule( + Convert.pyconvert_rule_range, + "builtins:range", + StepRange{<:Integer,<:Integer}, + Any, + ) + pyconvert_add_rule(Convert.pyconvert_rule_fraction, "numbers:Rational", Rational{<:Integer}, Any) + pyconvert_add_rule(Convert.pyconvert_rule_iterable, "builtins:tuple", NamedTuple, Any) + pyconvert_add_rule(Convert.pyconvert_rule_iterable, "builtins:tuple", Tuple, Any) + pyconvert_add_rule(Convert.pyconvert_rule_datetime, "datetime:datetime", DateTime, Any) + pyconvert_add_rule(Convert.pyconvert_rule_date, "datetime:date", Date, Any) + pyconvert_add_rule(Convert.pyconvert_rule_time, "datetime:time", Time, Any) + pyconvert_add_rule(Convert.pyconvert_rule_timedelta, "datetime:timedelta", Microsecond, Any) + pyconvert_add_rule(Convert.pyconvert_rule_exception, "builtins:BaseException", PyException, Any) + pyconvert_add_rule(Convert.pyconvert_rule_none, "builtins:NoneType", Missing, Missing) + pyconvert_add_rule(Convert.pyconvert_rule_bool, "builtins:bool", Number, Number) + pyconvert_add_rule(Convert.pyconvert_rule_float, "numbers:Real", Number, Number) + pyconvert_add_rule(Convert.pyconvert_rule_float, "builtins:float", Nothing, Nothing) + pyconvert_add_rule(Convert.pyconvert_rule_float, "builtins:float", Missing, Missing) + pyconvert_add_rule(Convert.pyconvert_rule_complex, "numbers:Complex", Number, Number) + pyconvert_add_rule(Convert.pyconvert_rule_int, "numbers:Integral", Number, Number) + pyconvert_add_rule(Convert.pyconvert_rule_str, "builtins:str", Symbol, Symbol) + pyconvert_add_rule(Convert.pyconvert_rule_str, "builtins:str", Char, Char) + pyconvert_add_rule(Convert.pyconvert_rule_bytes, "builtins:bytes", Vector{UInt8}, Vector{UInt8}) + pyconvert_add_rule( + Convert.pyconvert_rule_range, + "builtins:range", + UnitRange{<:Integer}, + UnitRange{<:Integer}, + ) + pyconvert_add_rule(Convert.pyconvert_rule_fraction, "numbers:Rational", Number, Number) + pyconvert_add_rule(Convert.pyconvert_rule_iterable, "collections.abc:Iterable", Vector, Vector) + pyconvert_add_rule(Convert.pyconvert_rule_iterable, "collections.abc:Iterable", Tuple, Tuple) + pyconvert_add_rule(Convert.pyconvert_rule_iterable, "collections.abc:Iterable", Pair, Pair) + pyconvert_add_rule(Convert.pyconvert_rule_iterable, "collections.abc:Iterable", Set, Set) + pyconvert_add_rule(Convert.pyconvert_rule_iterable, "collections.abc:Sequence", Vector, Vector) + pyconvert_add_rule(Convert.pyconvert_rule_iterable, "collections.abc:Sequence", Tuple, Tuple) + pyconvert_add_rule(Convert.pyconvert_rule_iterable, "collections.abc:Set", Set, Set) + pyconvert_add_rule(Convert.pyconvert_rule_mapping, "collections.abc:Mapping", Dict, Dict) + pyconvert_add_rule(Convert.pyconvert_rule_timedelta, "datetime:timedelta", Millisecond, Millisecond) + pyconvert_add_rule(Convert.pyconvert_rule_timedelta, "datetime:timedelta", Second, Second) + pyconvert_add_rule(Convert.pyconvert_rule_timedelta, "datetime:timedelta", Nanosecond, Nanosecond) + + # ctypes rules + for (t, T) in Convert.CTYPES_SIMPLE_TYPES + name = "ctypes:c_$t" + rule = Convert.pyconvert_rule_ctypessimplevalue{T,false}() + saferule = Convert.pyconvert_rule_ctypessimplevalue{T,true}() + isptr = endswith(t, "_p") + isreal = !isptr + isfloat = t in ("float", "double") + isint = isreal && !isfloat + isuint = isint && (startswith(t, "u") || t == "size_t") + + t == "char_p" && pyconvert_add_rule(saferule, name, Cstring, Cstring) + t == "wchar_p" && pyconvert_add_rule(saferule, name, Cwstring, Cwstring) + pyconvert_add_rule(saferule, name, T, T) + isuint && pyconvert_add_rule(sizeof(T) ≤ sizeof(UInt) ? saferule : rule, name, UInt, UInt) + isuint && pyconvert_add_rule(sizeof(T) < sizeof(Int) ? saferule : rule, name, Int, Int) + isint && !isuint && pyconvert_add_rule(sizeof(T) ≤ sizeof(Int) ? saferule : rule, name, Int, Int) + isint && pyconvert_add_rule(rule, name, Integer, Integer) + isfloat && pyconvert_add_rule(saferule, name, Float64, Float64) + isreal && pyconvert_add_rule(rule, name, Real, Real) + isreal && pyconvert_add_rule(rule, name, Number, Number) + isptr && pyconvert_add_rule(saferule, name, Ptr, Ptr) + end + # numpy rules + for (t, T) in Convert.NUMPY_SIMPLE_TYPES + name = "numpy:$t" + rule = Convert.pyconvert_rule_numpysimplevalue{T,false}() + saferule = Convert.pyconvert_rule_numpysimplevalue{T,true}() + isbool = occursin("bool", t) + isint = occursin("int", t) || isbool + isuint = occursin("uint", t) || isbool + isfloat = occursin("float", t) + iscomplex = occursin("complex", t) + isreal = isint || isfloat + isnumber = isreal || iscomplex + + pyconvert_add_rule(saferule, name, T, Any) + isuint && pyconvert_add_rule(sizeof(T) ≤ sizeof(UInt) ? saferule : rule, name, UInt, UInt) + isuint && pyconvert_add_rule(sizeof(T) < sizeof(Int) ? saferule : rule, name, Int, Int) + isint && !isuint && pyconvert_add_rule(sizeof(T) ≤ sizeof(Int) ? saferule : rule, name, Int, Int) + isint && pyconvert_add_rule(rule, name, Integer, Integer) + isfloat && pyconvert_add_rule(saferule, name, Float64, Float64) + isreal && pyconvert_add_rule(rule, name, Real, Real) + iscomplex && pyconvert_add_rule(saferule, name, ComplexF64, ComplexF64) + iscomplex && pyconvert_add_rule(rule, name, Complex, Complex) + isnumber && pyconvert_add_rule(rule, name, Number, Number) + end + pyconvert_add_rule(Convert.pyconvert_rule_datetime64, "numpy:datetime64", NumpyDates.DateTime64, Any) + pyconvert_add_rule( + Convert.pyconvert_rule_datetime64, + "numpy:datetime64", + NumpyDates.InlineDateTime64, + NumpyDates.InlineDateTime64, + ) + pyconvert_add_rule( + Convert.pyconvert_rule_datetime64, + "numpy:datetime64", + NumpyDates.DatesInstant, + NumpyDates.DatesInstant, + ) + pyconvert_add_rule(Convert.pyconvert_rule_datetime64, "numpy:datetime64", Missing, Missing) + pyconvert_add_rule(Convert.pyconvert_rule_datetime64, "numpy:datetime64", Nothing, Nothing) + pyconvert_add_rule(Convert.pyconvert_rule_timedelta64, "numpy:timedelta64", NumpyDates.TimeDelta64, Any) + pyconvert_add_rule( + Convert.pyconvert_rule_timedelta64, + "numpy:timedelta64", + NumpyDates.InlineTimeDelta64, + NumpyDates.InlineTimeDelta64, + ) + pyconvert_add_rule( + Convert.pyconvert_rule_timedelta64, + "numpy:timedelta64", + NumpyDates.DatesPeriod, + NumpyDates.DatesPeriod, + ) + pyconvert_add_rule(Convert.pyconvert_rule_timedelta64, "numpy:timedelta64", Missing, Missing) + pyconvert_add_rule(Convert.pyconvert_rule_timedelta64, "numpy:timedelta64", Nothing, Nothing) + + # pandas rules + pyconvert_add_rule( + Convert.pyconvert_rule_pandas_na, + "pandas._libs.missing:NAType", + Missing, + Any, + ) + pyconvert_add_rule(Convert.pyconvert_rule_pandas_na, "pandas._libs.missing:NAType", Nothing, Nothing) + + # wrapper rules + pyconvert_add_rule(Wrap.pyconvert_rule_array_nocopy, "", Wrap.PyArray, Any) + pyconvert_add_rule(Wrap.pyconvert_rule_array_nocopy, "", Wrap.PyArray, Any) + pyconvert_add_rule(Wrap.pyconvert_rule_array_nocopy, "", Wrap.PyArray, Any) + pyconvert_add_rule(Wrap.pyconvert_rule_array_nocopy, "", Wrap.PyArray, Any) + pyconvert_add_rule(Wrap.pyconvert_rule_iterable, "collections.abc:Iterable", Wrap.PyIterable, Wrap.PyIterable) + pyconvert_add_rule(Wrap.pyconvert_rule_sequence, "collections.abc:Sequence", Wrap.PyList, Wrap.PyList) + pyconvert_add_rule(Wrap.pyconvert_rule_set, "collections.abc:Set", Wrap.PySet, Wrap.PySet) + pyconvert_add_rule(Wrap.pyconvert_rule_mapping, "collections.abc:Mapping", Wrap.PyDict, Wrap.PyDict) + pyconvert_add_rule(Wrap.pyconvert_rule_io, "io:IOBase", Wrap.PyIO, Wrap.PyIO) + pyconvert_add_rule(Wrap.pyconvert_rule_io, "_io:_IOBase", Wrap.PyIO, Wrap.PyIO) + pyconvert_add_rule( + Wrap.pyconvert_rule_pandasdataframe, + "pandas.core.frame:DataFrame", + Wrap.PyPandasDataFrame, + Wrap.PyPandasDataFrame, + ) + pyconvert_add_rule( + Wrap.pyconvert_rule_sequence, + "pandas.core.arrays.base:ExtensionArray", + Wrap.PyList, + Wrap.PyList, + ) + pyconvert_add_rule(Wrap.pyconvert_rule_array, "", Array, Array) + pyconvert_add_rule(Wrap.pyconvert_rule_array, "", Array, Array) + pyconvert_add_rule(Wrap.pyconvert_rule_array, "", Array, Array) + pyconvert_add_rule(Wrap.pyconvert_rule_array, "", Array, Array) + pyconvert_add_rule(Wrap.pyconvert_rule_array, "", AbstractArray, AbstractArray) + pyconvert_add_rule(Wrap.pyconvert_rule_array, "", AbstractArray, AbstractArray) + pyconvert_add_rule(Wrap.pyconvert_rule_array, "", AbstractArray, AbstractArray) + pyconvert_add_rule(Wrap.pyconvert_rule_array, "", AbstractArray, AbstractArray) + + # JlWrap rules + pyconvert_add_rule(JlWrap.pyconvert_rule_jlvalue, "juliacall:JlBase", Any, Any) + + # Fallback + pyconvert_add_rule(Convert.pyconvert_rule_object, "builtins:object", Py, Any) +end + end diff --git a/src/Wrap/Wrap.jl b/src/Wrap/Wrap.jl index e67129cb..e5032d2d 100644 --- a/src/Wrap/Wrap.jl +++ b/src/Wrap/Wrap.jl @@ -31,52 +31,4 @@ include("PyIO.jl") include("PyTable.jl") include("PyPandasDataFrame.jl") -function __init__() - priority = PYCONVERT_PRIORITY_ARRAY - pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) - pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) - pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) - pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) - - priority = PYCONVERT_PRIORITY_CANONICAL - pyconvert_add_rule( - "collections.abc:Iterable", - PyIterable, - pyconvert_rule_iterable, - priority, - ) - pyconvert_add_rule( - "collections.abc:Sequence", - PyList, - pyconvert_rule_sequence, - priority, - ) - pyconvert_add_rule("collections.abc:Set", PySet, pyconvert_rule_set, priority) - pyconvert_add_rule("collections.abc:Mapping", PyDict, pyconvert_rule_mapping, priority) - pyconvert_add_rule("io:IOBase", PyIO, pyconvert_rule_io, priority) - pyconvert_add_rule("_io:_IOBase", PyIO, pyconvert_rule_io, priority) - pyconvert_add_rule( - "pandas.core.frame:DataFrame", - PyPandasDataFrame, - pyconvert_rule_pandasdataframe, - priority, - ) - pyconvert_add_rule( - "pandas.core.arrays.base:ExtensionArray", - PyList, - pyconvert_rule_sequence, - priority, - ) - - priority = PYCONVERT_PRIORITY_NORMAL - pyconvert_add_rule("", Array, pyconvert_rule_array, priority) - pyconvert_add_rule("", Array, pyconvert_rule_array, priority) - pyconvert_add_rule("", Array, pyconvert_rule_array, priority) - pyconvert_add_rule("", Array, pyconvert_rule_array, priority) - pyconvert_add_rule("", AbstractArray, pyconvert_rule_array, priority) - pyconvert_add_rule("", AbstractArray, pyconvert_rule_array, priority) - pyconvert_add_rule("", AbstractArray, pyconvert_rule_array, priority) - pyconvert_add_rule("", AbstractArray, pyconvert_rule_array, priority) -end - end diff --git a/test/Compat.jl b/test/Compat.jl index ed230a71..407c5974 100644 --- a/test/Compat.jl +++ b/test/Compat.jl @@ -104,12 +104,16 @@ end end end -@testitem "Tables.jl" begin +@testitem "Tables.jl" setup=[Setup] begin @testset "pytable" begin x = (x = [1, 2, 3], y = ["a", "b", "c"]) # pandas - # TODO: install pandas and test properly - @test_throws PyException pytable(x, :pandas) + if Setup.devdeps + y = pytable(x, :pandas) + @test pyisinstance(y, pyimport("pandas").DataFrame) + else + @test_throws PyException pytable(x, :pandas) + end # columns y = pytable(x, :columns) @test pyeq(Bool, y, pydict(x = [1, 2, 3], y = ["a", "b", "c"])) diff --git a/test/Convert.jl b/test/Convert.jl index 4f137459..d81d4787 100644 --- a/test/Convert.jl +++ b/test/Convert.jl @@ -265,6 +265,52 @@ end @test_throws Exception pyconvert(Microsecond, td(days = -200_000_000)) end +@testitem "ctypes scalars" begin + ctypes = pyimport("ctypes") + + @test pyconvert(Int, ctypes.c_int(42)) === 42 + @test pyconvert(Float64, ctypes.c_double(3.25)) === 3.25 + @test pyconvert(Ptr{Cvoid}, ctypes.c_void_p(0)) == C_NULL +end + +@testitem "numpy scalars" setup=[Setup] begin + if Setup.devdeps + np = pyimport("numpy") + + @test pyconvert(Int, np.int64(5)) === 5 + @test pyconvert(Float32, np.float32(1.25)) === Float32(1.25) + @test pyconvert(Missing, np.datetime64("NaT")) === missing + @test pyconvert(Missing, np.timedelta64("NaT")) === missing + else + @test_skip Setup.devdeps + end +end + +@testitem "numpy array → PyArray" setup=[Setup] begin + if Setup.devdeps + np = pyimport("numpy") + + arr = np.array([1, 2, 3], dtype = np.int64) + pyarr = pyconvert(PyArray, arr) + + @test pyarr isa PyArray + @test collect(pyarr) == [1, 2, 3] + else + @test_skip Setup.devdeps + end +end + +@testitem "pandas NA" setup=[Setup] begin + if Setup.devdeps + pd = pyimport("pandas") + + @test pyconvert(Missing, pd.NA) === missing + @test pyconvert(Nothing, pd.NA) === nothing + else + @test_skip Setup.devdeps + end +end + @testitem "timedelta → Millisecond" begin using Dates td = pyimport("datetime").timedelta @@ -318,9 +364,10 @@ end @test pyconvert(Any, x) === x # This test has a side effect of influencing the rules cache t = pytype(x) PythonCall.pyconvert_add_rule( + (_, _) -> "Hello!!", "$(t.__module__):$(t.__qualname__)", String, - (_, _) -> "Hello!!", + Any, ) @test pyconvert(String, x) == "Hello!!" @test pyconvert(Any, x) == "Hello!!" # Broken before PR #365