diff --git a/src/PyCall.jl b/src/PyCall.jl index 2b7003c2..95014f46 100644 --- a/src/PyCall.jl +++ b/src/PyCall.jl @@ -40,20 +40,11 @@ current_python() = _current_python[] const _current_python = Ref(pyprogramname) ######################################################################### - -# Mirror of C PyObject struct (for non-debugging Python builds). -# We won't actually access these fields directly; we'll use the Python -# C API for everything. However, we need to define a unique Ptr type -# for PyObject*, and we might as well define the actual struct layout -# while we're at it. -struct PyObject_struct - ob_refcnt::Int - ob_type::Ptr{Cvoid} -end - -const PyPtr = Ptr{PyObject_struct} # type for PythonObject* in ccall - -const PyPtr_NULL = PyPtr(C_NULL) +# interface to libpython +include("libpython/types.jl") +include("libpython/functions.jl") +include("libpython/globals.jl") +include("libpython/extensions.jl") ######################################################################### # Wrapper around Python's C PyObject* type, with hooks to Python reference @@ -154,17 +145,25 @@ wrapping/converted from `x` is created. """ pyreturn(x) = PyPtr(pyincref(PyObject(x))) +macro pyreturn(x) + :(return pyreturn($(esc(x)))) +end + function Base.copy!(dest::PyObject, src::PyObject) pydecref(dest) setfield!(dest, :o, PyPtr(pyincref(src))) return dest end -pyisinstance(o::PyObject, t::PyObject) = - !ispynull(t) && ccall((@pysym :PyObject_IsInstance), Cint, (PyPtr,PyPtr), o, t) == 1 +pyissubclass(o, t) = GC.@preserve o t pyissubclass(PyPtr(o), PyPtr(t)) + +pyissubclass(o::PyPtr, t::PyPtr) = + o != C_NULL && t != C_NULL && CPyObject_IsSubclass(o, t)==1 + +pyisinstance(o, t) = GC.@preserve o t pyisinstance(PyPtr(o), PyPtr(t)) -pyisinstance(o::PyObject, t::Union{Ptr{Cvoid},PyPtr}) = - t != C_NULL && ccall((@pysym :PyObject_IsInstance), Cint, (PyPtr,PyPtr), o, t) == 1 +pyisinstance(o::PyPtr, t::PyPtr) = + o != C_NULL && t != C_NULL && CPyObject_IsInstance(o, t)==1 pyquery(q::Ptr{Cvoid}, o::PyObject) = ccall(q, Cint, (PyPtr,), o) == 1 @@ -187,14 +186,19 @@ pytypeof(o::PyObject) = ispynull(o) ? throw(ArgumentError("NULL PyObjects have n ######################################################################### const TypeTuple = Union{Type,NTuple{N, Type}} where {N} +include("conversions.jl") include("pybuffer.jl") include("pyarray.jl") -include("conversions.jl") -include("pytype.jl") +include("pydict.jl") +include("pyvector.jl") +# include("pytype.jl") +include("jlwrap.jl") include("pyiterator.jl") include("pyclass.jl") include("callback.jl") include("io.jl") +include("numpy.jl") +include("pydates.jl") ######################################################################### @@ -212,18 +216,22 @@ PyObject(o::PyPtr, keep::Any) = pyembed(PyObject(o), keep) Return a string representation of `o` corresponding to `str(o)` in Python. """ -pystr(o::PyObject) = convert(AbstractString, - PyObject(@pycheckn ccall((@pysym :PyObject_Str), PyPtr, - (PyPtr,), o))) +function pystr(o::PyObject) + r = CPyObject_Str(String, o) + r===nothing && _handle_error("pystr") + r +end """ pyrepr(o::PyObject) Return a string representation of `o` corresponding to `repr(o)` in Python. """ -pyrepr(o::PyObject) = convert(AbstractString, - PyObject(@pycheckn ccall((@pysym :PyObject_Repr), PyPtr, - (PyPtr,), o))) +function pyrepr(o::PyObject) + r = CPyObject_Repr(String, o) + r===nothing && _handle_error("pyrepr") + r +end """ pystring(o::PyObject) @@ -236,16 +244,13 @@ function pystring(o::PyObject) if ispynull(o) return "NULL" else - s = ccall((@pysym :PyObject_Repr), PyPtr, (PyPtr,), o) - if (s == C_NULL) - pyerr_clear() - s = ccall((@pysym :PyObject_Str), PyPtr, (PyPtr,), o) - if (s == C_NULL) - pyerr_clear() - return string(PyPtr(o)) - end - end - return convert(AbstractString, PyObject(s)) + r = CPyObject_Repr(String, o) + r===nothing || (return r) + CPyErr_Clear() + r = CPyObject_Str(String, o) + r===nothing || (return r) + CPyErr_Clear() + return convert(String, string(PyPtr(o))) end end @@ -275,7 +280,7 @@ function hash(o::PyObject) elseif is_pyjlwrap(o) # call native Julia hash directly on wrapped Julia objects, # since on 64-bit Windows the Python 2.x hash is only 32 bits - hashsalt(unsafe_pyjlwrap_to_objref(o)) + hashsalt(unsafe_pyjlwrap_load_value(o)) else h = ccall((@pysym :PyObject_Hash), Py_hash_t, (PyPtr,), o) if h == -1 # error @@ -305,7 +310,8 @@ end getproperty(o::PyObject, s::Symbol) = convert(PyAny, getproperty(o, String(s))) -propertynames(o::PyObject) = ispynull(o) ? Symbol[] : map(x->Symbol(first(x)), pycall(inspect."getmembers", PyObject, o)) +propertynames(o::PyObject) = + ispynull(o) ? Symbol[] : map(Symbol∘pystr∘first∘PyIterator{PyObject}, PyIterator{PyObject}(pycall(inspect."getmembers", PyObject, o))) # avoiding method ambiguity setproperty!(o::PyObject, s::Symbol, v) = _setproperty!(o,s,v) @@ -917,11 +923,11 @@ include("pyinit.jl") # Here, we precompile functions that are passed to cfunction by __init__, # for the reasons described in JuliaLang/julia#12256. -precompile(pyjlwrap_call, (PyPtr,PyPtr,PyPtr)) -precompile(pyjlwrap_dealloc, (PyPtr,)) -precompile(pyjlwrap_repr, (PyPtr,)) -precompile(pyjlwrap_hash, (PyPtr,)) -precompile(pyjlwrap_hash32, (PyPtr,)) +precompile(_pyjlwrap_call, (PyPtr,PyPtr,PyPtr)) +precompile(_pyjlwrap_dealloc, (PyPtr,)) +precompile(_pyjlwrap_repr, (PyPtr,)) +precompile(_pyjlwrap_hash, (PyPtr,)) +precompile(_pyjlwrap_hash32, (PyPtr,)) # TODO: precompilation of the io.jl functions diff --git a/src/builtins.jl b/src/builtins.jl new file mode 100644 index 00000000..ab191c7c --- /dev/null +++ b/src/builtins.jl @@ -0,0 +1,109 @@ +""" + pystr(o::PyObject) + +Return a string representation of `o` corresponding to `str(o)` in Python. +""" +pystr(args...; kwargs...) = + pystr(AbstractString, args...; kwargs...) +pystr(T::Type{<:AbstractString}, args...; kwargs...) = + convert(T, pystr(PyObject, args...; kwargs...)) +pystr(::Type{PyObject}, o) = + PyObject(@pycheckn ccall(@pysym(:PyObject_Str), PyPtr, (PyPtr,), PyObject(o))) +pystr(::Type{PyObject}, o::AbstractString="") = + convertpystr(o) + +""" + pyrepr(o::PyObject) + +Return a string representation of `o` corresponding to `repr(o)` in Python. +""" +pyrepr(o) = + pystr(AbstractString, o) +pyrepr(::Type{T}, o) where {T<:AbstractString} = + convert(T, pyrepr(PyObject, o)) +pyrepr(::Type{PyObject}, o) = + PyObject(@pycheckn ccall(@pysym(:PyObject_Repr), PyPtr, (PyPtr,), PyObject(o))) + +""" + pyisinstance(o::PyObject, t::PyObject) + +True if `o` is an instance of `t`, corresponding to `isinstance(o,t)` in Python. +""" +pyisinstance(o, t) = + pyisinstance(Bool, o, t) +pyisinstance(::Type{Bool}, o, t) = + pyisinstance(Bool, PyObject(o), PyObject(t)) +pyisinstance(::Type{Bool}, o::PyObject, t::PyObject) = + !ispynull(t) && ccall((@pysym :PyObject_IsInstance), Cint, (PyPtr,PyPtr), o, t) == 1 +pyisinstance(::Type{Bool}, o::PyObject, t::Union{Ptr{Cvoid},PyPtr}) = + t != C_NULL && ccall((@pysym :PyObject_IsInstance), Cint, (PyPtr,PyPtr), o, t) == 1 +pyisinstance(::Type{PyObject}, o, t) = + convertpybool(pyisinstance(Bool, o, t)) + +""" + pyistrue(o::PyObject) + +True if `o` is considered to be true, corresponding to `not not o` in Python. +""" +pyistrue(o::PyObject) = + pyistrue(Bool, o) +pyistrue(::Type{Bool}, o::PyObject) = + (@pycheckz ccall(@pysym(:PyObject_IsTrue), Cint, (PyPtr,), o)) == 1 +pyistrue(::Type{PyObject}, o) = + convertpybool(pyistrue(Bool, o)) + +""" + pynot(o::PyObject) + +True if `o` is not considered to be true, corresponding to `not o` in Python. +""" +pynot(o::PyObject) = + pynot(Bool, o) +pynot(::Type{Bool}, o::PyObject) = + (@pycheckz ccall(@pysym(:PyObject_Not), Cint, (PyPtr,), o)) == 1 +pynot(::Type{PyObject}, o) = + convertpybool(pynot(Bool, o)) + +""" + pyint(o::PyObject) +""" +pyint(args...; kwargs...) = + pyint(PyObject, args...; kwargs...) +pyint(T::Type, args...; kwargs...) = + pycall(pybuiltin("int"), T, args...; kwargs...) +pyint(::Type{PyObject}, o::Integer=0) = + convertpyint(o) + +""" + pybool(o::PyObject) +""" +pybool(args...; kwargs...) = + pybool(PyObject, args...; kwargs...) +pybool(T::Type, args...; kwargs...) = + pycall(pybuiltin("bool"), T, args...; kwargs...) +pybool(::Type{PyObject}, o::Bool=false) = + convertpybool(o) + +""" + pystring(o::PyObject) + +Return a string representation of `o`. Normally, this corresponds to `repr(o)` +in Python, but unlike `repr` it should never fail (falling back to `str` or to +printing the raw pointer if necessary). +""" +function pystring(o::PyObject) + if ispynull(o) + return "NULL" + else + s = ccall((@pysym :PyObject_Repr), PyPtr, (PyPtr,), o) + if (s == C_NULL) + pyerr_clear() + s = ccall((@pysym :PyObject_Str), PyPtr, (PyPtr,), o) + if (s == C_NULL) + pyerr_clear() + return string(PyPtr(o)) + end + end + return convert(AbstractString, PyObject(s)) + end +end diff --git a/src/callback.jl b/src/callback.jl index 492018b7..1e2dd0f1 100644 --- a/src/callback.jl +++ b/src/callback.jl @@ -18,37 +18,6 @@ julia_args(f, args) = convert(PyAny, args) julia_kwarg(f, kw, arg) = convert(PyAny, arg) -function _pyjlwrap_call(f, args_::PyPtr, kw_::PyPtr) - args = PyObject(args_) # don't need pyincref because of finally clause below - try - jlargs = julia_args(f, args) - - # we need to use invokelatest to get execution in newest world - if kw_ == C_NULL - ret = Base.invokelatest(f, jlargs...) - else - kw = PyDict{Symbol,PyObject}(pyincref(kw_)) - kwargs = [ (k,julia_kwarg(f,k,v)) for (k,v) in kw ] - - # 0.6 `invokelatest` doesn't support kwargs, instead - # use a closure over kwargs. see: - # https://github.com/JuliaLang/julia/pull/22646 - f_kw_closure() = f(jlargs...; kwargs...) - ret = Core._apply_latest(f_kw_closure) - end - - return pyreturn(ret) - catch e - @pyraise e - finally - setfield!(args, :o, PyPtr_NULL) # don't decref - end - return PyPtr_NULL -end - -pyjlwrap_call(self_::PyPtr, args_::PyPtr, kw_::PyPtr) = - _pyjlwrap_call(unsafe_pyjlwrap_to_objref(self_), args_, kw_) - ################################################################ # allow the user to convert a Julia function into a Python # function with specified argument types, both to give more control diff --git a/src/conversions.jl b/src/conversions.jl index d5c82e0c..ddd5c5b3 100644 --- a/src/conversions.jl +++ b/src/conversions.jl @@ -1,158 +1,14 @@ # Conversions between Julia and Python types for the PyCall module. -######################################################################### -# Conversions of simple types (numbers and nothing) - -# conversions from Julia types to PyObject: - -@static if pyversion < v"3" - PyObject(i::Unsigned) = PyObject(@pycheckn ccall(@pysym(:PyInt_FromSize_t), - PyPtr, (UInt,), i)) - PyObject(i::Integer) = PyObject(@pycheckn ccall(@pysym(:PyInt_FromSsize_t), - PyPtr, (Int,), i)) -else - PyObject(i::Unsigned) = PyObject(@pycheckn ccall(@pysym(:PyLong_FromUnsignedLongLong), - PyPtr, (Culonglong,), i)) - PyObject(i::Integer) = PyObject(@pycheckn ccall(@pysym(:PyLong_FromLongLong), - PyPtr, (Clonglong,), i)) -end - -PyObject(b::Bool) = PyObject(@pycheckn ccall((@pysym :PyBool_FromLong), - PyPtr, (Clong,), b)) - -PyObject(r::Real) = PyObject(@pycheckn ccall((@pysym :PyFloat_FromDouble), - PyPtr, (Cdouble,), r)) - -PyObject(c::Complex) = PyObject(@pycheckn ccall((@pysym :PyComplex_FromDoubles), - PyPtr, (Cdouble,Cdouble), - real(c), imag(c))) - -PyObject(n::Nothing) = pyerr_check("PyObject(nothing)", pyincref(pynothing[])) - -# conversions to Julia types from PyObject - -@static if pyversion < v"3" - convert(::Type{T}, po::PyObject) where {T<:Integer} = - T(@pycheck ccall(@pysym(:PyInt_AsSsize_t), Int, (PyPtr,), po)) -elseif pyversion < v"3.2" - convert(::Type{T}, po::PyObject) where {T<:Integer} = - T(@pycheck ccall(@pysym(:PyLong_AsLongLong), Clonglong, (PyPtr,), po)) -else - function convert(::Type{T}, po::PyObject) where {T<:Integer} - overflow = Ref{Cint}() - val = T(@pycheck ccall(@pysym(:PyLong_AsLongLongAndOverflow), Clonglong, (PyPtr, Ref{Cint}), po, overflow)) - iszero(overflow[]) || throw(InexactError(:convert, T, po)) - return val - end - function convert(::Type{Integer}, po::PyObject) - overflow = Ref{Cint}() - val = @pycheck ccall(@pysym(:PyLong_AsLongLongAndOverflow), Clonglong, (PyPtr, Ref{Cint}), po, overflow) - iszero(overflow[]) || return convert(BigInt, po) - return val - end -end - -convert(::Type{Bool}, po::PyObject) = - 0 != @pycheck ccall(@pysym(:PyObject_IsTrue), Cint, (PyPtr,), po) +# Conversion from julia to python defers to CPyObject_From, which is partly defined in libpython/extensions.jl, and partly defined below +PyObject(x) = PyObject(CPyObject_From(x)) -convert(::Type{T}, po::PyObject) where {T<:Real} = - T(@pycheck ccall(@pysym(:PyFloat_AsDouble), Cdouble, (PyPtr,), po)) - -convert(::Type{T}, po::PyObject) where T<:Complex = - T(@pycheck ccall(@pysym(:PyComplex_AsCComplex), Complex{Cdouble}, (PyPtr,), po)) - -convert(::Type{Nothing}, po::PyObject) = nothing - -function Base.float(o::PyObject) - a = PyAny(o) - if a isa PyObject - hasproperty(o, :__float__) && return o.__float__() - throw(ArgumentError("don't know how convert $o to a Julia floating-point value")) - end - return float(a) -end - -######################################################################### -# String conversions (both bytes arrays and unicode strings) - -function PyObject(s::AbstractString) - sb = String(s) - if pyunicode_literals || !isascii(sb) - PyObject(@pycheckn ccall(@pysym(PyUnicode_DecodeUTF8), - PyPtr, (Ptr{UInt8}, Int, Ptr{UInt8}), - sb, sizeof(sb), C_NULL)) - else - pybytes(sb) - end -end - -const _ps_ptr= Ptr{UInt8}[C_NULL] -const _ps_len = Int[0] -function convert(::Type{T}, po::PyObject) where T<:AbstractString - if pyisinstance(po, @pyglobalobj :PyUnicode_Type) - convert(T, PyObject(@pycheckn ccall(@pysym(PyUnicode_AsUTF8String), - PyPtr, (PyPtr,), po))) - else - @pycheckz ccall(@pysym(PyString_AsStringAndSize), - Cint, (PyPtr, Ptr{Ptr{UInt8}}, Ptr{Int}), - po, _ps_ptr, _ps_len) - convert(T, unsafe_string(_ps_ptr[1], _ps_len[1])) - end -end - -# TODO: should symbols be converted to a subclass of Python strings/bytes, -# so that PyAny conversion can convert it back to a Julia symbol? -PyObject(s::Symbol) = PyObject(string(s)) -convert(::Type{Symbol}, po::PyObject) = Symbol(convert(AbstractString, po)) - -######################################################################### -# ByteArray conversions - -function PyObject(a::DenseVector{UInt8}) - if stride(a,1) != 1 - try - return NpyArray(a, true) - catch - return array2py(a) # fallback to non-NumPy version - end - end - PyObject(@pycheckn ccall((@pysym :PyByteArray_FromStringAndSize), - PyPtr, (Ptr{UInt8}, Int), a, length(a))) -end - -ispybytearray(po::PyObject) = - pyisinstance(po, @pyglobalobj :PyByteArray_Type) - -function convert(::Type{Vector{UInt8}}, po::PyObject) - b = PyBuffer(po) - iscontiguous(b) || error("a contiguous buffer is required") - return copy(unsafe_wrap(Array, Ptr{UInt8}(pointer(b)), sizeof(b))) -end - -# TODO: support zero-copy PyByteArray <: AbstractVector{UInt8} object - -######################################################################### -# Pointer conversions, using ctypes or PyCapsule - -PyObject(p::Ptr) = pycall(c_void_p_Type, PyObject, UInt(p)) - -function convert(::Type{Ptr{Cvoid}}, po::PyObject) - if pyisinstance(po, c_void_p_Type) - v = po."value" - # ctypes stores the NULL pointer specially, grrr - pynothing_query(v) == Nothing ? C_NULL : - convert(Ptr{Cvoid}, convert(UInt, po."value")) - elseif pyisinstance(po, @pyglobalobj(:PyCapsule_Type)) - @pycheck ccall((@pysym :PyCapsule_GetPointer), - Ptr{Cvoid}, (PyPtr,Ptr{UInt8}), - po, ccall((@pysym :PyCapsule_GetName), - Ptr{UInt8}, (PyPtr,), po)) - else - convert(Ptr{Cvoid}, convert(UInt, po)) - end -end - -pyptr_query(po::PyObject) = pyisinstance(po, c_void_p_Type) || pyisinstance(po, @pyglobalobj(:PyCapsule_Type)) ? Ptr{Cvoid} : Union{} +CPyObject_From(x) = + CPyJlWrap_From(x) +CPyObject_From(x::PyObject) = + CPyObject_From(PyPtr(x)) +CPyObject_From(t::Ref{CPyTypeObject}) = + CPyObject_From(Base.unsafe_convert(PyPtr, t)) ######################################################################### # for automatic conversions, I pass Vector{PyAny}, NTuple{N, PyAny}, etc., @@ -173,665 +29,607 @@ pyany_toany(t::Type{T}) where {T<:Tuple} = Tuple{map(pyany_toany, t.types)...} # PyAny acts like Any for conversions, except for converting PyObject (below) convert(::Type{PyAny}, x) = x -######################################################################### -# Function conversion (see callback.jl for conversion the other way) -# (rarely needed given call overloading in Julia 0.4) - -convert(::Type{Function}, po::PyObject) = - function fn(args...; kwargs...) - pycall(po, PyAny, args...; kwargs...) - end - -######################################################################### -# Tuple conversion. Julia Pairs are treated as Python tuples. - -function PyObject(t::Union{Tuple,Pair}) - len = lastindex(t) # lastindex, not length, because of julia#14924 - o = PyObject(@pycheckn ccall((@pysym :PyTuple_New), PyPtr, (Int,), len)) - for i = 1:len - oi = PyObject(t[i]) - @pycheckz ccall((@pysym :PyTuple_SetItem), Cint, (PyPtr,Int,PyPtr), - o, i-1, oi) - pyincref(oi) # PyTuple_SetItem steals the reference - end - return o -end - -# somewhat annoying to get the length and types in a tuple type -# ... would be better not to have to use undocumented internals! -istuplen(T,isva,n) = isva ? n ≥ length(T.parameters)-1 : n == length(T.parameters) -function tuptype(T::DataType,isva,i) - if isva && i ≥ length(T.parameters) - return Base.unwrapva(T.parameters[end]) - else - return T.parameters[i] - end -end -tuptype(T::UnionAll,isva,i) = tuptype(T.body,isva,i) -isvatuple(T::UnionAll) = isvatuple(T.body) -isvatuple(T::DataType) = !isempty(T.parameters) && Base.isvarargtype(T.parameters[end]) - -function convert(tt::Type{T}, o::PyObject) where T<:Tuple - isva = isvatuple(T) - len = @pycheckz ccall((@pysym :PySequence_Size), Int, (PyPtr,), o) - if !istuplen(tt, isva, len) - throw(BoundsError()) - end - ntuple((i -> - convert(tuptype(T, isva, i), - PyObject(ccall((@pysym :PySequence_GetItem), PyPtr, - (PyPtr, Int), o, i-1)))), - len) -end - -function convert(::Type{Pair{K,V}}, o::PyObject) where {K,V} - k, v = convert(Tuple{K,V}, o) - return Pair(k, v) -end - -######################################################################### -# PyVector: no-copy wrapping of a Julia object around a Python sequence - -""" - PyVector(o::PyObject) - -This returns a PyVector object, which is a wrapper around an arbitrary Python list or sequence object. - -Alternatively, `PyVector` can be used as the return type for a `pycall` that returns a sequence object (including tuples). -""" -mutable struct PyVector{T} <: AbstractVector{T} - o::PyObject - function PyVector{T}(o::PyObject) where T - if ispynull(o) - throw(ArgumentError("cannot make PyVector from NULL PyObject")) - end - new{T}(o) - end -end - -PyVector(o::PyObject) = PyVector{PyAny}(o) -PyObject(a::PyVector) = a.o -convert(::Type{PyVector}, o::PyObject) = PyVector(o) -convert(::Type{PyVector{T}}, o::PyObject) where {T} = PyVector{T}(o) -unsafe_convert(::Type{PyPtr}, a::PyVector) = PyPtr(a.o) -PyVector(a::PyVector) = a -PyVector(a::AbstractVector{T}) where {T} = PyVector{T}(array2py(a)) - -# when a PyVector is copied it is converted into an ordinary Julia Vector -similar(a::PyVector, T, dims::Dims) = Array{T}(dims) -similar(a::PyVector{T}) where {T} = similar(a, pyany_toany(T), size(a)) -similar(a::PyVector{T}, dims::Dims) where {T} = similar(a, pyany_toany(T), dims) -similar(a::PyVector{T}, dims::Int...) where {T} = similar(a, pyany_toany(T), dims) -eltype(::PyVector{T}) where {T} = pyany_toany(T) -eltype(::Type{PyVector{T}}) where {T} = pyany_toany(T) - -size(a::PyVector) = (length(a.o),) - -getindex(a::PyVector) = getindex(a, 1) -getindex(a::PyVector{T}, i::Integer) where {T} = convert(T, PyObject(@pycheckn ccall((@pysym :PySequence_GetItem), PyPtr, (PyPtr, Int), a, i-1))) - -setindex!(a::PyVector, v) = setindex!(a, v, 1) -function setindex!(a::PyVector, v, i::Integer) - @pycheckz ccall((@pysym :PySequence_SetItem), Cint, (PyPtr, Int, PyPtr), a, i-1, PyObject(v)) - v -end - -summary(a::PyVector{T}) where {T} = string(Base.dims2string(size(a)), " ", - string(pyany_toany(T)), " PyVector") - -splice!(a::PyVector, i::Integer) = splice!(a.o, i) -function splice!(a::PyVector{T}, indices::AbstractVector{I}) where {T,I<:Integer} - v = pyany_toany(T)[a[i] for i in indices] - for i in sort(indices, rev=true) - @pycheckz ccall((@pysym :PySequence_DelItem), Cint, (PyPtr, Int), a, i-1) - end - v -end -pop!(a::PyVector) = pop!(a.o) -popfirst!(a::PyVector) = popfirst!(a.o) -empty!(a::PyVector) = empty!(a.o) - -# only works for List subtypes: -push!(a::PyVector, item) = push!(a.o, item) -insert!(a::PyVector, i::Integer, item) = insert!(a.o, i, item) -pushfirst!(a::PyVector, item) = pushfirst!(a.o, item) -prepend!(a::PyVector, items) = prepend!(a.o, items) -append!(a::PyVector{T}, items) where {T} = PyVector{T}(append!(a.o, items)) - -######################################################################### -# Lists and 1d arrays. - -if VERSION < v"1.1.0-DEV.392" # #29440 - cirange(I,J) = CartesianIndices(map((i,j) -> i:j, Tuple(I), Tuple(J))) -else - cirange(I,J) = I:J -end - -# recursive conversion of A to a list of list of lists... starting -# with dimension dim and Cartesian index i in A. -function array2py(A::AbstractArray{<:Any, N}, dim::Integer, i::CartesianIndex{N}) where {N} - if dim > N # base case - return PyObject(A[i]) - else # recursively store multidimensional array as list of lists - ilast = CartesianIndex(ntuple(j -> j == dim ? lastindex(A, dim) : i[j], Val{N}())) - o = PyObject(@pycheckn ccall((@pysym :PyList_New), PyPtr, (Int,), size(A, dim))) - for icur in cirange(i,ilast) - oi = array2py(A, dim+1, icur) - @pycheckz ccall((@pysym :PyList_SetItem), Cint, (PyPtr,Int,PyPtr), - o, icur[dim]-i[dim], oi) - pyincref(oi) # PyList_SetItem steals the reference - end +function convert(::Type{PyAny}, o::PyObject) + if ispynull(o) return o end -end - -array2py(A::AbstractArray) = array2py(A, 1, first(CartesianIndices(A))) - -PyObject(A::AbstractArray) = - ndims(A) <= 1 || hasmethod(stride, Tuple{typeof(A),Int}) ? array2py(A) : - pyjlwrap_new(A) - -function py2array(T, A::Array{TA,N}, o::PyObject, - dim::Integer, i::Integer) where {TA,N} - if dim > N - A[i] = convert(T, o) - return A - elseif dim == N - len = @pycheckz ccall((@pysym :PySequence_Size), Int, (PyPtr,), o) - if len != size(A, dim) - error("dimension mismatch in py2array") - end - s = stride(A, dim) - for j = 0:len-1 - A[i+j*s] = convert(T, PyObject(ccall((@pysym :PySequence_GetItem), - PyPtr, (PyPtr, Int), o, j))) - end - return A - else # dim < N: recursively extract list of lists into A - len = @pycheckz ccall((@pysym :PySequence_Size), Int, (PyPtr,), o) - if len != size(A, dim) - error("dimension mismatch in py2array") - end - s = stride(A, dim) - for j = 0:len-1 - py2array(T, A, PyObject(ccall((@pysym :PySequence_GetItem), - PyPtr, (PyPtr, Int), o, j)), - dim+1, i+j*s) - end - return A - end -end - -# figure out if we can treat o as a multidimensional array, and return -# the dimensions -function pyarray_dims(o::PyObject, forcelist=true) - if !(forcelist || pyisinstance(o, @pyglobalobj :PyList_Type)) - return () # too many non-List types can pretend to be sequences - end - len = ccall((@pysym :PySequence_Size), Int, (PyPtr,), o) - len < 0 && error("not a PySequence object") - if len == 0 - return (0,) - end - dims0 = pyarray_dims(PyObject(ccall((@pysym :PySequence_GetItem), - PyPtr, (PyPtr, Int), o, 0)), - false) - if isempty(dims0) # not a nested sequence - return (len,) - end - for j = 1:len-1 - dims = pyarray_dims(PyObject(ccall((@pysym :PySequence_GetItem), - PyPtr, (PyPtr, Int), o, j)), - false) - if dims != dims0 - # elements don't have equal lengths, cannot - # treat as multidimensional array - return (len,) - end - end - return tuple(len, dims0...) -end - -function py2array(T, o::PyObject) - b = PyBuffer() - if isbuftype!(o, b) - dims = size(b) - else - dims = pyarray_dims(o) - end - pydecref(b) # safe for immediate release - A = Array{pyany_toany(T)}(undef, dims) - py2array(T, A, o, 1, 1) # fixme: faster conversion for supported buffer types? -end - -function py2vector(T, o::PyObject) - len = ccall((@pysym :PySequence_Size), Int, (PyPtr,), o) - if len < 0 || # not a sequence - len+1 < 0 # object pretending to be a sequence of infinite length - pyerr_clear() - throw(ArgumentError("expected Python sequence")) - end - py2array(T, Array{pyany_toany(T)}(undef, len), o, 1, 1) -end -convert(::Type{Vector{T}}, o::PyObject) where T = py2vector(T, o) - -convert(::Type{Array}, o::PyObject) = map(identity, py2array(PyAny, o)) -convert(::Type{Array{T}}, o::PyObject) where {T} = py2array(T, o) - -PyObject(a::BitArray) = PyObject(Array(a)) - -# NumPy conversions (multidimensional arrays) -include("numpy.jl") - -######################################################################### -# PyDict: no-copy wrapping of a Julia object around a Python dictionary - -# we check for "items" attr since PyMapping_Check doesn't do this (it only -# checks for __getitem__) and PyMapping_Check returns true for some -# scipy scalar array members, grrr. -function is_mapping_object(o::PyObject) - pyisinstance(o, @pyglobalobj :PyDict_Type) || - (pyquery((@pyglobal :PyMapping_Check), o) && - ccall((@pysym :PyObject_HasAttrString), Cint, (PyPtr,Ptr{UInt8}), o, "items") == 1) -end - -""" - PyDict(o::PyObject) - PyDict(d::Dict{K,V}) - -This returns a PyDict, which is a no-copy wrapper around a Python dictionary. - -Alternatively, you can specify the return type of a `pycall` as PyDict. -""" -mutable struct PyDict{K,V,isdict} <: AbstractDict{K,V} - o::PyObject - # isdict = true for python dict, otherwise is a generic Mapping object - - function PyDict{K,V,isdict}(o::PyObject) where {K,V,isdict} - if !isdict && !ispynull(o) && !is_mapping_object(o) - throw(ArgumentError("only Dict and Mapping objects can be converted to PyDict")) - end - return new{K,V,isdict}(o) - end -end - -PyDict{K,V}(o::PyObject) where {K,V} = PyDict{K,V,pyisinstance(o, @pyglobalobj :PyDict_Type)}(o) -PyDict{K,V}() where {K,V} = PyDict{K,V,true}(PyObject(@pycheckn ccall((@pysym :PyDict_New), PyPtr, ()))) - -PyDict(o::PyObject) = PyDict{PyAny,PyAny}(o) -PyObject(d::PyDict) = d.o -PyDict() = PyDict{PyAny,PyAny}() -PyDict(d::AbstractDict{K,V}) where {K,V} = PyDict{K,V}(PyObject(d)) -PyDict(d::AbstractDict{Any,Any}) = PyDict{PyAny,PyAny}(PyObject(d)) -PyDict(d::AbstractDict{Any,V}) where {V} = PyDict{PyAny,V}(PyObject(d)) -PyDict(d::AbstractDict{K,Any}) where {K} = PyDict{K,PyAny}(PyObject(d)) -convert(::Type{PyDict}, o::PyObject) = PyDict(o) -convert(::Type{PyDict{K,V}}, o::PyObject) where {K,V} = PyDict{K,V}(o) -unsafe_convert(::Type{PyPtr}, d::PyDict) = PyPtr(d.o) - -haskey(d::PyDict{K,V,true}, key) where {K,V} = 1 == ccall(@pysym(:PyDict_Contains), Cint, (PyPtr, PyPtr), d, PyObject(key)) -keys(::Type{T}, d::PyDict{K,V,true}) where {T,K,V} = convert(Vector{T}, PyObject(@pycheckn ccall((@pysym :PyDict_Keys), PyPtr, (PyPtr,), d))) -values(::Type{T}, d::PyDict{K,V,true}) where {T,K,V} = convert(Vector{T}, PyObject(@pycheckn ccall((@pysym :PyDict_Values), PyPtr, (PyPtr,), d))) - -keys(::Type{T}, d::PyDict{K,V,false}) where {T,K,V} = convert(Vector{T}, pycall(d.o["keys"], PyObject)) -values(::Type{T}, d::PyDict{K,V,false}) where {T,K,V} = convert(Vector{T}, pycall(d.o["values"], PyObject)) -haskey(d::PyDict{K,V,false}, key) where {K,V} = 1 == ccall(@pysym(:PyMapping_HasKey), Cint, (PyPtr, PyPtr), d, PyObject(key)) - -similar(d::PyDict{K,V}) where {K,V} = Dict{pyany_toany(K),pyany_toany(V)}() -eltype(::Type{PyDict{K,V}}) where {K,V} = Pair{pyany_toany(K),pyany_toany(V)} -Base.keytype(::PyDict{K,V}) where {K,V} = pyany_toany(K) -Base.valtype(::PyDict{K,V}) where {K,V} = pyany_toany(V) -Base.keytype(::Type{PyDict{K,V}}) where {K,V} = pyany_toany(K) -Base.valtype(::Type{PyDict{K,V}}) where {K,V} = pyany_toany(V) - -function setindex!(d::PyDict, v, k) - @pycheckz ccall((@pysym :PyObject_SetItem), Cint, (PyPtr, PyPtr, PyPtr), - d, PyObject(k), PyObject(v)) - v -end - -get(d::PyDict{K,V}, k, default) where {K,V} = get(d.o, V, k, default) - -function pop!(d::PyDict{K,V,true}, k) where {K,V} - v = d[k] - @pycheckz ccall(@pysym(:PyDict_DelItem), Cint, (PyPtr, PyPtr), d, PyObject(k)) - return v -end -function pop!(d::PyDict{K,V,false}, k) where {K,V} - v = d[k] - @pycheckz ccall(@pysym(:PyObject_DelItem), Cint, (PyPtr, PyPtr), d, PyObject(k)) - return v -end - -function pop!(d::PyDict, k, default) - try - return pop!(d, k) - catch - return default - end -end - -function delete!(d::PyDict{K,V,true}, k) where {K,V} - e = ccall(@pysym(:PyDict_DelItem), Cint, (PyPtr, PyPtr), d, PyObject(k)) - e == -1 && pyerr_clear() # delete! ignores errors in Julia - return d -end -function delete!(d::PyDict{K,V,false}, k) where {K,V} - e = ccall(@pysym(:PyObject_DelItem), Cint, (PyPtr, PyPtr), d, PyObject(k)) - e == -1 && pyerr_clear() # delete! ignores errors in Julia - return d -end - -function empty!(d::PyDict{K,V,true}) where {K,V} - @pycheck ccall((@pysym :PyDict_Clear), Cvoid, (PyPtr,), d) - return d -end -function empty!(d::PyDict{K,V,false}) where {K,V} - # for generic Mapping items we must delete keys one by one - for k in keys(d) - delete!(d, k) - end - return d -end - -length(d::PyDict{K,V,true}) where {K,V} = @pycheckz ccall(@pysym(:PyDict_Size), Int, (PyPtr,), d) -length(d::PyDict{K,V,false}) where {K,V} = @pycheckz ccall(@pysym(:PyObject_Size), Int, (PyPtr,), d) -isempty(d::PyDict) = length(d) == 0 - - -struct PyDict_Iterator - # arrays to pass key, value, and pos pointers to PyDict_Next - ka::Ref{PyPtr} - va::Ref{PyPtr} - pa::Ref{Int} - i::Int # current position in items list (0-based) - len::Int # length of items list -end - -function Base.iterate(d::PyDict{K,V,true}, itr=PyDict_Iterator(Ref{PyPtr}(), Ref{PyPtr}(), Ref(0), 0, length(d))) where {K,V} - itr.i >= itr.len && return nothing - if 0 == ccall((@pysym :PyDict_Next), Cint, - (PyPtr, Ref{Int}, Ref{PyPtr}, Ref{PyPtr}), - d, itr.pa, itr.ka, itr.va) - error("unexpected end of PyDict_Next") - end - ko = pyincref(itr.ka[]) # PyDict_Next returns - vo = pyincref(itr.va[]) # borrowed ref, so incref - (Pair(convert(K,ko), convert(V,vo)), - PyDict_Iterator(itr.ka, itr.va, itr.pa, itr.i+1, itr.len)) -end - -# Iterator for generic mapping, using Python items iterator. -# Our approach is to wrap an iterator over d.o["items"] -# which necessitates including d.o["items"] in the state. -function _start(d::PyDict{K,V,false}) where {K,V} - d_items = pycall(d.o."items", PyObject) - (d_items, iterate(d_items)) -end -function Base.iterate(d::PyDict{K,V,false}, itr=_start(d)) where {K,V} - d_items, iter_result = itr - iter_result === nothing && return nothing - item, state = iter_result - iter_result = iterate(d_items, state) - (item[1] => item[2], (d_items, iter_result)) -end - -######################################################################### -# Dictionary conversions (copies) - -function PyObject(d::AbstractDict) - o = PyObject(@pycheckn ccall((@pysym :PyDict_New), PyPtr, ())) - for k in keys(d) - @pycheckz ccall((@pysym :PyDict_SetItem), Cint, (PyPtr,PyPtr,PyPtr), - o, PyObject(k), PyObject(d[k])) - end + # automatic conversion back disabled for now return o -end - -function convert(::Type{Dict{K,V}}, o::PyObject) where {K,V} - copy(PyDict{K,V}(o)) -end - -######################################################################### -# AbstractRange: integer ranges are converted to xrange, -# while other ranges (<: AbstractVector) are converted to lists - -xrange(start, stop, step) = pycall(pyxrange[], PyObject, - start, stop, step) - -function PyObject(r::AbstractRange{T}) where T<:Integer - s = step(r) - f = first(r) - l = last(r) + s - if max(f,l) > typemax(Clong) || min(f,l) < typemin(Clong) - # in Python 2.x, xrange is limited to Clong - PyObject(T[r...]) - else - xrange(f, l, s) - end -end - -function convert(::Type{T}, o::PyObject) where T<:AbstractRange - v = PyVector(o) - len = length(v) - if len == 0 - return 1:0 # no way to get more info from an xrange - elseif len == 1 - start = v[1] - return start:start - else - start = v[1] - stop = v[len] - step = v[2] - start - return step == 1 ? (start:stop) : (start:step:stop) - end -end - -######################################################################### -# BigFloat and Complex{BigFloat}: convert to/from Python mpmath types - -# load mpmath module & initialize. Currently, this is done -# the first time a BigFloat is converted to Python. Alternatively, -# we could do it when PyCall is initialized (if mpmath is available), -# at the cost of slowing down initialization in the common case where -# BigFloat conversion is not needed. -const mpprec = [0] -const mpmath = PyNULL() -const mpf = PyNULL() -const mpc = PyNULL() -function mpmath_init() - if ispynull(mpmath) - copy!(mpmath, pyimport("mpmath")) - copy!(mpf, mpmath."mpf") - copy!(mpc, mpmath."mpc") - end - curprec = precision(BigFloat) - if mpprec[1] != curprec - mpprec[1] = curprec - mpmath."mp"."prec" = mpprec[1] - end -end - -# TODO: When mpmath uses MPFR internally, can we avoid the string conversions? -# Using strings will work regardless of the mpmath backend, but is annoying -# both from a performance perspective and because it is a lossy conversion -# (since strings use a decimal representation, while MPFR is binary). - -function PyObject(x::BigFloat) - mpmath_init() - pycall(mpf, PyObject, string(x)) -end - -function PyObject(x::Complex{BigFloat}) - mpmath_init() - pycall(mpc, PyObject, string(real(x)), string(imag(x))) -end - -convert(::Type{BigFloat}, o::PyObject) = parse(BigFloat, pystr(o)) - -function convert(::Type{Complex{BigFloat}}, o::PyObject) - try - Complex{BigFloat}(convert(BigFloat, o."real"), - convert(BigFloat, o."imag")) - catch - convert(Complex{BigFloat}, convert(Complex{Float64}, o)) - end -end - -pymp_query(o::PyObject) = pyisinstance(o, mpf) ? BigFloat : pyisinstance(o, mpc) ? Complex{BigFloat} : Union{} - -######################################################################### -# (Int64), Int128 and BigInt conversion to Python "long" integers - -const LongInt = @static (Sys.WORD_SIZE==32) ? Union{Int64,UInt64,Int128,UInt128,BigInt} : Union{Int128,UInt128,BigInt} - -function PyObject(i::LongInt) - PyObject(@pycheckn ccall((@pysym :PyLong_FromString), PyPtr, - (Ptr{UInt8}, Ptr{Cvoid}, Cint), - String(string(i)), C_NULL, 10)) -end - -convert(::Type{BigInt}, o::PyObject) = parse(BigInt, pystr(o)) + # try + # T = pytype_query(o) + # if T == PyObject && is_pyjlwrap(o) + # return unsafe_pyjlwrap_load_value(o) + # end + # convert(T, o) + # catch + # pyerr_clear() # just in case + # o + # end +end + +# ######################################################################### +# # Conversions of simple types (numbers and nothing) + +# # conversions from Julia types to PyObject: + +# @static if pyversion < v"3" +# PyObject(i::Unsigned) = PyObject(@pycheckn ccall(@pysym(:PyInt_FromSize_t), +# PyPtr, (UInt,), i)) +# PyObject(i::Integer) = PyObject(@pycheckn ccall(@pysym(:PyInt_FromSsize_t), +# PyPtr, (Int,), i)) +# else +# PyObject(i::Unsigned) = PyObject(@pycheckn ccall(@pysym(:PyLong_FromUnsignedLongLong), +# PyPtr, (Culonglong,), i)) +# PyObject(i::Integer) = PyObject(@pycheckn ccall(@pysym(:PyLong_FromLongLong), +# PyPtr, (Clonglong,), i)) +# end + +# PyObject(b::Bool) = PyObject(@pycheckn ccall((@pysym :PyBool_FromLong), +# PyPtr, (Clong,), b)) + +# PyObject(r::Real) = PyObject(@pycheckn ccall((@pysym :PyFloat_FromDouble), +# PyPtr, (Cdouble,), r)) + +# PyObject(c::Complex) = PyObject(@pycheckn ccall((@pysym :PyComplex_FromDoubles), +# PyPtr, (Cdouble,Cdouble), +# real(c), imag(c))) + +# PyObject(n::Nothing) = pyerr_check("PyObject(nothing)", pyincref(pynothing[])) + +# # conversions to Julia types from PyObject + +# @static if pyversion < v"3" +# convert(::Type{T}, po::PyObject) where {T<:Integer} = +# T(@pycheck ccall(@pysym(:PyInt_AsSsize_t), Int, (PyPtr,), po)) +# elseif pyversion < v"3.2" +# convert(::Type{T}, po::PyObject) where {T<:Integer} = +# T(@pycheck ccall(@pysym(:PyLong_AsLongLong), Clonglong, (PyPtr,), po)) +# else +# function convert(::Type{T}, po::PyObject) where {T<:Integer} +# overflow = Ref{Cint}() +# val = T(@pycheck ccall(@pysym(:PyLong_AsLongLongAndOverflow), Clonglong, (PyPtr, Ref{Cint}), po, overflow)) +# iszero(overflow[]) || throw(InexactError(:convert, T, po)) +# return val +# end +# function convert(::Type{Integer}, po::PyObject) +# overflow = Ref{Cint}() +# val = @pycheck ccall(@pysym(:PyLong_AsLongLongAndOverflow), Clonglong, (PyPtr, Ref{Cint}), po, overflow) +# iszero(overflow[]) || return convert(BigInt, po) +# return val +# end +# end + +# convert(::Type{Bool}, po::PyObject) = +# 0 != @pycheck ccall(@pysym(:PyObject_IsTrue), Cint, (PyPtr,), po) + +# convert(::Type{T}, po::PyObject) where {T<:Real} = +# T(@pycheck ccall(@pysym(:PyFloat_AsDouble), Cdouble, (PyPtr,), po)) + +# convert(::Type{T}, po::PyObject) where T<:Complex = +# T(@pycheck ccall(@pysym(:PyComplex_AsCComplex), Complex{Cdouble}, (PyPtr,), po)) + +# convert(::Type{Nothing}, po::PyObject) = nothing + +# function Base.float(o::PyObject) +# a = PyAny(o) +# if a isa PyObject +# hasproperty(o, :__float__) && return o.__float__() +# throw(ArgumentError("don't know how convert $o to a Julia floating-point value")) +# end +# return float(a) +# end + +# ######################################################################### +# # String conversions (both bytes arrays and unicode strings) + +# function PyObject(s::AbstractString) +# sb = String(s) +# if pyunicode_literals || !isascii(sb) +# PyObject(@pycheckn ccall(@pysym(PyUnicode_DecodeUTF8), +# PyPtr, (Ptr{UInt8}, Int, Ptr{UInt8}), +# sb, sizeof(sb), C_NULL)) +# else +# pybytes(sb) +# end +# end + +# const _ps_ptr= Ptr{UInt8}[C_NULL] +# const _ps_len = Int[0] +# function convert(::Type{T}, po::PyObject) where T<:AbstractString +# if pyisinstance(po, @pyglobalobj :PyUnicode_Type) +# convert(T, PyObject(@pycheckn ccall(@pysym(PyUnicode_AsUTF8String), +# PyPtr, (PyPtr,), po))) +# else +# @pycheckz ccall(@pysym(PyString_AsStringAndSize), +# Cint, (PyPtr, Ptr{Ptr{UInt8}}, Ptr{Int}), +# po, _ps_ptr, _ps_len) +# convert(T, unsafe_string(_ps_ptr[1], _ps_len[1])) +# end +# end + +# # TODO: should symbols be converted to a subclass of Python strings/bytes, +# # so that PyAny conversion can convert it back to a Julia symbol? +# PyObject(s::Symbol) = PyObject(string(s)) +# convert(::Type{Symbol}, po::PyObject) = Symbol(convert(AbstractString, po)) + +# ######################################################################### +# # ByteArray conversions + +# function PyObject(a::DenseVector{UInt8}) +# if stride(a,1) != 1 +# try +# return NpyArray(a, true) +# catch +# return array2py(a) # fallback to non-NumPy version +# end +# end +# PyObject(@pycheckn ccall((@pysym :PyByteArray_FromStringAndSize), +# PyPtr, (Ptr{UInt8}, Int), a, length(a))) +# end + + +# ispybytearray(po::PyObject) = +# pyisinstance(po, @pyglobalobj :PyByteArray_Type) + +# function convert(::Type{Vector{UInt8}}, po::PyObject) +# b = PyBuffer(po) +# iscontiguous(b) || error("a contiguous buffer is required") +# return copy(unsafe_wrap(Array, Ptr{UInt8}(pointer(b)), sizeof(b))) +# end + +# # TODO: support zero-copy PyByteArray <: AbstractVector{UInt8} object + +# ######################################################################### +# # Pointer conversions, using ctypes or PyCapsule + +# PyObject(p::Ptr) = pycall(c_void_p_Type, PyObject, UInt(p)) + +# function convert(::Type{Ptr{Cvoid}}, po::PyObject) +# if pyisinstance(po, c_void_p_Type) +# v = po."value" +# # ctypes stores the NULL pointer specially, grrr +# pynothing_query(v) == Nothing ? C_NULL : +# convert(Ptr{Cvoid}, convert(UInt, po."value")) +# elseif pyisinstance(po, @pyglobalobj(:PyCapsule_Type)) +# @pycheck ccall((@pysym :PyCapsule_GetPointer), +# Ptr{Cvoid}, (PyPtr,Ptr{UInt8}), +# po, ccall((@pysym :PyCapsule_GetName), +# Ptr{UInt8}, (PyPtr,), po)) +# else +# convert(Ptr{Cvoid}, convert(UInt, po)) +# end +# end + +# pyptr_query(po::PyObject) = pyisinstance(po, c_void_p_Type) || pyisinstance(po, @pyglobalobj(:PyCapsule_Type)) ? Ptr{Cvoid} : Union{} + +# ######################################################################### +# # Function conversion (see callback.jl for conversion the other way) +# # (rarely needed given call overloading in Julia 0.4) + +# convert(::Type{Function}, po::PyObject) = +# function fn(args...; kwargs...) +# pycall(po, PyAny, args...; kwargs...) +# end + +# ######################################################################### +# # Tuple conversion. Julia Pairs are treated as Python tuples. + +# function PyObject(t::Union{Tuple,Pair}) +# len = lastindex(t) # lastindex, not length, because of julia#14924 +# o = PyObject(@pycheckn ccall((@pysym :PyTuple_New), PyPtr, (Int,), len)) +# for i = 1:len +# oi = PyObject(t[i]) +# @pycheckz ccall((@pysym :PyTuple_SetItem), Cint, (PyPtr,Int,PyPtr), +# o, i-1, oi) +# pyincref(oi) # PyTuple_SetItem steals the reference +# end +# return o +# end + +# # somewhat annoying to get the length and types in a tuple type +# # ... would be better not to have to use undocumented internals! +# istuplen(T,isva,n) = isva ? n ≥ length(T.parameters)-1 : n == length(T.parameters) +# function tuptype(T::DataType,isva,i) +# if isva && i ≥ length(T.parameters) +# return Base.unwrapva(T.parameters[end]) +# else +# return T.parameters[i] +# end +# end +# tuptype(T::UnionAll,isva,i) = tuptype(T.body,isva,i) +# isvatuple(T::UnionAll) = isvatuple(T.body) +# isvatuple(T::DataType) = !isempty(T.parameters) && Base.isvarargtype(T.parameters[end]) + +# function convert(tt::Type{T}, o::PyObject) where T<:Tuple +# isva = isvatuple(T) +# len = @pycheckz ccall((@pysym :PySequence_Size), Int, (PyPtr,), o) +# if !istuplen(tt, isva, len) +# throw(BoundsError()) +# end +# ntuple((i -> +# convert(tuptype(T, isva, i), +# PyObject(ccall((@pysym :PySequence_GetItem), PyPtr, +# (PyPtr, Int), o, i-1)))), +# len) +# end + +# function convert(::Type{Pair{K,V}}, o::PyObject) where {K,V} +# k, v = convert(Tuple{K,V}, o) +# return Pair(k, v) +# end + +# ######################################################################### +# # Lists and 1d arrays. + +# if VERSION < v"1.1.0-DEV.392" # #29440 +# cirange(I,J) = CartesianIndices(map((i,j) -> i:j, Tuple(I), Tuple(J))) +# else +# cirange(I,J) = I:J +# end + +# # recursive conversion of A to a list of list of lists... starting +# # with dimension dim and Cartesian index i in A. +# function array2py(A::AbstractArray{<:Any, N}, dim::Integer, i::CartesianIndex{N}) where {N} +# if dim > N # base case +# return PyObject(A[i]) +# else # recursively store multidimensional array as list of lists +# ilast = CartesianIndex(ntuple(j -> j == dim ? lastindex(A, dim) : i[j], Val{N}())) +# o = PyObject(@pycheckn ccall((@pysym :PyList_New), PyPtr, (Int,), size(A, dim))) +# for icur in cirange(i,ilast) +# oi = array2py(A, dim+1, icur) +# @pycheckz ccall((@pysym :PyList_SetItem), Cint, (PyPtr,Int,PyPtr), +# o, icur[dim]-i[dim], oi) +# pyincref(oi) # PyList_SetItem steals the reference +# end +# return o +# end +# end + +# array2py(A::AbstractArray) = array2py(A, 1, first(CartesianIndices(A))) + +# #=PyObject(A::AbstractArray) = +# ndims(A) <= 1 || hasmethod(stride, Tuple{typeof(A),Int}) ? array2py(A) : +# pyjlwrap_new(A)=# + +# function py2array(T, A::Array{TA,N}, o::PyObject, +# dim::Integer, i::Integer) where {TA,N} +# if dim > N +# A[i] = convert(T, o) +# return A +# elseif dim == N +# len = @pycheckz ccall((@pysym :PySequence_Size), Int, (PyPtr,), o) +# if len != size(A, dim) +# error("dimension mismatch in py2array") +# end +# s = stride(A, dim) +# for j = 0:len-1 +# A[i+j*s] = convert(T, PyObject(ccall((@pysym :PySequence_GetItem), +# PyPtr, (PyPtr, Int), o, j))) +# end +# return A +# else # dim < N: recursively extract list of lists into A +# len = @pycheckz ccall((@pysym :PySequence_Size), Int, (PyPtr,), o) +# if len != size(A, dim) +# error("dimension mismatch in py2array") +# end +# s = stride(A, dim) +# for j = 0:len-1 +# py2array(T, A, PyObject(ccall((@pysym :PySequence_GetItem), +# PyPtr, (PyPtr, Int), o, j)), +# dim+1, i+j*s) +# end +# return A +# end +# end + +# # figure out if we can treat o as a multidimensional array, and return +# # the dimensions +# function pyarray_dims(o::PyObject, forcelist=true) +# if !(forcelist || pyisinstance(o, @pyglobalobj :PyList_Type)) +# return () # too many non-List types can pretend to be sequences +# end +# len = ccall((@pysym :PySequence_Size), Int, (PyPtr,), o) +# len < 0 && error("not a PySequence object") +# if len == 0 +# return (0,) +# end +# dims0 = pyarray_dims(PyObject(ccall((@pysym :PySequence_GetItem), +# PyPtr, (PyPtr, Int), o, 0)), +# false) +# if isempty(dims0) # not a nested sequence +# return (len,) +# end +# for j = 1:len-1 +# dims = pyarray_dims(PyObject(ccall((@pysym :PySequence_GetItem), +# PyPtr, (PyPtr, Int), o, j)), +# false) +# if dims != dims0 +# # elements don't have equal lengths, cannot +# # treat as multidimensional array +# return (len,) +# end +# end +# return tuple(len, dims0...) +# end + +# function py2array(T, o::PyObject) +# b = PyBuffer() +# if isbuftype!(o, b) +# dims = size(b) +# else +# dims = pyarray_dims(o) +# end +# pydecref(b) # safe for immediate release +# A = Array{pyany_toany(T)}(undef, dims) +# py2array(T, A, o, 1, 1) # fixme: faster conversion for supported buffer types? +# end + +# function py2vector(T, o::PyObject) +# len = ccall((@pysym :PySequence_Size), Int, (PyPtr,), o) +# if len < 0 || # not a sequence +# len+1 < 0 # object pretending to be a sequence of infinite length +# pyerr_clear() +# throw(ArgumentError("expected Python sequence")) +# end +# py2array(T, Array{pyany_toany(T)}(undef, len), o, 1, 1) +# end +# convert(::Type{Vector{T}}, o::PyObject) where T = py2vector(T, o) + +# convert(::Type{Array}, o::PyObject) = map(identity, py2array(PyAny, o)) +# convert(::Type{Array{T}}, o::PyObject) where {T} = py2array(T, o) + +# PyObject(a::BitArray) = PyObject(Array(a)) + +# ######################################################################### +# # Dictionary conversions (copies) + +# function PyObject(d::AbstractDict) +# o = PyObject(@pycheckn ccall((@pysym :PyDict_New), PyPtr, ())) +# for k in keys(d) +# @pycheckz ccall((@pysym :PyDict_SetItem), Cint, (PyPtr,PyPtr,PyPtr), +# o, PyObject(k), PyObject(d[k])) +# end +# return o +# end + +# function convert(::Type{Dict{K,V}}, o::PyObject) where {K,V} +# copy(PyDict{K,V}(o)) +# end + +# ######################################################################### +# # AbstractRange: integer ranges are converted to xrange, +# # while other ranges (<: AbstractVector) are converted to lists + +# xrange(start, stop, step) = pycall(pyxrange[], PyObject, +# start, stop, step) + +# function PyObject(r::AbstractRange{T}) where T<:Integer +# s = step(r) +# f = first(r) +# l = last(r) + s +# if max(f,l) > typemax(Clong) || min(f,l) < typemin(Clong) +# # in Python 2.x, xrange is limited to Clong +# PyObject(T[r...]) +# else +# xrange(f, l, s) +# end +# end + +# function convert(::Type{T}, o::PyObject) where T<:AbstractRange +# v = PyVector(o) +# len = length(v) +# if len == 0 +# return 1:0 # no way to get more info from an xrange +# elseif len == 1 +# start = v[1] +# return start:start +# else +# start = v[1] +# stop = v[len] +# step = v[2] - start +# return step == 1 ? (start:stop) : (start:step:stop) +# end +# end + +# ######################################################################### +# # BigFloat and Complex{BigFloat}: convert to/from Python mpmath types + +# # load mpmath module & initialize. Currently, this is done +# # the first time a BigFloat is converted to Python. Alternatively, +# # we could do it when PyCall is initialized (if mpmath is available), +# # at the cost of slowing down initialization in the common case where +# # BigFloat conversion is not needed. +# const mpprec = [0] +# const mpmath = PyNULL() +# const mpf = PyNULL() +# const mpc = PyNULL() +# function mpmath_init() +# if ispynull(mpmath) +# copy!(mpmath, pyimport("mpmath")) +# copy!(mpf, mpmath."mpf") +# copy!(mpc, mpmath."mpc") +# end +# curprec = precision(BigFloat) +# if mpprec[1] != curprec +# mpprec[1] = curprec +# mpmath."mp"."prec" = mpprec[1] +# end +# end + +# # TODO: When mpmath uses MPFR internally, can we avoid the string conversions? +# # Using strings will work regardless of the mpmath backend, but is annoying +# # both from a performance perspective and because it is a lossy conversion +# # (since strings use a decimal representation, while MPFR is binary). + +# function PyObject(x::BigFloat) +# mpmath_init() +# pycall(mpf, PyObject, string(x)) +# end + +# function PyObject(x::Complex{BigFloat}) +# mpmath_init() +# pycall(mpc, PyObject, string(real(x)), string(imag(x))) +# end + +# convert(::Type{BigFloat}, o::PyObject) = parse(BigFloat, pystr(o)) + +# function convert(::Type{Complex{BigFloat}}, o::PyObject) +# try +# Complex{BigFloat}(convert(BigFloat, o."real"), +# convert(BigFloat, o."imag")) +# catch +# convert(Complex{BigFloat}, convert(Complex{Float64}, o)) +# end +# end + +# pymp_query(o::PyObject) = pyisinstance(o, mpf) ? BigFloat : pyisinstance(o, mpc) ? Complex{BigFloat} : Union{} + +# ######################################################################### +# # (Int64), Int128 and BigInt conversion to Python "long" integers + +# const LongInt = @static (Sys.WORD_SIZE==32) ? Union{Int64,UInt64,Int128,UInt128,BigInt} : Union{Int128,UInt128,BigInt} + +# function PyObject(i::LongInt) +# PyObject(@pycheckn ccall((@pysym :PyLong_FromString), PyPtr, +# (Ptr{UInt8}, Ptr{Cvoid}, Cint), +# String(string(i)), C_NULL, 10)) +# end + +# convert(::Type{BigInt}, o::PyObject) = parse(BigInt, pystr(o)) ######################################################################### # Dates (Calendar time) -include("pydates.jl") #init_datetime() = nothing #pydate_query(o) = Union{} -######################################################################### -# Inferring Julia types at runtime from Python objects: -# -# [Note that we sometimes use the PyFoo_Check API and sometimes we use -# PyObject_IsInstance(o, PyFoo_Type), since sometimes the former API -# is a macro (hence inaccessible in Julia).] - -# A type-query function f(o::PyObject) returns the Julia type -# for use with the convert function, or Union{} if there isn't one. - -@static if pyversion < v"3" - pyint_query(o::PyObject) = pyisinstance(o, @pyglobalobj :PyInt_Type) ? - (pyisinstance(o, @pyglobalobj :PyBool_Type) ? Bool : Int) : - pyisinstance(o, @pyglobalobj :PyLong_Type) ? BigInt : - pyisinstance(o, npy_integer) ? Int : Union{} -else - pyint_query(o::PyObject) = pyisinstance(o, @pyglobalobj :PyLong_Type) ? - (pyisinstance(o, @pyglobalobj :PyBool_Type) ? Bool : Integer) : - pyisinstance(o, npy_integer) ? Integer : Union{} -end - -pyfloat_query(o::PyObject) = pyisinstance(o, @pyglobalobj :PyFloat_Type) || pyisinstance(o, npy_floating) ? Float64 : Union{} - -pycomplex_query(o::PyObject) = - pyisinstance(o, @pyglobalobj :PyComplex_Type) || pyisinstance(o, npy_complexfloating) ? ComplexF64 : Union{} - -pystring_query(o::PyObject) = pyisinstance(o, @pyglobalobj PyString_Type) ? AbstractString : pyisinstance(o, @pyglobalobj :PyUnicode_Type) ? String : Union{} - -# Given call overloading, all PyObjects are callable already, so -# we never automatically convert to Function. -pyfunction_query(o::PyObject) = Union{} - -pynothing_query(o::PyObject) = o ≛ pynothing[] ? Nothing : Union{} - -# We refrain from converting all objects that support the mapping protocol (PyMapping_Check) -# to avoid converting types like Pandas `DataFrame` that are only lossily -# representable as a Julia dictionary (issue #376). -pydict_query(o::PyObject) = pyisinstance(o, @pyglobalobj :PyDict_Type) ? Dict{PyAny,PyAny} : Union{} - -typetuple(Ts) = Tuple{Ts...} - -function pysequence_query(o::PyObject) - # pyquery(:PySequence_Check, o) always succeeds according to the docs, - # but it seems we need to be careful; I've noticed that things like - # scipy define "fake" sequence types with intmax lengths and other - # problems - if pyisinstance(o, @pyglobalobj :PyTuple_Type) - len = @pycheckz ccall((@pysym :PySequence_Size), Int, (PyPtr,), o) - return typetuple(pytype_query(PyObject(ccall((@pysym :PySequence_GetItem), PyPtr, (PyPtr,Int), o,i-1)), PyAny) for i = 1:len) - elseif pyisinstance(o, pyxrange[]) - return AbstractRange - elseif ispybytearray(o) - return Vector{UInt8} - elseif !isbuftype(o) - # only handle PyList for now - return pyisinstance(o, @pyglobalobj :PyList_Type) ? Array : Union{} - else - T, native_byteorder = array_format(o) - if T == PyPtr - T = PyObject - end - return Array{T} - end -end - -macro return_not_None(ex) - quote - T = $(esc(ex)) - if T != Union{} - return T - end - end -end +# ######################################################################### +# # Inferring Julia types at runtime from Python objects: +# # +# # [Note that we sometimes use the PyFoo_Check API and sometimes we use +# # PyObject_IsInstance(o, PyFoo_Type), since sometimes the former API +# # is a macro (hence inaccessible in Julia).] + +# # A type-query function f(o::PyObject) returns the Julia type +# # for use with the convert function, or Union{} if there isn't one. + +# @static if pyversion < v"3" +# pyint_query(o::PyObject) = pyisinstance(o, @pyglobalobj :PyInt_Type) ? +# (pyisinstance(o, @pyglobalobj :PyBool_Type) ? Bool : Int) : +# pyisinstance(o, @pyglobalobj :PyLong_Type) ? BigInt : +# pyisinstance(o, npy_integer) ? Int : Union{} +# else +# pyint_query(o::PyObject) = pyisinstance(o, @pyglobalobj :PyLong_Type) ? +# (pyisinstance(o, @pyglobalobj :PyBool_Type) ? Bool : Integer) : +# pyisinstance(o, npy_integer) ? Integer : Union{} +# end + +# pyfloat_query(o::PyObject) = pyisinstance(o, @pyglobalobj :PyFloat_Type) || pyisinstance(o, npy_floating) ? Float64 : Union{} + +# pycomplex_query(o::PyObject) = +# pyisinstance(o, @pyglobalobj :PyComplex_Type) || pyisinstance(o, npy_complexfloating) ? ComplexF64 : Union{} + +# pystring_query(o::PyObject) = pyisinstance(o, @pyglobalobj PyString_Type) ? AbstractString : pyisinstance(o, @pyglobalobj :PyUnicode_Type) ? String : Union{} + +# # Given call overloading, all PyObjects are callable already, so +# # we never automatically convert to Function. +# pyfunction_query(o::PyObject) = Union{} + +# pynothing_query(o::PyObject) = o ≛ pynothing[] ? Nothing : Union{} + +# # We refrain from converting all objects that support the mapping protocol (PyMapping_Check) +# # to avoid converting types like Pandas `DataFrame` that are only lossily +# # representable as a Julia dictionary (issue #376). +# pydict_query(o::PyObject) = pyisinstance(o, @pyglobalobj :PyDict_Type) ? Dict{PyAny,PyAny} : Union{} + +# typetuple(Ts) = Tuple{Ts...} + +# function pysequence_query(o::PyObject) +# # pyquery(:PySequence_Check, o) always succeeds according to the docs, +# # but it seems we need to be careful; I've noticed that things like +# # scipy define "fake" sequence types with intmax lengths and other +# # problems +# if pyisinstance(o, @pyglobalobj :PyTuple_Type) +# len = @pycheckz ccall((@pysym :PySequence_Size), Int, (PyPtr,), o) +# return typetuple(pytype_query(PyObject(ccall((@pysym :PySequence_GetItem), PyPtr, (PyPtr,Int), o,i-1)), PyAny) for i = 1:len) +# elseif pyisinstance(o, pyxrange[]) +# return AbstractRange +# # elseif ispybytearray(o) +# # return Vector{UInt8} +# # elseif !isbuftype(o) +# # # only handle PyList for now +# # return pyisinstance(o, @pyglobalobj :PyList_Type) ? Array : Union{} +# # else +# # T, native_byteorder = array_format(o) +# # if T == PyPtr +# # T = PyObject +# # end +# # return Array{T} +# end +# end + +# macro return_not_None(ex) +# quote +# T = $(esc(ex)) +# if T != Union{} +# return T +# end +# end +# end const pytype_queries = Tuple{PyObject,Type}[] -""" - pytype_mapping(pytype, jltype) - -Given a Python type object `pytype`, tell PyCall to convert it to -`jltype` in `PyAny(object)` conversions. -""" -function pytype_mapping(py::PyObject, jl::Type) - for (i,(p,j)) in enumerate(pytype_queries) - if p == py - pytype_queries[i] = (py,jl) - return pytype_queries - end - end - push!(pytype_queries, (py,jl)) -end -""" - pytype_query(o::PyObject, default=PyObject) - -Given a Python object `o`, return the corresponding -native Julia type (defaulting to `default`) that we convert -`o` to in `PyAny(o)` conversions. -""" -function pytype_query(o::PyObject, default::TypeTuple=PyObject) - # TODO: Use some kind of hashtable (e.g. based on PyObject_Type(o)). - # (A bit tricky to correctly handle Tuple and other containers.) - @return_not_None pyint_query(o) - pyisinstance(o, npy_bool) && return Bool - @return_not_None pyfloat_query(o) - @return_not_None pycomplex_query(o) - @return_not_None pystring_query(o) - @return_not_None pyfunction_query(o) - @return_not_None pydate_query(o) - @return_not_None pydict_query(o) - @return_not_None pyptr_query(o) - @return_not_None pysequence_query(o) - @return_not_None pynothing_query(o) - @return_not_None pymp_query(o) - for (py,jl) in pytype_queries - if pyisinstance(o, py) - return jl - end - end - return default -end - -function convert(::Type{PyAny}, o::PyObject) - if ispynull(o) - return o - end - try - T = pytype_query(o) - if T == PyObject && is_pyjlwrap(o) - return unsafe_pyjlwrap_to_objref(o) - end - convert(T, o) - catch - pyerr_clear() # just in case - o - end -end +# """ +# pytype_mapping(pytype, jltype) + +# Given a Python type object `pytype`, tell PyCall to convert it to +# `jltype` in `PyAny(object)` conversions. +# """ +# function pytype_mapping(py::PyObject, jl::Type) +# for (i,(p,j)) in enumerate(pytype_queries) +# if p == py +# pytype_queries[i] = (py,jl) +# return pytype_queries +# end +# end +# push!(pytype_queries, (py,jl)) +# end +# """ +# pytype_query(o::PyObject, default=PyObject) + +# Given a Python object `o`, return the corresponding +# native Julia type (defaulting to `default`) that we convert +# `o` to in `PyAny(o)` conversions. +# """ +# function pytype_query(o::PyObject, default::TypeTuple=PyObject) +# # TODO: Use some kind of hashtable (e.g. based on PyObject_Type(o)). +# # (A bit tricky to correctly handle Tuple and other containers.) +# @return_not_None pyint_query(o) +# pyisinstance(o, npy_bool) && return Bool +# @return_not_None pyfloat_query(o) +# @return_not_None pycomplex_query(o) +# @return_not_None pystring_query(o) +# # @return_not_None pyfunction_query(o) +# @return_not_None pydate_query(o) +# # @return_not_None pydict_query(o) +# # @return_not_None pyptr_query(o) +# @return_not_None pysequence_query(o) +# @return_not_None pynothing_query(o) +# @return_not_None pymp_query(o) +# for (py,jl) in pytype_queries +# if pyisinstance(o, py) +# return jl +# end +# end +# return default +# end + +# convert(U::Union, o::PyObject) = +# try +# convert(U.a, o) +# catch +# try +# convert(U.b, o) +# catch +# throw(MethodError(convert, (U,o))) +# end +# end diff --git a/src/exception.jl b/src/exception.jl index 7ced7a09..a7e80efa 100644 --- a/src/exception.jl +++ b/src/exception.jl @@ -47,14 +47,36 @@ function show(io::IO, e::PyError) end end +function Base.showerror(io::IO, e::PyError) + print(io, "PyCall") + if ispynull(e.T) + println(io, " (null)") + else + print(io, ": ", e.T.__name__) + if !ispynull(e.val) + print(io, ": ", pystr(e.val)) + end + end +end + ######################################################################### # Conversion of Python exceptions into Julia exceptions # whether a Python exception has occurred -pyerr_occurred() = ccall((@pysym :PyErr_Occurred), PyPtr, ()) != C_NULL +""" + pyerr_occurred([t]) + +True if a Python exception has occurred and (optionally) is of type `t`. +""" +pyerr_occurred() = CPyErr_Occurred() != C_NULL + +function pyerr_occurred(t) + e = CPyErr_Occurred() + e == C_NULL ? false : pyissubclass(e, t) +end # call to discard Python exceptions -pyerr_clear() = ccall((@pysym :PyErr_Clear), Cvoid, ()) +pyerr_clear() = CPyErr_Clear() function pyerr_check(msg::AbstractString, val::Any) pyerr_occurred() && throw(PyError(msg)) @@ -108,6 +130,7 @@ end # Mapping of Julia Exception types to Python exceptions const pyexc = Dict{DataType, PyPtr}() +const pysymexc = Dict{Symbol, PyPtr}() mutable struct PyIOError <: Exception end function pyexc_initialize() @@ -131,6 +154,23 @@ function pyexc_initialize() pyexc[UndefRefError] = @pyglobalobjptr :PyExc_RuntimeError pyexc[InterruptException] = @pyglobalobjptr :PyExc_KeyboardInterrupt pyexc[PyIOError] = @pyglobalobjptr :PyExc_IOError + + pysymexc[:RuntimeError] = @pyglobalobjptr :PyExc_RuntimeError + pysymexc[:SystemError] = @pyglobalobjptr :PyExc_SystemError + pysymexc[:TypeError] = @pyglobalobjptr :PyExc_TypeError + pysymexc[:SyntaxError] = @pyglobalobjptr :PyExc_SyntaxError + pysymexc[:ValueError] = @pyglobalobjptr :PyExc_ValueError + pysymexc[:KeyError] = @pyglobalobjptr :PyExc_KeyError + pysymexc[:ImportError] = @pyglobalobjptr :PyExc_ImportError + pysymexc[:EOFError] = @pyglobalobjptr :PyExc_EOFError + pysymexc[:IndexError] = @pyglobalobjptr :PyExc_IndexError + pysymexc[:ZeroDivisionError] = @pyglobalobjptr :PyExc_ZeroDivisionError + pysymexc[:OverflowError] = @pyglobalobjptr :PyExc_OverflowError + pysymexc[:ArithmeticError] = @pyglobalobjptr :PyExc_ArithmeticError + pysymexc[:MemoryError] = @pyglobalobjptr :PyExc_MemoryError + pysymexc[:KeyboardInterrupt] = @pyglobalobjptr :PyExc_KeyboardInterrupt + pysymexc[:IOError] = @pyglobalobjptr :PyExc_IOError + pysymexc[:AttributeError] = @pyglobalobjptr :PyExc_AttributeError end _showerror_string(io::IO, e, ::Nothing) = showerror(io, e) @@ -178,11 +218,22 @@ function showerror_string(e::T, bt = nothing) where {T} end end -function pyraise(e, bt = nothing) +function pyraise(e::Exception, bt = nothing) eT = typeof(e) pyeT = haskey(pyexc::Dict, eT) ? pyexc[eT] : pyexc[Exception] - ccall((@pysym :PyErr_SetString), Cvoid, (PyPtr, Cstring), - pyeT, string("Julia exception: ", showerror_string(e, bt))) + pyraise(pyeT, e, bt) +end + +function pyraise(T::PyPtr, e::Exception, bt=nothing) + pyraise(T, string("Julia exception: ", showerror_string(e, bt))) +end + +function pyraise(T::Symbol, msg::AbstractString) + pyraise(pysymexc[T], msg) +end + +function pyraise(T::PyPtr, msg::AbstractString) + ccall(@pysym(:PyErr_SetString), Cvoid, (PyPtr, Cstring), T, msg) end # Second argument allows for backtraces passed to `pyraise` to be ignored. diff --git a/src/gc.jl b/src/gc.jl index e000cc37..ca43e450 100644 --- a/src/gc.jl +++ b/src/gc.jl @@ -23,7 +23,7 @@ const weakref_callback_obj = PyNULL() # weakref_callback Python method # Python expects the PyMethodDef structure to be a constant, so # we put it in a global to prevent gc. -const weakref_callback_meth = Ref{PyMethodDef}() +const weakref_callback_meth = Ref{CPyMethodDef}() # "embed" a reference to jo in po, using the weak-reference mechanism function pyembed(po::PyObject, jo::Any) @@ -35,7 +35,7 @@ function pyembed(po::PyObject, jo::Any) weakref_callback_meth[] = PyMethodDef("weakref_callback", cf, METH_O) copy!(weakref_callback_obj, PyObject(@pycheckn ccall((@pysym :PyCFunction_NewEx), PyPtr, - (Ref{PyMethodDef}, Ptr{Cvoid}, Ptr{Cvoid}), + (Ref{CPyMethodDef}, Ptr{Cvoid}, Ptr{Cvoid}), weakref_callback_meth, C_NULL, C_NULL))) end wo = @pycheckn ccall((@pysym :PyWeakref_NewRef), PyPtr, (PyPtr,PyPtr), diff --git a/src/io.jl b/src/io.jl index 145b2fea..151c3504 100644 --- a/src/io.jl +++ b/src/io.jl @@ -65,7 +65,7 @@ end ########################################################################## -pyio_jl(self::PyObject) = unsafe_pyjlwrap_to_objref(self."io")::IO +pyio_jl(self::PyObject) = unsafe_pyjlwrap_load_value(self."io")::IO const PyIO = PyNULL() @@ -120,11 +120,11 @@ end ########################################################################## -function PyObject(io::IO) - pyio_initialize() - # pyjlwrap_new is necessary to avoid PyIO(io) calling PyObject(::IO) - PyIO(pyjlwrap_new(io)) -end +# function PyObject(io::IO) +# pyio_initialize() +# # pyjlwrap_new is necessary to avoid PyIO(io) calling PyObject(::IO) +# PyIO(pyjlwrap_new(io)) +# end """ PyTextIO(io::IO) diff --git a/src/jlwrap-notes.md b/src/jlwrap-notes.md new file mode 100644 index 00000000..4d702970 --- /dev/null +++ b/src/jlwrap-notes.md @@ -0,0 +1,25 @@ +# jlwrap notes + +* All the structs should be immutable really, with better constructors (i.e. `PyType_Ready` should be called before the inner constructor finishes, after which the type is immutable) +* `jlwrap_type(::Type{T})` looks up `T` in a `IdDict{Type,PyObject}`, if it's there returns it, otherwise returns `jlwrap_new_type(T)` +* `jlwrap_new_type(::Type{T})` returns a (new) `PyObject` which is a `type` (or maybe some subtype of `type` specifically for julia types? call it `juliatype`?) representing the julia type `T`, it has one supertype (either its julia supertype, or `object` if `T` is `Any`) --- so the julia type system is embedded into the python one, with the root `Any` being a direct subtype of `object`. +* such a type object should have methods to interact with the julia type system, such as getting its supertypes, subtypes, parameters, or specializing parameters (note that each parameterization of a type is a new type in the python system) +* `jlwrap_object(x)` returns `jlwrap_new_object(x)` +* `jlwrap_new_object(x::T)` returns a (new) `PyObject` wrapping `x` whose type is `jlwrap_type(T)` +* `jlwrap(x)` returns `jlwrap_type` or `jlwrap_object` depending on whether `x` is a type or not +* have a test to determine if a python object is a wrap of some julia thing, namely it was created by `jlwrap_type` or `jlwrap_new_type` +* if the version of python supports abstract base classes (ABCs), then: + * have an ABC for `jlwrap`, which `jlwrap_type(Any)` and `juliatype` are both subclassses of? + * apply other ABCs to objects which support certain operations + + + +We should have a type that represents the translation of semantics between julia and python. For example, it should have a `getattr` field, which is a namedtuple mapping symbols to functions, then we define something like: + +``` +jlwrap_getattr(o, a) = _jlwrap_getattr(unwrap(pytypeof(o)), unwrap(o), Symbol(string(a))) + +_jlwrap_getattr(t, o, a) = haskey(t.getattr, a) ? t.getattr[a](o) : _jlwrap_getattr_dflt(o, a) + +_jlwrap_getattr_dflt(o, a) = startswith(string(a), "__") ? (do the right python thing) : getproperty(o, a) +``` \ No newline at end of file diff --git a/src/jlwrap.jl b/src/jlwrap.jl new file mode 100644 index 00000000..eade07bb --- /dev/null +++ b/src/jlwrap.jl @@ -0,0 +1,1203 @@ +################################################################ +# Wrap a Python type around a Julia Any object + + + +########################################################## +# permanent cacheing + +const GLOBAL_CACHE = Base.IdDict() + +function cached(x) + get!(GLOBAL_CACHE, x, x) :: typeof(x) +end + +cached_string_pointer(x::String) = unsafe_convert(Ptr{UInt8}, cached(x)) +cached_string_pointer(x::AbstractString) = cached_string_pointer(String(x)) +cached_string_pointer(x::Ptr) = convert(Ptr{UInt8}, x) + +cached_string_pointer_or_NULL(x::AbstractString) = + isempty(x) ? Ptr{UInt8}(C_NULL) : cached_string_pointer(x) +cached_string_pointer_or_NULL(x::Ptr) = cached_string_pointer(x) + +cached_ref(x) = cached(Ref(x)) +cached_ref(x::Ref) = cached(x) + +cached_ref_pointer(x) = cached_ref_pointer(Ref(x)) +cached_ref_pointer(x::Ref) = unsafe_convert(Ptr{eltype(x)}, cached(x)) + + + + + +########################################################## +# convenience functions for making C structures + +CPyMethodDef(; name=C_NULL, meth=C_NULL, flags=0, doc=C_NULL) = + CPyMethodDef(cached_string_pointer(name), convert(Ptr{Cvoid}, meth), convert(Cint, flags), cached_string_pointer_or_NULL(doc)) + +CPyGetSetDef(; name=C_NULL, get=C_NULL, set=C_NULL, doc=C_NULL, closure=C_NULL) = + CPyGetSetDef(cached_string_pointer(name), convert(Ptr{Cvoid}, get), convert(Ptr{Cvoid}, set), cached_string_pointer_or_NULL(doc), convert(Ptr{Cvoid}, closure)) + +CPyMemberDef(; name=C_NULL, typ=0, offset=0, flags=0, doc=C_NULL) = + CPyMemberDef(cached_string_pointer(name), convert(Cint, typ), convert(Int, offset), convert(Cint, flags), cached_string_pointer_or_NULL(doc)) + +@eval CPyNumberMethods(; $([Expr(:kw, n, C_NULL) for n in fieldnames(CPyNumberMethods)]...)) = CPyNumberMethods($([:(convert(Ptr{Cvoid}, $n)) for n in fieldnames(CPyNumberMethods)]...)) + +@eval CPySequenceMethods(; $([Expr(:kw, n, C_NULL) for n in fieldnames(CPySequenceMethods)]...)) = CPySequenceMethods($([:(convert(Ptr{Cvoid}, $n)) for n in fieldnames(CPySequenceMethods)]...)) + +@eval CPyMappingMethods(; $([Expr(:kw, n, C_NULL) for n in fieldnames(CPyMappingMethods)]...)) = CPyMappingMethods($([:(convert(Ptr{Cvoid}, $n)) for n in fieldnames(CPyMappingMethods)]...)) + +@eval function CPyTypeObject(; initialize=true, $([Expr(:kw, n, t<:Ptr ? C_NULL : 0) for (n,t) in zip(fieldnames(CPyTypeObject), fieldtypes(CPyTypeObject))]...)) + # convert inputs + if tp_name isa AbstractString + tp_name = cached_string_pointer(tp_name) + end + if tp_as_number isa CPyNumberMethods + tp_as_number = cached_ref_pointer(tp_as_number) + end + if tp_as_sequence isa CPySequenceMethods + tp_as_sequence = cached_ref_pointer(tp_as_sequence) + end + if tp_as_mapping isa CPyMappingMethods + tp_as_mapping = cached_ref_pointer(tp_as_mapping) + end + if tp_members isa AbstractVector{CPyMemberDef} + tp_members = pointer(cached([tp_members; CPyMemberDef_NULL])) + end + if tp_methods isa AbstractVector{CPyMethodDef} + tp_methods = pointer(cached([tp_methods; CPyMethodDef_NULL])) + end + if tp_getset isa AbstractVector{CPyGetSetDef} + tp_getset = pointer(cached([tp_getset; CPyGetSetDef_NULL])) + end + if tp_base isa Ref{CPyTypeObject} + tp_base = cached_ref_pointer(tp_base) + end + # make the type + CPyTypeObject($([:(convert($t, $n)) for (n,t) in zip(fieldnames(CPyTypeObject), fieldtypes(CPyTypeObject))]...)) +end + +macro cpymethod(name) + :(@cfunction($name, PyPtr, (PyPtr, PyPtr))) +end + +macro cpygetfunc(name) + :(@cfunction($name, PyPtr, (PyPtr, Ptr{Cvoid}))) +end + + + + + + + + + + +########################################################## +# The jlwrap object + +struct CPyJlWrapObject + # PyObject_HEAD (for non-Py_TRACE_REFS build): + ob_refcnt::Int + ob_type::PyPtr # actually Ptr{PyTypeObject} + + ob_weakrefs::PyPtr + jl_value::Any +end + +field_pointer(ptr::Ptr{T}, i) where {T} = + convert(Ptr{fieldtype(T, i)}, convert(Ptr{Cvoid}, ptr) + fieldoffset(T, i)) + +unsafe_store_field!(ptr::Ptr, val, i) = + unsafe_store!(field_pointer(ptr, i), val) + +unsafe_load_field(ptr::Ptr, i) = + unsafe_load(field_pointer(ptr, i)) + +unsafe_pytype(o::PyPtr) = + convert(Ptr{CPyTypeObject}, unsafe_load_field(o, 2)) + +unsafe_pyjlwrap_ptr(o::Union{PyPtr, PyObject}) = + convert(Ptr{CPyJlWrapObject}, PyPtr(o)) + +unsafe_pyjlwrap_value_ptr(o::Union{PyPtr, PyObject}) = + Ptr{Ptr{Cvoid}}(field_pointer(unsafe_pyjlwrap_ptr(o), 4)) + +unsafe_pyjlwrap_load_value(o::Union{PyPtr, PyObject}) = + GC.@preserve o unsafe_pointer_to_objref(unsafe_load(unsafe_pyjlwrap_value_ptr(o))) + +unsafe_pyjlwrap_store_value!(o::Union{PyPtr, PyObject}, p::Ptr) = + unsafe_store!(unsafe_pyjlwrap_value_ptr(o), p) + +function CPyJlWrap_New(T::Ref{CPyTypeObject}, value) :: PyPtr + # make the new object + o = CPyObject_New(T) + o == C_NULL && (return C_NULL) + # make a pointer to the value + if isimmutable(value) + # It is undefined to call `pointer_from_objref` on immutable objects. + # The compiler is free to return basically anything since the boxing is not + # significant at all. + # Below is a well defined way to get a pointer (`ptr`) and an object that defines + # the lifetime of the pointer `ref`. + ref = Ref{Any}(value) + pycall_gc[o] = ref + ptr = unsafe_load(Ptr{Ptr{Cvoid}}(pointer_from_objref(ref))) + else + pycall_gc[o] = value + ptr = pointer_from_objref(value) + end + # store the relevant pointers + unsafe_store_field!(unsafe_pyjlwrap_ptr(o), C_NULL, 3) + unsafe_pyjlwrap_store_value!(o, ptr) + return o +end + +is_pyjlwrap(o::Union{PyObject,PyPtr}) = + CPyJlWrap_Type[].tp_new != C_NULL && + CPyObject_IsInstance(o, CPyJlWrap_Type) == 1 + + + + + + + + +########################################################## +# members and methods + + +# destructor for jlwrap instance, assuming it was created with pyjlwrap_new +function _pyjlwrap_dealloc(o::PyPtr) + p = unsafe_pyjlwrap_ptr(o) + if unsafe_load_field(p, 3) != C_NULL + PyObject_ClearWeakRefs(o) + end + delete!(pycall_gc, o) + return nothing +end + +pyerrorval(::Type{PyPtr}) = PyPtr_NULL +pyerrorval(::Type{T}) where {T<:Integer} = zero(T) - one(T) + +macro pyjlwrapfunc(ex) + def = MacroTools.splitdef(ex) + def[:name] = Symbol(:_pyjlwrap_, def[:name]) + selfarg = def[:args][1] + (self, selftype, selfslurp, selfdefault) = MacroTools.splitarg(selfarg) + _self = Symbol(:_, self) + err = gensym() + def[:args][1] = :($_self :: PyPtr) + def[:body] = quote + $self :: $selftype = unsafe_pyjlwrap_load_value($_self) + try + $(def[:body]) + catch $err + @pyraise $err + end + $(pyerrorval(eval(def[:rtype]))) + end + r = MacroTools.combinedef(def) +end + +@pyjlwrapfunc function repr(o)::PyPtr + return CPyUnicode_From(repr(o)) +end + +@pyjlwrapfunc function str(o)::PyPtr + return CPyUnicode_From(string(o)) +end + +@pyjlwrapfunc function hash(o)::UInt + h = hash(o) + # Python hashes are not permitted to return -1!! + return h == reinterpret(UInt, -1) ? pysalt::UInt : h::UInt +end + +# 32-bit hash on 64-bit machines, needed for Python < 3.2 with Windows +const pysalt32 = 0xb592cd9b # hash("PyCall") % UInt32 +@pyjlwrapfunc function hash32(o)::UInt32 + h = ccall(:int64to32hash, UInt32, (UInt64,), hash(o)) + # Python hashes are not permitted to return -1!! + return h == reinterpret(UInt32, Int32(-1)) ? pysalt32 : h::UInt32 +end + +function _pyjlwrap_call(f_::PyPtr, args_::PyPtr, kw_::PyPtr)::PyPtr + f = unsafe_pyjlwrap_load_value(f_) + args = PyObject(args_) # don't need pyincref because of finally clause below + try + jlargs = julia_args(f, args) + + # we need to use invokelatest to get execution in newest world + if kw_ == C_NULL + ret = Base.invokelatest(f, jlargs...) + else + kw = PyDict{Symbol,PyObject}(pyincref(kw_)) + kwargs = [ (k,julia_kwarg(f,k,v)) for (k,v) in kw ] + + # 0.6 `invokelatest` doesn't support kwargs, instead + # use a closure over kwargs. see: + # https://github.com/JuliaLang/julia/pull/22646 + f_kw_closure() = f(jlargs...; kwargs...) + ret = Core._apply_latest(f_kw_closure) + end + + @pyreturn ret + catch e + @pyraise e + finally + setfield!(args, :o, PyPtr_NULL) # don't decref + end + return PyPtr_NULL +end + +@pyjlwrapfunc function length(o)::Cssize_t + return length(o) +end + +@pyjlwrapfunc function istrue(o)::Cint + return _pyistrue(x)::Bool +end + +@pyjlwrapfunc function istrue_always(o)::Cint + return return true +end + +_pyistrue(x) = + try + !iszero(x) + catch + try + !isempty(x) + catch + true + end + end +_pyistrue(::Union{Ref,Function}) = true +_pyistrue(::Union{Nothing,Missing}) = false +_pyistrue(x::Bool) = x +_pyistrue(x::Number) = !iszero(x) +_pyistrue(x::Union{AbstractArray,AbstractDict,Tuple,Pair,NamedTuple,AbstractSet,AbstractString}) = !isempty(x) +_pyistrue(x::Symbol) = x != Symbol() +_pyistrue(x::Ptr) = x != C_NULL + +@pyjlwrapfunc function istrue_number(o)::Cint + return !iszero(o) +end + +@pyjlwrapfunc function istrue_collection(o)::Cint + return !isempty(o) +end + + +@pyjlwrapfunc function int(o)::PyPtr + return CPyLong_From(o) +end + +@pyjlwrapfunc function float(o)::PyPtr + return CPyFloat_From(o) +end + +@pyjlwrapfunc function complex(o, ::PyPtr)::PyPtr + return CPyComplex_From(o) +end + +function _pyjlwrap_richcompare(_a::PyPtr, _b::PyPtr, op::Cint)::PyPtr + (is_pyjlwrap(_a) && is_pyjlwrap(_b)) || (return CPy_NotImplemented_NewRef()) + a = unsafe_pyjlwrap_load_value(_a) + b = unsafe_pyjlwrap_load_value(_b) + try + r = + op == Py_LT ? a < b : + op == Py_LE ? a ≤ b : + op == Py_EQ ? a == b : + op == Py_NE ? a != b : + op == Py_GT ? a > b : + op == Py_GE ? a ≥ b : + error("invalid op") + return CPyBool_From(r) + catch e + e isa MethodError && (return CPy_NotImplemented_NewRef()) + @pyraise e + end + return C_NULL +end + +@pyjlwrapfunc function getitem(o, i::PyPtr)::PyPtr + @pyreturn _pygetitem(o, pyincref(i)) +end + +@pyjlwrapfunc function getitem_oneup(o, i::Cssize_t)::PyPtr + @pyreturn o[i+1] +end + +@pyjlwrapfunc function getitem_oneup(o, _i::PyPtr)::PyPtr + i = convert(Union{Int,Tuple{Vararg{Int}}}, pyincref(_i)) .+ 1 + @pyreturn o[i...] +end + +@pyjlwrapfunc function setitem_oneup(o, _i::PyPtr, _x::PyPtr)::Cint + i = convert(Union{Int, Tuple{Vararg{Int}}}, pyincref(_i)) .+ 1 + o[i...] = _pyvalueatindex(o, i, _x) + return 0 +end + +@pyjlwrapfunc function getitem_namedtuple(o, _i::PyPtr)::PyPtr + i = convert(Union{Int,Symbol}, pyincref(_i)) + if i isa Int + @pyreturn o[i+1] + else + @pyreturn o[i] + end +end + +_pygetitem(o, i::PyObject) = _pygetitem(o, _pyindex(o, i)) +_pygetitem(o, i) = getindex(o, i) +_pygetitem(o, i::Tuple) = applicable(getindex, o, i...) ? getindex(o, i...) : getindex(o, i) + +function _pyindex(o, i) + T = applicable(keytype, o) ? keytype(o) : Any + T = T==Any ? PyAny : Union{T,PyAny} + convert(T, i) +end +_pyindex(o::NamedTuple, i::PyObject) = convert(Union{Symbol,Int}, i) + +@pyjlwrapfunc function setitem(o, i::PyPtr, x::PyPtr)::Cint + _pysetitem(o, pyincref(i), pyincref(x)) + return 0 +end + +function _pysetitem(o, i, x) + i = _pyindex(o, i) + x = _pyvalueatindex(o, i, x) + setindex!(o, x, i) +end + +function _pyvalueatindex(o, i, x) + T = applicable(eltype, o) ? eltype(o) : Any + T = T==Any ? PyAny : Union{T,PyAny} + convert(T, x) +end + +function _pyjlwrap_getattr(_self::PyPtr, _attr::PyPtr)::PyPtr + self = unsafe_pyjlwrap_load_value(_self) + attr = CPyUnicode_As(String, _attr) + attr===nothing && (return PyPtr_NULL) + try + # do the generic lookup in __dict__ first + r = CPyObject_GenericGetAttr(_self, _attr) + pyerr_occurred(CPyExc_AttributeError[]) || (return r) + # now do special attributes + if attr == "__doc__" + return CPyUnicode_From(string(Docs.doc(self))) + elseif startswith(attr, "__julia_field_") + a = Symbol(attr[15:end]) + if hasfield(typeof(self), a) + pyerr_clear() + @pyreturn getfield(self, a) + end + elseif startswith(attr, "__julia_property_") + a = Symbol(attr[18:end]) + if hasproperty(self, a) + pyerr_clear() + @pyreturn getproperty(self, a) + end + else + a = Symbol(attr) + if hasproperty(self, a) + pyerr_clear() + @pyreturn getproperty(self, a) + end + end + # on failure, propagate the attribute error + return r + catch e + @pyraise e + end + return PyPtr_NULL +end + +function _pyjlwrap_setattr(self_::PyPtr, attr__::PyPtr, value_::PyPtr)::Cint + value_ == C_NULL && return pyjlwrap_delattr(self_, attr__) + attr_ = PyObject(attr__) + value = pyincref(value_) + try + self = unsafe_pyjlwrap_load_value(self_) + attr = convert(String, attr_) + @show self attr + if startswith(attr, "__") + if startswith(attr, "__julia_field_") + _pysetfield(self, Symbol(attr[15:end]), value) + elseif startswith(attr, "__julia_property_") + _pysetproperty(self, Symbol(attr[18:end]), value) + else + return PyObject_GenericSetAttr(self_, attr__, value_) + end + else + _pysetproperty(self, Symbol(attr), value) + end + return 0 + catch e + @pyraise e + finally + setfield!(attr_, :o, PyPtr_NULL) + end + return -1 +end + +function _pyjlwrap_dir(_o::PyPtr, ::PyPtr)::PyPtr + # the default implementation + d = PyObject(CPyObject_GetAttrString(CPyBaseObject_Type[], "__dir__")) + ispynull(d) && return C_NULL + r = PyObject(CPyObject_CallFunction(d, _o)) + ispynull(r) && return C_NULL + # add fields and properties of the julia object + o = unsafe_pyjlwrap_load_value(_o) + try + custom = String[] + for n in fieldnames(typeof(o)) + push!(custom, "__julia_field_$n") + end + for n in propertynames(o) + push!(custom, "$n") + push!(custom, "__julia_property_$n") + end + for n in custom + s = PyObject(CPyUnicode_From(n)) + ispynull(s) && return C_NULL + z = CPyList_Append(r, s) + z == -1 && return C_NULL + end + return pystealref!(r) + catch e + @pyraise e + end + return C_NULL +end + +function _pysetproperty(o, f, x) + x = _pyvalueatproperty(o, f, x) + setproperty!(o, f, x) +end + +function _pyvalueatproperty(o, f, x) + convert(PyAny, x) +end + +function _pysetfield(o, f, x) + x = _pyvalueatfield(o, f, x) + setfield!(o, f, x) +end + +function _pyvalueatfield(o, f, x) + T = fieldtype(typeof(o), f) + T = T==Any ? PyAny : T + convert(T, x) +end + +# tp_iternext object of a jlwrap_iterator object, similar to PyIter_Next +@pyjlwrapfunc function iternext(self)::PyPtr + iter, iter_result_ref = self + iter_result = iter_result_ref[] + if iter_result !== nothing + item, state = iter_result + iter_result_ref[] = iterate(iter, state) + @pyreturn item + end +end + +# the tp_iter slot of jlwrap object: like PyObject_GetIter, it +# returns a reference to a new jlwrap_iterator object +@pyjlwrapfunc function getiter(self)::PyPtr + return CPyJlWrapIterator_From(self) +end + +@pyjlwrapfunc function getiter_keys(self)::PyPtr + return CPyJlWrapIterator_From(keys(self)) +end + +@pyjlwrapfunc function getiter_lines(self)::PyPtr + return CPyJlWrapIterator_From(eachline(self)) +end + +for (a,b) in [(:negative,:-), (:positive, :+), (:absolute, :abs), (:invert, :~)] + @eval @pyjlwrapfunc function $a(self)::PyPtr + return CPyJlWrap_From($b(self)) + end +end + +for (a,b) in [(:add, :+), (:subtract, :-), (:multiply, :*), (:remainder, :mod), (:lshift, :(<<)), (:rshift, :(>>)), (:and, :&), (:xor, :⊻), (:or, :|), (:floordivide, :fld), (:truedivide, :/)] + @eval function $(Symbol(:_pyjlwrap_, a))(a_::PyPtr, b_::PyPtr)::PyPtr + (is_pyjlwrap(a_) && is_pyjlwrap(b_)) || + (return CPy_NotImplemented_NewRef()) + try + a = unsafe_pyjlwrap_load_value(a_) + b = unsafe_pyjlwrap_load_value(b_) + return CPyJlWrap_From($b(a, b)) + catch e + @pyraise e + end + return C_NULL + end +end + +function _pyjlwrap_power(_a::PyPtr, _b::PyPtr, _c::PyPtr)::PyPtr + if _c == CPy_None[] + (is_pyjlwrap(_a) && is_pyjlwrap(_b)) || + (return CPy_NotImplemented_NewRef()) + a = unsafe_pyjlwrap_load_value(_a) + b = unsafe_pyjlwrap_load_value(_b) + try + return CPyJlWrap_From(a^b) + catch e + @pyraise e + end + return C_NULL + else + return CPy_NotImplemented_NewRef() + end +end + +@pyjlwrapfunc function get_real(self, ::Ptr{Cvoid})::PyPtr + return CPyJlWrapReal_From(real(self)) +end + +@pyjlwrapfunc function get_imag(self, ::Ptr{Cvoid})::PyPtr + return CPyJlWrapReal_From(imag(self)) +end + +@pyjlwrapfunc function conjugate(self, ::PyPtr)::PyPtr + return CPyJlWrapNumber_From(conj(self)) +end + +@pyjlwrapfunc function trunc(self, ::PyPtr)::PyPtr + return CPyJlWrapIntegral_From(trunc(Integer, self)) +end + +@pyjlwrapfunc function round(self, ::PyPtr)::PyPtr + return CPyJlWrapIntegral_From(round(Integer, self)) +end + +@pyjlwrapfunc function floor(self, ::PyPtr)::PyPtr + return CPyJlWrapIntegral_From(floor(Integer, self)) +end + +@pyjlwrapfunc function ceil(self, ::PyPtr)::PyPtr + return CPyJlWrapIntegral_From(ceil(Integer, self)) +end + +@pyjlwrapfunc function get_numerator(self, ::Ptr{Cvoid})::PyPtr + return CPyJlWrapIntegral_From(numerator(self)) +end + +@pyjlwrapfunc function get_denominator(self, ::Ptr{Cvoid})::PyPtr + return CPyJlWrapIntegral_From(denominator(self)) +end + +@pyjlwrapfunc function io_close(self, ::PyPtr)::PyPtr + close(self) + return CPy_None_NewRef() +end + +@pyjlwrapfunc function io_fileno(self, ::PyPtr)::PyPtr + return CPyLong_From(fd(self)) +end + +@pyjlwrapfunc function io_get_closed(self, ::Ptr{Cvoid})::PyPtr + return CPyBool_From(!isopen(self)) +end + +@pyjlwrapfunc function io_flush(self, ::PyPtr)::PyPtr + flush(self) + return CPy_None_NewRef() +end + +@pyjlwrapfunc function io_isatty(self, ::PyPtr)::PyPtr + return CPyBool_From(self isa Base.TTY) +end + +@pyjlwrapfunc function io_readable(self, ::PyPtr)::PyPtr + return CPyBool_From(isreadable(self)) +end + +@pyjlwrapfunc function io_writable(self, ::PyPtr)::PyPtr + return CPyBool_From(iswritable(self)) +end + +@pyjlwrapfunc function io_truncate(self, args::PyPtr)::PyPtr + r = @CPyArg_ParseTuple(args, sz::Int=nothing) + r===nothing && return C_NULL + (sz,) = r + sz = sz===nothing ? position(self) : sz + truncate(self, sz) + return CPyLong_From(sz) +end + +@pyjlwrapfunc function io_seek(self, args::PyPtr)::PyPtr + r = @CPyArg_ParseTuple(args, off::Int, wh::Int=0) + r===nothing && return C_NULL + (off, wh) = r + if wh==0 + seek(self, off) + elseif wh==1 + seek(self, position(self) + off) + elseif wh==2 + seekend(self) + len = position(self) + seek(self, len + off) + else + error("invalid whence") + end + return CPyLong_From(position(self)) +end + +@pyjlwrapfunc function io_seekable(self, ::PyPtr)::PyPtr + # can we improve this? + return CPy_True_NewRef() +end + +@pyjlwrapfunc function io_tell(self, ::PyPtr)::PyPtr + return CPyLong_From(position(self)) +end + +@pyjlwrapfunc function bufferedio_read(self, args::PyPtr)::PyPtr + r = @CPyArg_ParseTuple(args, n::Int=-1) + r===nothing && return C_NULL + (n,) = r + v = n<0 ? read(self) : read(self, n) + return CPyBytes_FromStringAndSize(v, length(v)) +end + +@pyjlwrapfunc function bufferedio_readinto(self, b::PyPtr)::PyPtr + buf = PyBuffer(b) + ptr = convert(Ptr{UInt8}, pointer(buf)) + len = sizeof(buf) + arr = unsafe_wrap(Array, ptr, len) + n = readbytes!(self, arr, len) + return CPyLong_From(n) +end + +@pyjlwrapfunc function bufferedio_write(self, b::PyPtr)::PyPtr + buf = PyBuffer(b) + ptr = convert(Ptr{UInt8}, pointer(buf)) + len = sizeof(buf) + unsafe_write(self, ptr, len) + return CPyLong_From(len) +end + +@pyjlwrapfunc function textio_write(self, x::PyPtr)::PyPtr + y = CPyObject_Str(String, x) + y===nothing && return C_NULL + return CPyLong_From(write(self, y)) +end + +@pyjlwrapfunc function textio_get_encoding_utf8(self, ::Ptr{Cvoid})::PyPtr + return CPyUnicode_From("utf-8") +end + +@pyjlwrapfunc function textio_read(self, args::PyPtr)::PyPtr + r = @CPyArg_ParseTuple(args, n::Int=-1) + r===nothing && return C_NULL + (n,) = r + v = n<0 ? read(self) : read(self, n) + return CPyUnicode_DecodeUTF8(v, length(v), C_NULL) +end + +@pyjlwrapfunc function textio_readline(self, args::PyPtr)::PyPtr + r = @CPyArg_ParseTuple(args, n::Int=-1) + r===nothing && return C_NULL + (n,) = r + s = readline(self, keep=true) + return CPyUnicode_From(s) +end + +@pyjlwrapfunc function get_name_string(o, ::Ptr{Cvoid})::PyPtr + return CPyUnicode_From(string(o)) +end + +@pyjlwrapfunc function get_name_nameof(o, ::Ptr{Cvoid})::PyPtr + return CPyUnicode_From(nameof(o)) +end + +@pyjlwrapfunc function get_doc(o, ::Ptr{Cvoid})::PyPtr + return CPyUnicode_From(string(Docs.doc(o))) +end + +function _pyjlwrap_get_none(o::PyPtr, ::Ptr{Cvoid})::PyPtr + return CPy_None_NewRef() +end + +function pyjlwrap_init_type!( + t::Ref{CPyTypeObject}; + name = missing, + extraflags = zero(Py_TPFLAGS_BASETYPE), + tp_name = name===missing ? missing : "PyCall.$name", + tp_basicsize = sizeof(CPyJlWrapObject), + tp_new = @pyglobal(:PyType_GenericNew), + tp_flags = pyjlwraptype_defaultflags() | extraflags, + opts...) + tp_name === missing && error("name required") + t[] = CPyTypeObject(; tp_name=tp_name, tp_basicsize=tp_basicsize, tp_new=tp_new, tp_flags=tp_flags, opts...) + @pycheckz CPyType_Ready(t) + CPy_IncRef(t) + t +end + +########################################################## +# jlwrap types + +# base type +const CPyJlWrap_Type = Ref(CPyTypeObject_NULL) +# for iterators (a tuple `(x, iterate(x))`) +# abstract base classes from `numbers` +const CPyJlWrapNumber_Type = Ref(CPyTypeObject_NULL) +const CPyJlWrapComplex_Type = Ref(CPyTypeObject_NULL) +const CPyJlWrapReal_Type = Ref(CPyTypeObject_NULL) +const CPyJlWrapRational_Type = Ref(CPyTypeObject_NULL) +const CPyJlWrapIntegral_Type = Ref(CPyTypeObject_NULL) +# abstract base classes from `collections.abc` +const CPyJlWrapContainer_Type = Ref(CPyTypeObject_NULL) +const CPyJlWrapIterable_Type = Ref(CPyTypeObject_NULL) +const CPyJlWrapIterator_Type = Ref(CPyTypeObject_NULL) +const CPyJlWrapCallable_Type = Ref(CPyTypeObject_NULL) +const CPyJlWrapFunction_Type = Ref(CPyTypeObject_NULL) +const CPyJlWrapCollection_Type = Ref(CPyTypeObject_NULL) +const CPyJlWrapSequence_Type = Ref(CPyTypeObject_NULL) +const CPyJlWrapMutableSequence_Type = Ref(CPyTypeObject_NULL) +const CPyJlWrapByteString_Type = Ref(CPyTypeObject_NULL) +const CPyJlWrapSet_Type = Ref(CPyTypeObject_NULL) +const CPyJlWrapMutableSet_Type = Ref(CPyTypeObject_NULL) +const CPyJlWrapMapping_Type = Ref(CPyTypeObject_NULL) +const CPyJlWrapJlNamedTuple_Type = Ref(CPyTypeObject_NULL) +const CPyJlWrapMutableMapping_Type = Ref(CPyTypeObject_NULL) +# abstract base classes from `io` +const CPyJlWrapIOBase_Type = Ref(CPyTypeObject_NULL) +const CPyJlWrapRawIO_Type = Ref(CPyTypeObject_NULL) +const CPyJlWrapBufferedIO_Type = Ref(CPyTypeObject_NULL) +const CPyJlWrapTextIO_Type = Ref(CPyTypeObject_NULL) + +const CPyNumberMethods_default = Ref(CPyNumberMethods_NULL) +const CPyMappingMethods_default = Ref(CPyMappingMethods_NULL) +const CPySequenceMethods_oneup = Ref(CPySequenceMethods_NULL) +const CPyMappingMethods_oneup = Ref(CPyMappingMethods_NULL) +const CPyMappingMethods_namedtuple = Ref(CPyMappingMethods_NULL) + +const have_stackless_extension = Ref(false) + +function pyjlwraptype_defaultflags() + flags = + if pyversion.major ≥ 3 + Py_TPFLAGS_HAVE_VERSION_TAG + else + Py_TPFLAGS_HAVE_GETCHARBUFFER | + Py_TPFLAGS_HAVE_SEQUENCE_IN | + Py_TPFLAGS_HAVE_INPLACEOPS | + Py_TPFLAGS_HAVE_RICHCOMPARE | + Py_TPFLAGS_HAVE_WEAKREFS | + Py_TPFLAGS_HAVE_ITER | + Py_TPFLAGS_HAVE_CLASS | + Py_TPFLAGS_HAVE_INDEX + end + if have_stackless_extension[] + flags |= Py_TPFLAGS_HAVE_STACKLESS_EXTENSION + end + flags +end + +function pyjlwrap_init() + + empty!(GLOBAL_CACHE) + + # detect at runtime whether we are using Stackless Python + try + pyimport("stackless") + have_stackless_extension[] = true + catch + have_stackless_extension[] = false + end + + CPyNumberMethods_default[] = CPyNumberMethods( + nb_bool = @cfunction(_pyjlwrap_istrue, Cint, (PyPtr,)), + nb_int = @cfunction(_pyjlwrap_int, PyPtr, (PyPtr,)), + nb_float = @cfunction(_pyjlwrap_float, PyPtr, (PyPtr,)), + nb_negative = @cfunction(_pyjlwrap_negative, PyPtr, (PyPtr,)), + nb_positive = @cfunction(_pyjlwrap_positive, PyPtr, (PyPtr,)), + nb_absolute = @cfunction(_pyjlwrap_absolute, PyPtr, (PyPtr,)), + nb_invert = @cfunction(_pyjlwrap_invert, PyPtr, (PyPtr,)), + nb_add = @cfunction(_pyjlwrap_add, PyPtr, (PyPtr, PyPtr)), + nb_subtract = @cfunction(_pyjlwrap_subtract, PyPtr, (PyPtr, PyPtr)), + nb_multiply = @cfunction(_pyjlwrap_multiply, PyPtr, (PyPtr, PyPtr)), + nb_remainder = @cfunction(_pyjlwrap_remainder, PyPtr, (PyPtr, PyPtr)), + nb_lshift = @cfunction(_pyjlwrap_lshift, PyPtr, (PyPtr, PyPtr)), + nb_rshift = @cfunction(_pyjlwrap_rshift, PyPtr, (PyPtr, PyPtr)), + nb_and = @cfunction(_pyjlwrap_and, PyPtr, (PyPtr, PyPtr)), + nb_xor = @cfunction(_pyjlwrap_xor, PyPtr, (PyPtr, PyPtr)), + nb_or = @cfunction(_pyjlwrap_or, PyPtr, (PyPtr, PyPtr)), + nb_floordivide = @cfunction(_pyjlwrap_floordivide, PyPtr, (PyPtr, PyPtr)), + nb_truedivide = @cfunction(_pyjlwrap_truedivide, PyPtr, (PyPtr, PyPtr)), + nb_power = @cfunction(_pyjlwrap_power, PyPtr, (PyPtr, PyPtr, PyPtr)), + ) + + CPyMappingMethods_default[] = CPyMappingMethods( + mp_length = @cfunction(_pyjlwrap_length, Cssize_t, (PyPtr,)), + mp_subscript = @cfunction(_pyjlwrap_getitem, PyPtr, (PyPtr, PyPtr)), + mp_ass_subscript = @cfunction(_pyjlwrap_setitem, Cint, (PyPtr, PyPtr, PyPtr)), + ) + + CPySequenceMethods_oneup[] = CPySequenceMethods( + sq_length = @cfunction(_pyjlwrap_length, Cssize_t, (PyPtr,)), + sq_item = @cfunction(_pyjlwrap_getitem_oneup, PyPtr, (PyPtr, Cssize_t)), + ) + + CPyMappingMethods_oneup[] = CPyMappingMethods( + mp_length = @cfunction(_pyjlwrap_length, Cssize_t, (PyPtr,)), + mp_subscript = @cfunction(_pyjlwrap_getitem_oneup, PyPtr, (PyPtr, PyPtr)), + mp_ass_subscript = @cfunction(_pyjlwrap_setitem_oneup, Cint, (PyPtr, PyPtr, PyPtr)), + ) + + CPyMappingMethods_namedtuple[] = CPyMappingMethods( + mp_subscript = @cfunction(_pyjlwrap_getitem_namedtuple, PyPtr, (PyPtr, PyPtr)), + ) + + pyjlwrap_init_type!(CPyJlWrap_Type, + name = "JlWrap", + extraflags = Py_TPFLAGS_BASETYPE, + tp_members = [ + CPyMemberDef(name="__julia_value", typ=T_PYSSIZET, offset=fieldoffset(CPyJlWrapObject, 4), flags=READONLY, doc="Julia jl_value_t* (Any object)"), + ], + tp_dealloc = @cfunction(_pyjlwrap_dealloc, Cvoid, (PyPtr,)), + tp_repr = @cfunction(_pyjlwrap_repr, PyPtr, (PyPtr,)), + tp_str = @cfunction(_pyjlwrap_str, PyPtr, (PyPtr,)), + tp_call = @cfunction(_pyjlwrap_call, PyPtr, (PyPtr,PyPtr,PyPtr)), + tp_getattro = @cfunction(_pyjlwrap_getattr, PyPtr, (PyPtr,PyPtr)), + tp_setattro = @cfunction(_pyjlwrap_setattr, Cint, (PyPtr,PyPtr,PyPtr)), + tp_iter = @cfunction(_pyjlwrap_getiter, PyPtr, (PyPtr,)), + tp_hash = sizeof(Py_hash_t) < sizeof(Int) ? + @cfunction(_pyjlwrap_hash32, UInt32, (PyPtr,)) : + @cfunction(_pyjlwrap_hash, UInt, (PyPtr,)), + tp_weaklistoffset = fieldoffset(CPyJlWrapObject, 3), + tp_richcompare = @cfunction(_pyjlwrap_richcompare, PyPtr, (PyPtr,PyPtr,Cint)), + tp_as_number = CPyNumberMethods_default[], + tp_as_mapping = CPyMappingMethods_default[], + tp_methods = [ + CPyMethodDef(name="__dir__", meth=@cpymethod(_pyjlwrap_dir), flags=METH_NOARGS), + CPyMethodDef(name="__complex__", meth=@cpymethod(_pyjlwrap_complex), flags=METH_NOARGS), + ], + tp_getset = [ + # CPyGetSetDef(name="__doc__", get=@cpygetfunc(_pyjlwrap_get_doc)), + CPyGetSetDef(name="__name__", get=@cpygetfunc(_pyjlwrap_get_name_string)), + ], + ) + + + # ABSTRACT BASE CLASSES FROM `numbers` + + pyjlwrap_init_type!(CPyJlWrapNumber_Type, + name = "JlWrapNumber", + tp_base = CPyJlWrap_Type, + ) + + pyjlwrap_init_type!(CPyJlWrapComplex_Type, + name = "JlWrapComplex", + tp_base = CPyJlWrapNumber_Type, + tp_getset = [ + CPyGetSetDef(name="real", get=@cpygetfunc(_pyjlwrap_get_real)), + CPyGetSetDef(name="imag", get=@cpygetfunc(_pyjlwrap_get_imag)), + ], + tp_methods = [ + CPyMethodDef(name="conjugate", meth=@cpymethod(_pyjlwrap_conjugate), flags=METH_NOARGS), + ] + ) + + pyjlwrap_init_type!(CPyJlWrapReal_Type, + name = "JlWrapReal", + tp_base = CPyJlWrapComplex_Type, + tp_methods = [ + CPyMethodDef(name="trunc", meth=@cpymethod(_pyjlwrap_trunc), flags=METH_NOARGS), + CPyMethodDef(name="round", meth=@cpymethod(_pyjlwrap_round), flags=METH_NOARGS), + CPyMethodDef(name="floor", meth=@cpymethod(_pyjlwrap_floor), flags=METH_NOARGS), + CPyMethodDef(name="ceil", meth=@cpymethod(_pyjlwrap_ceil), flags=METH_NOARGS), + ] + ) + + pyjlwrap_init_type!(CPyJlWrapRational_Type, + name = "JlWrapRational", + tp_base = CPyJlWrapReal_Type, + tp_getset = [ + CPyGetSetDef(name="numerator", get=@cpygetfunc(_pyjlwrap_get_numerator)), + CPyGetSetDef(name="denominator", get=@cpygetfunc(_pyjlwrap_get_denominator)), + ], + ) + + pyjlwrap_init_type!(CPyJlWrapIntegral_Type, + name = "JlWrapIntegral", + tp_base = CPyJlWrapRational_Type, + ) + + # ABSTRACT BASE CLASSES FROM `collections.abc` + + pyjlwrap_init_type!(CPyJlWrapIterable_Type, + name = "JlWrapIterable", + tp_base = CPyJlWrap_Type, + ) + + pyjlwrap_init_type!(CPyJlWrapIterator_Type, + name = "JlWrapIterator", + tp_base = CPyJlWrap_Type, + tp_iter = @cfunction(pyincref_, PyPtr, (PyPtr,)), + tp_iternext = @cfunction(_pyjlwrap_iternext, PyPtr, (PyPtr,)), + ) + + pyjlwrap_init_type!(CPyJlWrapCallable_Type, + name = "JlWrapCallable", + tp_base = CPyJlWrap_Type, + tp_getset = [ + CPyGetSetDef(name="__qualname__", get=@cpygetfunc(_pyjlwrap_get_none)), + CPyGetSetDef(name="__module__", get=@cpygetfunc(_pyjlwrap_get_none)), + CPyGetSetDef(name="__closure__", get=@cpygetfunc(_pyjlwrap_get_none)), + ], + ) + + pyjlwrap_init_type!(CPyJlWrapFunction_Type, + name = "JlWrapFunction", + tp_base = CPyJlWrapCallable_Type, + ) + + pyjlwrap_init_type!(CPyJlWrapContainer_Type, + name = "JlWrapContainer", + tp_base = CPyJlWrap_Type, + ) + + pyjlwrap_init_type!(CPyJlWrapCollection_Type, + name = "JlWrapCollection", + tp_base = CPyJlWrapIterable_Type, + ) + + pyjlwrap_init_type!(CPyJlWrapSequence_Type, + name = "PyCall.JlWrapSequence", + tp_base = CPyJlWrapCollection_Type, + tp_as_sequence = CPySequenceMethods_oneup[], + tp_as_mapping = CPyMappingMethods_oneup[], + ) + + pyjlwrap_init_type!(CPyJlWrapMutableSequence_Type, + name = "JlWrapMutableSequence", + tp_base = CPyJlWrapSequence_Type, + ) + + pyjlwrap_init_type!(CPyJlWrapByteString_Type, + name = "JlWrapByteString", + tp_base = CPyJlWrapSequence_Type, + ) + + pyjlwrap_init_type!(CPyJlWrapSet_Type, + name = "JlWrapSet", + tp_base = CPyJlWrapCollection_Type, + ) + + pyjlwrap_init_type!(CPyJlWrapMutableSet_Type, + name = "JlWrapMutableSet", + tp_base = CPyJlWrapSet_Type, + ) + + pyjlwrap_init_type!(CPyJlWrapMapping_Type, + name = "JlWrapMapping", + tp_base = CPyJlWrapCollection_Type, + tp_as_mapping = CPyMappingMethods_default[], + tp_iter = @cfunction(_pyjlwrap_getiter_keys, PyPtr, (PyPtr,)), + ) + + pyjlwrap_init_type!(CPyJlWrapJlNamedTuple_Type, + name = "JlWrapJlNamedTuple", + tp_base = CPyJlWrapMapping_Type, + tp_as_sequence = CPySequenceMethods_oneup[], + tp_as_mapping = CPyMappingMethods_namedtuple[], + ) + + pyjlwrap_init_type!(CPyJlWrapMutableMapping_Type, + name = "JlWrapMutableMapping", + tp_base = CPyJlWrapMapping_Type, + ) + + # ABSTRACT BASE CLASSES FROM `io` + + pyjlwrap_init_type!(CPyJlWrapIOBase_Type, + name = "JlWrapIOBase", + tp_base = CPyJlWrap_Type, + tp_iter = @cfunction(_pyjlwrap_getiter_lines, PyPtr, (PyPtr,)), + tp_methods = [ + CPyMethodDef(name="close", meth=@cpymethod(_pyjlwrap_io_close), flags=METH_NOARGS), + CPyMethodDef(name="fileno", meth=@cpymethod(_pyjlwrap_io_fileno), flags=METH_NOARGS), + CPyMethodDef(name="flush", meth=@cpymethod(_pyjlwrap_io_flush), flags=METH_NOARGS), + CPyMethodDef(name="isatty", meth=@cpymethod(_pyjlwrap_io_isatty), flags=METH_NOARGS), + CPyMethodDef(name="readable", meth=@cpymethod(_pyjlwrap_io_readable), flags=METH_NOARGS), + CPyMethodDef(name="seek", meth=@cpymethod(_pyjlwrap_io_seek), flags=METH_VARARGS), + CPyMethodDef(name="seekable", meth=@cpymethod(_pyjlwrap_io_seekable), flags=METH_NOARGS), + CPyMethodDef(name="tell", meth=@cpymethod(_pyjlwrap_io_tell), flags=METH_NOARGS), + CPyMethodDef(name="truncate", meth=@cpymethod(_pyjlwrap_io_truncate), flags=METH_VARARGS), + CPyMethodDef(name="writable", meth=@cpymethod(_pyjlwrap_io_writable), flags=METH_NOARGS), + ], + tp_getset = [ + CPyGetSetDef(name="closed", get=@cpygetfunc(_pyjlwrap_io_get_closed)), + ], + ) + + pyjlwrap_init_type!(CPyJlWrapRawIO_Type, + name = "JlWrapRawIO", + tp_base = CPyJlWrapIOBase_Type, + ) + + pyjlwrap_init_type!(CPyJlWrapBufferedIO_Type, + name = "JlWrapBufferedIO", + tp_base = CPyJlWrapIOBase_Type, + tp_methods = [ + CPyMethodDef(name="read", meth=@cpymethod(_pyjlwrap_bufferedio_read), flags=METH_VARARGS), + CPyMethodDef(name="readinto", meth=@cpymethod(_pyjlwrap_bufferedio_readinto), flags=METH_O), + CPyMethodDef(name="write", meth=@cpymethod(_pyjlwrap_bufferedio_write), flags=METH_O), + ], + ) + + pyjlwrap_init_type!(CPyJlWrapTextIO_Type, + name = "JlWrapTextIO", + tp_base = CPyJlWrapIOBase_Type, + tp_methods = [ + CPyMethodDef(name="read", meth=@cpymethod(_pyjlwrap_textio_read), flags=METH_VARARGS), + CPyMethodDef(name="readline", meth=@cpymethod(_pyjlwrap_textio_readline), flags=METH_VARARGS), + CPyMethodDef(name="write", meth=@cpymethod(_pyjlwrap_textio_write), flags=METH_O), + ], + tp_getset = [ + CPyGetSetDef(name="encoding", get=@cpygetfunc(_pyjlwrap_textio_get_encoding_utf8)) + ], + ) + + # register with abstract base classes + m = pyimport("numbers") + m.Number.register(CPyJlWrapNumber_Type) + m.Complex.register(CPyJlWrapComplex_Type) + m.Real.register(CPyJlWrapReal_Type) + m.Rational.register(CPyJlWrapRational_Type) + m.Integral.register(CPyJlWrapIntegral_Type) + + m = pyimport("collections.abc") + m.Iterable.register(CPyJlWrapIterable_Type) + m.Iterator.register(CPyJlWrapIterator_Type) + m.Callable.register(CPyJlWrapCallable_Type) + m.Container.register(CPyJlWrapContainer_Type) + m.Collection.register(CPyJlWrapCollection_Type) + m.Sequence.register(CPyJlWrapSequence_Type) + m.MutableSequence.register(CPyJlWrapMutableSequence_Type) + m.ByteString.register(CPyJlWrapByteString_Type) + m.Set.register(CPyJlWrapSet_Type) + m.MutableSet.register(CPyJlWrapMutableSet_Type) + m.Mapping.register(CPyJlWrapMapping_Type) + m.MutableMapping.register(CPyJlWrapMutableMapping_Type) + + m = pyimport("io") + m.IOBase.register(CPyJlWrapIOBase_Type) + m.RawIOBase.register(CPyJlWrapRawIO_Type) + m.TextIOBase.register(CPyJlWrapTextIO_Type) + +end + + + + + + + + +########################################################## +# constructors for specific types + +CPyJlWrap_From(x) = CPyJlWrap_New(CPyJlWrap_Type, x) +CPyJlWrap_From(x::Union{AbstractDict,AbstractArray,AbstractSet,NamedTuple,Tuple}) = CPyJlWrapIterable_From(x) +CPyJlWrap_From(x::Number) = CPyJlWrapNumber_From(x) +CPyJlWrap_From(x::IO) = CPyJlWrapIO_From(x) +CPyJlWrap_From(x::Union{Function,Type}) = CPyJlWrapCallable_From(x) + +CPyJlWrapIterator_From(o) = + let it = iterate(o) + CPyJlWrap_New(CPyJlWrapIterator_Type, (o, Ref{Any}(it))) + end + +CPyJlWrapNumber_From(x) = CPyJlWrap_New(CPyJlWrapNumber_Type, x) +CPyJlWrapNumber_From(x::Complex) = CPyJlWrapComplex_From(x) +CPyJlWrapNumber_From(x::Real) = CPyJlWrapReal_From(x) + +CPyJlWrapComplex_From(x) = CPyJlWrap_New(CPyJlWrapComplex_Type, x) +CPyJlWrapComplex_From(x::Real) = CPyJlWrapReal_From(x) + +CPyJlWrapReal_From(x) = CPyJlWrap_New(CPyJlWrapReal_Type, x) +CPyJlWrapReal_From(x::Integer) = CPyJlWrapIntegral_From(x) +CPyJlWrapReal_From(x::Rational) = CPyJlWrapRational_From(x) + +CPyJlWrapRational_From(x) = CPyJlWrap_New(CPyJlWrapRational_Type, x) +CPyJlWrapRational_From(x::Integer) = CPyJlWrapIntegral_From(x) + +CPyJlWrapIntegral_From(x) = CPyJlWrap_New(CPyJlWrapIntegral_Type, x) + +CPyJlWrapIterable_From(o) = CPyJlWrap_New(CPyJlWrapIterable_Type, o) +CPyJlWrapIterable_From(o::Union{AbstractDict,AbstractArray,AbstractSet,NamedTuple,Tuple}) = CPyJlWrapCollection_From(o) + +CPyJlWrapCallable_From(o) = CPyJlWrap_New(CPyJlWrapCallable_Type, o) +CPyJlWrapCallable_From(o::Union{Function,Type}) = CPyJlWrapFunction_From(o) + +CPyJlWrapFunction_From(o) = CPyJlWrap_New(CPyJlWrapFunction_Type, o) + +CPyJlWrapCollection_From(o) = CPyJlWrap_New(CPyJlWrapCollection_Type, o) +CPyJlWrapCollection_From(o::Union{Tuple,AbstractArray}) = CPyJlWrapSequence_From(o) +CPyJlWrapCollection_From(o::AbstractSet) = CPyJlWrapSet_From(o) +CPyJlWrapCollection_From(o::Union{AbstractDict,NamedTuple}) = CPyJlWrapMapping_From(o) + +CPyJlWrapSequence_From(o) = CPyJlWrap_New(CPyJlWrapSequence_Type, o) +CPyJlWrapSequence_From(o::AbstractArray) = CPyJlWrapMutableSequence_From(o) + +CPyJlWrapMutableSequence_From(o) = CPyJlWrap_New(CPyJlWrapMutableSequence_Type, o) + +CPyJlWrapMapping_From(o) = CPyJlWrap_New(CPyJlWrapMapping_Type, o) +CPyJlWrapMapping_From(o::Base.ImmutableDict) = CPyJlWrap_New(CPyJlWrapMapping_Type, o) +CPyJlWrapMapping_From(o::AbstractDict) = CPyJlWrapMutableMapping_From(o) +CPyJlWrapMapping_From(o::NamedTuple) = CPyJlWrapJlNamedTuple_From(o) + +CPyJlWrapJlNamedTuple_From(o) = CPyJlWrap_New(CPyJlWrapJlNamedTuple_Type, o) + +CPyJlWrapMutableMapping_From(o) = CPyJlWrap_New(CPyJlWrapMutableMapping_Type, o) + +CPyJlWrapSet_From(o) = CPyJlWrap_New(CPyJlWrapSet_Type, o) +CPyJlWrapSet_From(o::AbstractSet) = CPyJlWrapMutableSet_From(o) + +CPyJlWrapMutableSet_From(o) = CPyJlWrap_New(CPyJlWrapMutableSet_Type, o) + +CPyJlWrapIO_From(o) = CPyJlWrap_New(CPyJlWrapIOBase_Type, o) +CPyJlWrapIO_From(o::IO) = CPyJlWrapBufferedIO_From(o) + +CPyJlWrapBufferedIO_From(o) = CPyJlWrap_New(CPyJlWrapBufferedIO_Type, o) + +CPyJlWrapRawIO_From(o) = CPyJlWrap_New(CPyJlWrapRawIO_Type, o) + +CPyJlWrapTextIO_From(o) = CPyJlWrap_New(CPyJlWrapTextIO_Type, o) + + + +export pyjlwrap, pyjlwrap_textio, pyjlwrap_rawio, pyjlwrap_bufferedio +pyjlwrap(x) = PyObject(CPyJlWrap_From(x)) +pyjlwrap_rawio(x) = PyObject(CPyJlWrapRawIO_From(x)) +pyjlwrap_textio(x) = PyObject(CPyJlWrapTextIO_From(x)) +pyjlwrap_bufferedio(x) = PyObject(CPyJlWrapBufferedIO_From(x)) + + + + +######################################################################### +# Precompilation: just an optimization to speed up initialization. +# Here, we precompile functions that are passed to cfunction by __init__, +# for the reasons described in JuliaLang/julia#12256. +precompile(_pyjlwrap_call, (PyPtr,PyPtr,PyPtr)) +precompile(_pyjlwrap_dealloc, (PyPtr,)) +precompile(_pyjlwrap_repr, (PyPtr,)) +precompile(_pyjlwrap_hash, (PyPtr,)) +precompile(_pyjlwrap_hash32, (PyPtr,)) + diff --git a/src/libpython/extensions.jl b/src/libpython/extensions.jl new file mode 100644 index 00000000..93cafd59 --- /dev/null +++ b/src/libpython/extensions.jl @@ -0,0 +1,587 @@ +# Extensions to the C API +# +# These functions will only raise exceptions arising from julia. Python exceptions are propagated by return value as usual. +# +# Functions `CPy_From(x)` convert `x` to a python `type`. If `x` is a `PyPtr`, this is equivalent to `(x)` in python. Otherwise, it typically is a wrapper for one of the `CPy_From` functions in the C API. Returns `C_NULL` on error. +# +# Functions `CPy_As(T, x)` convert `x` (which should be a `PyPtr` of python type `type`) to a `T`. Returns `nothing` on error. +# +# `CPyObject_From(x)` defines the default conversion from julia to python. + + + +""" + CPyDict_FromIterator([fk=identity, fv=identity,] x) + +A python `dict` whose keys are `fk(k)` and values are `fv(v)` for `(k,v) ∈ x`. + +The keys and values must be `PyPtr`. This steals references, so the elements must be new references. +""" +function CPyDict_FromIterator(fk, fv, x) :: PyPtr + t = CPyDict_New() + t == C_NULL && (return C_NULL) + for (k, v) in x + kk = fk(k) :: PyPtr + kk == C_NULL && (CPy_DecRef(t); return C_NULL) + vv = fv(v) :: PyPtr + vv == C_NULL && (CPy_DecRef(kk); CPy_DecRef(t); return C_NULL) + e = CPyDict_SetItem(t, kk, vv) + CPy_DecRef(kk) + CPy_DecRef(vv) + e == -1 && (CPy_DecRef(t); return C_NULL) + end + t +end + +CPyDict_FromIterator(x) = + CPyDict_FromIterator(identity, identity, x) + +function CPyDict_From(x::PyPtr) :: PyPtr + CPyObject_CallFunction(CPyDict_Type[], x) +end + + + + + +""" + CPyTuple_FromIterator([f=identity,] x) + +A python `tuple` whose elements come from `map(f, x)`. + +The length of `x` must be known. The elements must be `PyPtr`. This steals references, so the elements must be new references. +""" +function CPyTuple_FromIterator(f, x) :: PyPtr + t = CPyTuple_New(length(x)) + t == C_NULL && (return C_NULL) + i = 0 + for y in x + z = f(y) :: PyPtr + z == C_NULL && (CPy_DecRef(t); return C_NULL) + e = CPyTuple_SetItem(t, i, z) + e == -1 && (CPy_DecRef(t); return C_NULL) + i += 1 + end + t +end + +CPyTuple_FromIterator(x) = CPyTuple_FromIterator(identity, x) + +function CPyTuple_From(x::PyPtr)::PyPtr + CPyObject_CallFunction(CPyTuple_Type[], x) +end + + + + + + + + +""" + CPyList_FromIterator([f=identity,] x) + +A python `list` whose elements come from `map(f, x)`. + +The elements must be `PyPtr`. This steals references, so the elements must be new references. +""" +function CPyList_FromIterator(f, x) :: PyPtr + # potential optimization: check if the length of x is known in advance + t = CPyList_New(0) + t == C_NULL && (return C_NULL) + for y in x + z = f(y) :: PyPtr + z == C_NULL && (CPy_DecRef(t); return C_NULL) + e = CPyList_Append(t, z) + e == -1 && (CPy_DecRef(t); return C_NULL) + end + t +end + +CPyList_FromIterator(x) = CPyList_FromIterator(identity, x) + +function CPyList_From(x::PyPtr)::PyPtr + CPyObject_CallFunction(CPyList_Type[], x) +end + + + +function CPyUnicode_From(x::PyPtr)::PyPtr + CPyObject_CallFunction(CPyUnicode_Type[], x) +end + +function CPyUnicode_From(x::String)::PyPtr + CPyUnicode_DecodeUTF8(Base.unsafe_convert(Ptr{UInt8}, x), sizeof(x), C_NULL) +end + +function CPyUnicode_From(x::AbstractString)::PyPtr + CPyUnicode_From(convert(String, x)) +end + +function CPyUnicode_As(::Type{T}, x) where {T<:AbstractString} + b = CPyUnicode_AsUTF8String(x) + b == C_NULL && (return nothing) + buf = Ref{Ptr{UInt8}}(C_NULL) + len = Ref{Cssize_t}(0) + z = CPyBytes_AsStringAndSize(b, buf, len) + z == -1 && (CPy_DecRef(b); return nothing) + s = unsafe_string(buf[], len[]) + CPy_DecRef(b) + convert(T, s) +end + + + + + +function CPyBool_From(x::PyPtr)::PyPtr + r = CPyObject_IsTrue(x) + r == -1 && (return C_NULL) + CPyBool_From(r == 1) +end + +function CPyBool_From(x::Bool)::PyPtr + r = x ? CPy_True[] : CPy_False[] + CPy_IncRef(r) + r +end + + + + +function CPyLong_From(x::PyPtr)::PyPtr + CPyObject_CallFunction(CPyLong_Type[], x) +end + +function CPyLong_From(x::AbstractString)::PyPtr + y = CPyUnicode_From(x) + y == C_NULL && (return C_NULL) + z = CPyLong_From(y) + CPy_DecRef(y) + z +end + +function CPyLong_From(x::Real)::PyPtr + y = CPyFloat_From(x) + y == C_NULL && (return C_NULL) + z = CPyLong_From(y) + CPy_DecRef(y) + z +end + +function CPyLong_From(x::Complex)::PyPtr + y = CPyComplex_From(x) + y == C_NULL && (return C_NULL) + z = CPyLong_From(y) + CPy_DecRef(y) + z +end + +function CPyLong_From(x::Number)::PyPtr + CPyLong_From(convert(Real, x)::Real) +end + +function CPyLong_From(x)::PyPtr + CPyLong_From(convert(Number, x)::Number) +end + +@static if pyversion < v"3" + function CPyLong_From(x::T) where {T<:Integer} + if isbitstype(T) + if T <: Unsigned + if sizeof(T) ≤ sizeof(Csize_t) + return CPyInt_FromSize_t(x) + end + else + if sizeof(T) ≤ sizeof(Cssize_t) + return CPyInt_FromSsize_t(x) + end + end + end + CPyLong_From(string(x)) + end +else + function CPyLong_From(x::T) where {T<:Integer} + if isbitstype(T) + if T <: Unsigned + if sizeof(T) ≤ sizeof(Culonglong) + return CPyLong_FromUnsignedLongLong(x) + end + else + if sizeof(T) ≤ sizeof(Clonglong) + return CPyLong_FromLongLong(x) + end + end + end + CPyLong_From(string(x)) + end +end + +@static if pyversion < v"3" + function CPyLong_As(::Type{T}, o) where {T<:Integer} + val = CPyInt_AsSsize_t(o) + if val == -1 && CPyErr_Occurred() != C_NULL + CPyErr_Clear() + convert(T, convert(BigInt, o)) + else + convert(T, val) + end + end +elseif pyversion < v"3.2" + function CPyLong_As(::Type{T}, o) where {T<:Integer} + val = CPyLong_AsLongLong(o) + if val == -1 && CPyErr_Occurred() != C_NULL + CPyErr_Clear() + convert(T, convert(BigInt, o)) + else + convert(T, val) + end + end +else + function CPyLong_As(::Type{T}, o) where {T<:Integer} + overflow = Ref{Cint}() + val = CPyLong_AsLongLongAndOverflow(o, overflow) + val == -1 && CPyErr_Occurred() != C_NULL && (return nothing) + if iszero(overflow[]) + convert(T, val) + else + convert(T, convert(BigInt, o)) + end + end +end + +function CPyLong_As(::Type{BigInt}, o) + s = CPyObject_Str(String, o) + s===nothing && return nothing + parse(BigInt, s) +end + + + + +function CPyFloat_From(x::PyPtr)::PyPtr + CPyObject_CallFunction(CPyFloat_Type[], x) +end + +function CPyFloat_From(x::Float64)::PyPtr + CPyFloat_FromDouble(x) +end + +function CPyFloat_From(x::Real)::PyPtr + CPyFloat_From(convert(Float64, x)) +end + +function CPyFloat_From(x::Number)::PyPtr + CPyFloat_From(convert(Real, x)) +end + +function CPyFloat_From(x)::PyPtr + CPyFloat_From(convert(Number, x)) +end + +function CPyFloat_From(x::AbstractString)::PyPtr + y = CPyUnicode_From(x) + y == C_NULL && (return C_NULL) + z = CPyFloat_From(y) + CPy_DecRef(y) + z +end + +function CPyFloat_As(::Type{T}, x) where {T<:Number} + y = CPyFloat_AsDouble(x) + y == -1 && CPyErr_Occurred() != C_NULL && (return nothing) + convert(T, y) +end + + + + +function CPyComplex_From(x::PyPtr)::PyPtr + CPyObject_CallFunction(CPyComplex_Type[], x) +end + +function CPyComplex_From(x::Complex)::PyPtr + CPyComplex_FromDoubles(real(x), imag(x)) +end + +function CPyComplex_From(x::Number)::PyPtr + y = CPyFloat_From(x) + y == C_NULL && (return C_NULL) + z = CPyComplex_From(y) + CPy_DecRef(y) + z +end + +function CPyComplex_As(::Type{T}, x) where {T<:Number} + y = CPyComplex_AsCComplex(x) + CPyErr_Occurred() == C_NULL || (return nothing) + convert(T, Complex(y.real, y.imag)) +end + + + +function CPyBytes_From(x::PyPtr)::PyPtr + CPyObject_CallFunction(CPyBytes_Type[], x) +end + +function CPyBytes_From(x::AbstractVector{UInt8})::PyPtr + # try to avoid copying x + try + if stride(x, 1) == 1 + ptr = Base.unsafe_convert(Ptr{UInt8}, x) + sz = sizeof(x) + return CPyBytes_FromStringAndSize(ptr, sz) + end + catch + end + y = convert(Vector, x) + ptr = Base.unsafe_convert(Ptr{UInt8}, y) + sz = sizeof(y) + return CPyBytes_FromStringAndSize(ptr, sz) +end + + + + + +function CPyObject_Repr(::Type{T}, x) where {T<:AbstractString} + y = CPyObject_Repr(x) + y == C_NULL && (return nothing) + s = CPyUnicode_As(T, y) + CPy_DecRef(y) + s +end + +function CPyObject_ASCII(::Type{T}, x) where {T<:AbstractString} + y = CPyObject_ASCII(x) + y == C_NULL && (return nothing) + s = CPyUnicode_As(T, y) + CPy_DecRef(y) + s +end + +function CPyObject_Str(::Type{T}, x) where {T<:AbstractString} + y = CPyObject_Str(x) + y == C_NULL && (return nothing) + s = CPyUnicode_As(T, y) + CPy_DecRef(y) + s +end + + +function CPyObject_As end + +function CPyObject_As(::Type{Nothing}, x) + nothing +end + +function CPyObject_As(::Type{Missing}, x) + missing +end + +function CPyObject_As(::Type{Bool}, x) + r = CPyObject_IsTrue(x) + r == -1 ? nothing : r == 1 +end + +function CPyObject_As(::Type{T}, x) where {T<:AbstractString} + CPyObject_Str(T, x) +end + +function CPyObject_As(::Type{T}, x) where {T<:Real} + CPyFloat_As(T, x) +end + + + + +function CPyObject_From(x::PyPtr)::PyPtr + x == C_NULL || CPy_IncRef(x) + return x +end + +function CPyObject_From(x::Integer)::PyPtr + CPyLong_From(x) +end + +function CPyObject_From(x::Real)::PyPtr + CPyFloat_From(x) +end + +function CPyObject_From(x::Complex)::PyPtr + CPyComplex_From(x) +end + +function CPyObject_From(x::Nothing)::PyPtr + CPy_None_NewRef() +end + +function CPyObject_From(x::Bool)::PyPtr + CPyBool_From(x) +end + +function CPyObject_From(x::AbstractString)::PyPtr + CPyUnicode_From(x) +end + +function CPyObject_From(x::Symbol)::PyPtr + CPyUnicode_From(string(x)) +end + +function CPyObject_From(x::Tuple)::PyPtr + CPyTuple_FromIterator(CPyObject_From, x) +end + +function CPyObject_From(x::AbstractDict)::PyPtr + CPyDict_FromIterator(CPyObject_From, CPyObject_From, pairs(x)) +end + +function CPyObject_From(x::AbstractVector)::PyPtr + CPyList_FromIterator(CPyObject_From, x) +end + +function CPyObject_From(x::AbstractSet)::PyPtr + CPySet_FromIterator(CPyObject_From, x) +end + + + + + +function CPyObject_CallFunction(f, args...; kwargs...)::PyPtr + if isempty(kwargs) + _args = CPyTuple_FromIterator(CPyObject_From, args) + _args == C_NULL && (return C_NULL) + r = CPyObject_CallObject(f, _args) + CPy_DecRef(_args) + return r + else + _args = CPyTuple_FromIterator(CPyObject_From, args) + _args == C_NULL && (return C_NULL) + _kwargs = CPyDict_FromIterator(CPyObject_From, CPyObject_From, kwargs) + _kwargs == C_NULL && (CPy_DecRef(_args); return C_NULL) + r = CPyObject_Call(f, _args, _kwargs) + CPy_DecRef(_args) + CPy_DecRef(_kwargs) + return r + end +end + +function cpyargdata(arg) + # parse out the default + if arg isa Expr && arg.head == :(=) + lhs, dflt = arg.args + dflt = Some(dflt) + else + lhs = arg + dflt = nothing + end + # parse out the type + if lhs isa Expr && lhs.head == :(::) + argname, typ = lhs.args + else + argname = lhs + typ = :PyPtr + end + # check the argname + argname isa Symbol || error("invalid argument: $arg") + argname = argname==:_ ? nothing : argname + # done + (name=argname, typ=typ, dflt=dflt) +end + +function cpyargsdata(args) + argsdata = map(cpyargdata, args) + # number of required arguments + numreq = 0 + for arg in argsdata + if arg.dflt===nothing + numreq += 1 + else + break + end + end + # (minimum) number of positional arguments + numpos = 0 + for arg in argsdata + if arg.name == nothing + numpos += 1 + else + break + end + end + # + (args=argsdata, nreq=numreq, npos=numpos) +end + +cpyargparse(::Type{PyPtr}, x::PyPtr) = (CPy_IncRef(x); x) +cpyargparse(::Type{T}, x::PyPtr) where {T<:Integer} = + CPyLong_As(T, x) +cpyargparse(::Type{T}, x::PyPtr) where {T<:Real} = + CPyFloat_As(T, x) +cpyargparse(::Type{T}, x::PyPtr) where {T<:AbstractString} = + CPyUnicode_As(T, x) + +cpyargfree(x) = nothing +cpyargfree(x::PyPtr) = CPy_DecRef(x) + +function _cpyargsparse(t, k, args) + data = cpyargsdata(args) + tnames = [Symbol(:t, i-1) for i in 1:length(args)] + anames = [Symbol(:x, i-1) for i in 1:length(args)] + body = quote + # check the length + len = CPyTuple_Size(t) + len == -1 && @goto(ERROR) + len ≤ $(length(args)) || @goto(ERROR) # TODO: set an error + # default PyPtr values (these are borrowed references) + $([:($n :: PyPtr = C_NULL) for n in tnames]...) + # parse the tuple + $([:(len ≥ $i && ($n = CPyTuple_GetItem(t, $(i-1)); $n == C_NULL && @goto(ERROR))) for (i,(a,n)) in enumerate(zip(data.args, tnames))]...) + # parse the kwargs + $(k===nothing ? nothing : error("parsing keywords not implemented")) + # extra parsing + $([quote + $n = + if $t == C_NULL + $(a.dflt===nothing ? :(#=TODO: set an error=# nothing) : :(Some($(esc(something(a.dflt)))))) + else + cpyargparse($(esc(a.typ)), $t) + end + $n === nothing && @goto($(Symbol(:ERROR_,n))) + end for (a,n,t) in zip(data.args, anames, tnames)]...) + # return the tuple + return ($([:(something($n)) for n in anames]...),) + # errors + $([:(cpyargfree($n); @label($(Symbol(:ERROR_,n)))) for n in reverse(anames)]...) + @label ERROR + return nothing + end + :((function (t, k); $body; end)($(esc(t)), $(k===nothing ? C_NULL : esc(k)))) +end + +""" + @CPyArg_ParseTuple(args, NAME::TYPE=DEFAULT) + +Similar to `@CPyArg_ParseTupleAndKeywords` but only takes an `args` tuple. + +Note that the `NAME` is ignored in this case. +""" +macro CPyArg_ParseTuple(t, args...) + _cpyargsparse(t, nothing, args) +end + +""" + @CPyArg_ParseTupleAndKeywords(args, kwargs, NAME::TYPE=DEFAULT, ...) + +Parse `args` (a `PyPtr` to a tuple) and `kwargs` (a `PyPtr` to a dict) according to the argument specifiers of the form `NAME::TYPE=DEFAULT`. Return a tuple whose entries are of types `TYPE`, or `nothing` on error. + +The `NAME` may be `_` to signify a positional-only argument. Otherwise, it is the name of a possibly-keyword argument. + +The `TYPE` is optional, and defaults to `PyPtr`. Currently it can be one of `PyPtr`, `<:Integer`, `<:Real`, `<:AbstractString`. + +The `DEFAULT` is optional. When not specified, the argument is required. +""" +macro CPyArg_ParseTupleAndKeywords(t, k, args...) + _cpyargsparse(t, k, args...) +end \ No newline at end of file diff --git a/src/libpython/functions.jl b/src/libpython/functions.jl new file mode 100644 index 00000000..b52c6271 --- /dev/null +++ b/src/libpython/functions.jl @@ -0,0 +1,175 @@ +CPYTHON_FUNCTIONS = [ + # initialization + (:Py_IsInitialized, Cint, ()), + (:Py_Initialize, Cvoid, ()), + (:Py_InitializeEx, Cvoid, (Cint,)), + (:Py_SetPythonHome, Cvoid, (Cwstring,)), + (:Py_SetProgramName, Cvoid, (Cwstring,)), + (:Py_GetVersion, Cstring, ()), + (:Py_AtExit, Cint, (Ptr{Cvoid},)), + (:Py_Finalize, Cvoid, ()), + # refcount + (:Py_DecRef, Cvoid, (PyPtr,)), + (:Py_IncRef, Cvoid, (PyPtr,)), + # errors + (:PyErr_Clear, Cvoid, ()), + (:PyErr_Print, Cvoid, ()), + (:PyErr_Occurred, PyPtr, ()), + (:PyErr_Fetch, Cvoid, (Ptr{PyPtr}, Ptr{PyPtr}, Ptr{PyPtr})), + (:PyErr_NormalizeException, Cvoid, (Ptr{PyPtr}, Ptr{PyPtr}, Ptr{PyPtr})), + # import + (:PyImport_ImportModule, PyPtr, (Cstring,)), + (:PyImport_Import, PyPtr, (PyPtr,)), + # types + (:PyType_Ready, Cint, (Ptr{CPyTypeObject},)), + # sys + (:PySys_SetArgvEx, Cvoid, (Cint, Ptr{Ptr{Cvoid}}, Cint)), + # object + (:_PyObject_New=>:PyObject_New, PyPtr, (Ptr{CPyTypeObject},)), + (:PyObject_RichCompare, PyPtr, (PyPtr,PyPtr,Cint)), + (:PyObject_RichCompareBool, Cint, (PyPtr,PyPtr,Cint)), + (:PyObject_IsTrue, Cint, (PyPtr,)), + (:PyObject_Not, Cint, (PyPtr,)), + (:PyObject_IsInstance, Cint, (PyPtr, PyPtr)), + (:PyObject_Type, PyPtr, (PyPtr,)), + (:PyObject_IsSubclass, Cint, (PyPtr, PyPtr)), + (:PyObject_Repr, PyPtr, (PyPtr,)), + (:PyObject_ASCII, PyPtr, (PyPtr,)), + (:PyObject_Str, PyPtr, (PyPtr,)), + (:PyObject_Bytes, PyPtr, (PyPtr,)), + (:PyObject_GetItem, PyPtr, (PyPtr, PyPtr)), + (:PyObject_SetItem, Cint, (PyPtr, PyPtr, PyPtr)), + (:PyObject_DelItem, Cint, (PyPtr, PyPtr)), + (:PyObject_Dir, PyPtr, (PyPtr,)), + (:PyObject_GetIter, PyPtr, (PyPtr,)), + (:PyObject_HasAttr, Cint, (PyPtr, PyPtr)), + (:PyObject_HasAttrString, Cint, (PyPtr, Cstring)), + (:PyObject_GetAttr, PyPtr, (PyPtr, PyPtr)), + (:PyObject_GenericGetAttr, PyPtr, (PyPtr, PyPtr)), + (:PyObject_GetAttrString, PyPtr, (PyPtr, Cstring)), + (:PyObject_SetAttr, Cint, (PyPtr, PyPtr, PyPtr)), + (:PyObject_GenericSetAttr, Cint, (PyPtr, PyPtr, PyPtr)), + (:PyObject_SetAttrString, Cint, (PyPtr, Cstring, PyPtr)), + (:PyObject_Length, Cssize_t, (PyPtr,)), + (:PyObject_Call, PyPtr, (PyPtr, PyPtr, PyPtr)), + (:PyObject_CallObject, PyPtr, (PyPtr, PyPtr)), + (:PyObject_ClearWeakRefs, Cvoid, (PyPtr,)), + # number + (:PyNumber_Check, Cint, (PyPtr,)), + (:PyNumber_Add, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_Subtract, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_Multiply, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_MatrixMultiply, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_FloorDivide, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_TrueDivide, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_Remainder, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_Divmod, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_Power, PyPtr, (PyPtr,PyPtr,PyPtr)), + (:PyNumber_Negative, PyPtr, (PyPtr,)), + (:PyNumber_Positive, PyPtr, (PyPtr,)), + (:PyNumber_Absolute, PyPtr, (PyPtr,)), + (:PyNumber_Invert, PyPtr, (PyPtr,)), + (:PyNumber_Lshift, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_Rshift, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_And, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_Xor, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_Or, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_InPlaceAdd, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_InPlaceSubtract, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_InPlaceMultiply, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_InPlaceMatrixMultiply, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_InPlaceFloorDivide, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_InPlaceTrueDivide, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_InPlaceRemainder, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_InPlaceLshift, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_InPlaceRshift, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_InPlaceAnd, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_InPlaceXor, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_InPlaceOr, PyPtr, (PyPtr, PyPtr)), + (:PyNumber_Long, PyPtr, (PyPtr,)), + (:PyNumber_Float, PyPtr, (PyPtr,)), + (:PyNumber_Index, PyPtr, (PyPtr,)), + (:PyNumber_ToBase, PyPtr, (PyPtr, Cint)), + # sequence + (:PySequence_Check, Cint, (PyPtr,)), + (:PySequence_Length, Cssize_t, (PyPtr,)), + (:PySequence_SetItem, Cint, (PyPtr, Cssize_t, PyPtr)), + (:PySequence_GetItem, PyPtr, (PyPtr, Cssize_t)), + (:PySequence_Concat, PyPtr, (PyPtr, PyPtr)), + (:PySequence_Repeat, PyPtr, (PyPtr, Cssize_t)), + (:PySequence_Contains, Cint, (PyPtr, PyPtr)), + # mapping + (:PyMapping_Check, Cint, (PyPtr,)), + (:PyMapping_Length, Cssize_t, (PyPtr,)), + (:PyMapping_HasKey, Cint, (PyPtr, PyPtr)), + (:PyMapping_Keys, PyPtr, (PyPtr,)), + (:PyMapping_Values, PyPtr, (PyPtr,)), + (:PyMapping_Items, PyPtr, (PyPtr,)), + # buffer + (:PyObject_GetBuffer, Cint, (PyPtr, Ptr{CPy_buffer}, Cint)), + (:PyBuffer_Release, Cvoid, (Ptr{CPy_buffer},)), + # iter + (:PyIter_Next, PyPtr, (PyPtr,)), + # int + (:PyLong_FromLong, PyPtr, (Clong,)), + (:PyLong_FromUnsignedLong, PyPtr, (Culong,)), + (:PyLong_FromSsize_t, PyPtr, (Cssize_t,)), + (:PyLong_FromSize_t, PyPtr, (Csize_t,)), + (:PyLong_FromLongLong, PyPtr, (Clonglong,)), + (:PyLong_FromUnsignedLongLong, PyPtr, (Culonglong,)), + (:PyLong_FromDouble, PyPtr, (Cdouble,)), + (:PyLong_AsLong, Clong, (PyPtr,)), + (:PyLong_AsLongAndOverflow, Clong, (PyPtr, Ptr{Cint})), + (:PyLong_AsLongLong, Clonglong, (PyPtr,)), + (:PyLong_AsLongLongAndOverflow, Clonglong, (PyPtr, Ptr{Cint})), + (:PyLong_AsSsize_t, Cssize_t, (PyPtr,)), + (:PyLong_AsUnsignedLong, Culong, (PyPtr,)), + (:PyLong_AsSize_t, Csize_t, (PyPtr,)), + (:PyLong_AsUnsignedLongLong, Culonglong, (PyPtr,)), + (:PyLong_AsUnsignedLongMask, Clong, (PyPtr,)), + (:PyLong_AsUnsignedLongLongMask, Culonglong, (PyPtr,)), + (:PyLong_AsDouble, Cdouble, (PyPtr,)), + # float + (:PyFloat_FromString, PyPtr, (PyPtr,)), + (:PyFloat_FromDouble, PyPtr, (Cdouble,)), + (:PyFloat_AsDouble, Cdouble, (PyPtr,)), + # complex + (:PyComplex_FromCComplex, PyPtr, (CPy_complex,)), + (:PyComplex_FromDoubles, PyPtr, (Cdouble, Cdouble)), + (:PyComplex_AsCComplex, CPy_complex, (PyPtr,)), + # bytes + (:PyBytes_FromStringAndSize, PyPtr, (Ptr{Cchar}, Cssize_t)), + (:PyBytes_AsStringAndSize, Cint, (PyPtr, Ptr{Ptr{UInt8}}, Ptr{Cssize_t})), + # str + (:PyUnicode_AsUTF8String, PyPtr, (PyPtr,)), + (:PyUnicode_DecodeUTF8, PyPtr, (Ptr{UInt8}, Cssize_t, Ptr{UInt8})), + # list + (:PyList_New, PyPtr, (Cssize_t,)), + (:PyList_SetItem, Cint, (PyPtr, Cssize_t, PyPtr)), # steals + (:PyList_Insert, Cint, (PyPtr, Cssize_t, PyPtr)), + (:PyList_Append, Cint, (PyPtr, PyPtr)), + (:PyList_Reverse, Cint, (PyPtr,)), + # tuple + (:PyTuple_New, PyPtr, (Cssize_t,)), + (:PyTuple_Size, Cssize_t, (PyPtr,)), + (:PyTuple_SetItem, Cint, (PyPtr, Cssize_t, PyPtr)), # steals + (:PyTuple_GetItem, PyPtr, (PyPtr, Cssize_t)), + # dict + (:PyDict_New, PyPtr, ()), + (:PyDict_SetItem, Cint, (PyPtr, PyPtr, PyPtr)), + # slice + (:PySlice_New, PyPtr, (PyPtr, PyPtr, PyPtr)), +] + +for (name, rettype, argtypes) in CPYTHON_FUNCTIONS + cname, jname = name isa Pair ? name : (name, name) + jname = Symbol(:C, jname) + args = [Symbol(:_,i) for i in 1:length(argtypes)] + cnamesym = QuoteNode(cname) + @eval @inline function $jname($(args...),) :: $rettype + ccall(@pysym($cnamesym), $rettype, ($(argtypes...),), $(args...)) + end +end + +Base.unsafe_convert(::Type{PyPtr}, x::Ref{CPyTypeObject}) = + convert(PyPtr, Base.unsafe_convert(Ptr{CPyTypeObject}, x)) diff --git a/src/libpython/globals.jl b/src/libpython/globals.jl new file mode 100644 index 00000000..47a36c76 --- /dev/null +++ b/src/libpython/globals.jl @@ -0,0 +1,80 @@ +CPYTHON_OBJECTS = [ + # types + :PyType_Type, # builtin 'type' + :PyBaseObject_Type, # builtin 'object' + :PySuper_Type, # builtin 'super' + :PyLong_Type, # builtin 'int' + :PyUnicode_Type, # builtin 'str' + :PyTuple_Type, # builtin 'tuple' + :PyList_Type, # builtin 'list' + :PyBool_Type, # bulitin 'bool' + :PyFloat_Type, # builtin 'float' + :PyComplex_Type, # builtin 'complex' + :PyDict_Type, # builtin 'dict' + :PySlice_Type, # builtin 'slice' + :PyRange_Type, # builtin 'range' ('xrange' in python 2) + # objects + :_Py_NoneStruct => :Py_None, + :_Py_TrueStruct => :Py_True, + :_Py_FalseStruct => :Py_False, + :_Py_EllipsisObject => :Py_Ellipsis, + :_Py_NotImplementedStruct => :Py_NotImplemented, +] + +CPYTHON_OBJECT_POINTERS = [ + # exception types + :PyExc_ArithmeticError, + :PyExc_AttributeError, + :PyExc_EOFError, + :PyExc_ImportError, + :PyExc_IndexError, + :PyExc_IOError, + :PyExc_KeyboardInterrupt, + :PyExc_KeyError, + :PyExc_MemoryError, + :PyExc_OverflowError, + :PyExc_RuntimeError, + :PyExc_SystemError, + :PyExc_SyntaxError, + :PyExc_TypeError, + :PyExc_ValueError, + :PyExc_ZeroDivisionError, +] + +for name in CPYTHON_OBJECTS + jname = Symbol(:C, name isa Pair ? name[2] : name) + jfname = Symbol(jname, :_NewRef) + @eval const $jname = Ref{PyPtr}(C_NULL) + @eval function $jfname() + r = $jname[] + CPy_IncRef(r) + r + end +end + +for name in CPYTHON_OBJECT_POINTERS + jname = Symbol(:C, name isa Pair ? name[2] : name) + jfname = Symbol(jname, :_NewRef) + @eval const $jname = Ref{PyPtr}(C_NULL) + @eval function $jfname() + r = $jname[] + CPy_IncRef(r) + r + end +end + +@eval function capi_init() + $([begin + cname, jname = name isa Pair ? name : name=>name + jname = Symbol(:C, jname) + cnamesym = QuoteNode(cname) + :($jname[] = @pyglobalobj($cnamesym)) + end for name in CPYTHON_OBJECTS]...) + $([begin + cname, jname = name isa Pair ? name : name=>name + jname = Symbol(:C, jname) + cnamesym = QuoteNode(cname) + :($jname[] = @pyglobalobjptr($cnamesym)) + end for name in CPYTHON_OBJECT_POINTERS]...) +end + diff --git a/src/libpython/types.jl b/src/libpython/types.jl new file mode 100644 index 00000000..19a11660 --- /dev/null +++ b/src/libpython/types.jl @@ -0,0 +1,425 @@ +# Mirror of C PyObject struct (for non-debugging Python builds). +# We won't actually access these fields directly; we'll use the Python +# C API for everything. However, we need to define a unique Ptr type +# for PyObject*, and we might as well define the actual struct layout +# while we're at it. +struct CPyObject + ob_refcnt::Int + ob_type::Ptr{Cvoid} +end + +const PyPtr = Ptr{CPyObject} # type for PythonObject* in ccall +const PyPtr_NULL = PyPtr(C_NULL) + +const sizeof_CPyObject_HEAD = sizeof(Int) + sizeof(PyPtr) + +################################################################ +# complex + +struct CPy_complex + real :: Cdouble + imag :: Cdouble +end + +################################################################ +# buffer + +struct CPy_buffer + buf::Ptr{Cvoid} + obj::PyPtr + len::Cssize_t + itemsize::Cssize_t + + readonly::Cint + ndim::Cint + format::Ptr{Cchar} + shape::Ptr{Cssize_t} + strides::Ptr{Cssize_t} + suboffsets::Ptr{Cssize_t} + + # some opaque padding fields to account for differences between + # Python versions (the structure changed in Python 2.7 and 3.3) + internal0::Ptr{Cvoid} + internal1::Ptr{Cvoid} + internal2::Ptr{Cvoid} +end + +const CPy_buffer_NULL = CPy_buffer(C_NULL, C_NULL, 0, 0, 0, 0, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL) + +const PyBUF_SIMPLE = convert(Cint, 0) +const PyBUF_WRITABLE = convert(Cint, 0x0001) +const PyBUF_FORMAT = convert(Cint, 0x0004) +const PyBUF_ND = convert(Cint, 0x0008) +const PyBUF_STRIDES = convert(Cint, 0x0010) | PyBUF_ND +const PyBUF_C_CONTIGUOUS = convert(Cint, 0x0020) | PyBUF_STRIDES +const PyBUF_F_CONTIGUOUS = convert(Cint, 0x0040) | PyBUF_STRIDES +const PyBUF_ANY_CONTIGUOUS = convert(Cint, 0x0080) | PyBUF_STRIDES +const PyBUF_INDIRECT = convert(Cint, 0x0100) | PyBUF_STRIDES +const PyBUF_ND_STRIDED = Cint(PyBUF_WRITABLE | PyBUF_FORMAT | PyBUF_ND | + PyBUF_STRIDES) +const PyBUF_ND_CONTIGUOUS = PyBUF_ND_STRIDED | PyBUF_ANY_CONTIGUOUS + +################################################################ +# mirror of Python API types and constants from methodobject.h + +struct CPyMethodDef + ml_name::Ptr{UInt8} + ml_meth::Ptr{Cvoid} + ml_flags::Cint + ml_doc::Ptr{UInt8} +end + +const CPyMethodDef_NULL = CPyMethodDef(C_NULL, C_NULL, 0, C_NULL) + +# A PyCFunction is a C function of the form +# PyObject *func(PyObject *self, PyObject *args) +# or +# PyObject *func(PyObject *self, PyObject *args, PyObject *kwargs) +# The first parameter is the "self" function for method, or +# for module functions it is the module object. The second +# parameter is either a tuple of args (for METH_VARARGS), +# a single arg (for METH_O), or NULL (for METH_NOARGS). func +# must return non-NULL (Py_None is okay) unless there was an +# error, in which case an exception must have been set. + +# ml_flags should be one of: +const METH_VARARGS = 0x0001 # args are a tuple of arguments +const METH_KEYWORDS = 0x0002 # two arguments: the varargs and the kwargs +const METH_NOARGS = 0x0004 # no arguments (NULL argument pointer) +const METH_O = 0x0008 # single argument (not wrapped in tuple) + +# not sure when these are needed: +const METH_CLASS = 0x0010 # for class methods +const METH_STATIC = 0x0020 # for static methods + +################################################################ +# mirror of Python API types and constants from descrobject.h + +struct CPyGetSetDef + name::Ptr{UInt8} + get::Ptr{Cvoid} + set::Ptr{Cvoid} # may be NULL for read-only members + doc::Ptr{UInt8} # may be NULL + closure::Ptr{Cvoid} # pass-through thunk, may be NULL +end + +const CPyGetSetDef_NULL = CPyGetSetDef(C_NULL, C_NULL, C_NULL, C_NULL, C_NULL) + + +################################################################ +# from Python structmember.h: + +# declare immutable because we need a C-like array of these +struct CPyMemberDef + name::Ptr{UInt8} + typ::Cint + offset::Int # warning: was Cint for Python <= 2.4 + flags::Cint + doc::Ptr{UInt8} +end + +const CPyMemberDef_NULL = CPyMemberDef(C_NULL, 0, 0, 0, C_NULL) + +# types: +const T_SHORT =0 +const T_INT =1 +const T_LONG =2 +const T_FLOAT =3 +const T_DOUBLE =4 +const T_STRING =5 +const T_OBJECT =6 +const T_CHAR =7 +const T_BYTE =8 +const T_UBYTE =9 +const T_USHORT =10 +const T_UINT =11 +const T_ULONG =12 +const T_STRING_INPLACE =13 +const T_BOOL =14 +const T_OBJECT_EX =16 +const T_LONGLONG =17 # added in Python 2.5 +const T_ULONGLONG =18 # added in Python 2.5 +const T_PYSSIZET =19 # added in Python 2.6 +const T_NONE =20 # added in Python 3.0 + +# flags: +const READONLY = 1 +const READ_RESTRICTED = 2 +const PY_WRITE_RESTRICTED = 4 +const RESTRICTED = (READ_RESTRICTED | PY_WRITE_RESTRICTED) + +################################################################ +# type-flag constants, from Python object.h: + +# Python 2.7 +const Py_TPFLAGS_HAVE_GETCHARBUFFER = (0x00000001<<0) +const Py_TPFLAGS_HAVE_SEQUENCE_IN = (0x00000001<<1) +const Py_TPFLAGS_GC = 0 # was sometimes (0x00000001<<2) in Python <= 2.1 +const Py_TPFLAGS_HAVE_INPLACEOPS = (0x00000001<<3) +const Py_TPFLAGS_CHECKTYPES = (0x00000001<<4) +const Py_TPFLAGS_HAVE_RICHCOMPARE = (0x00000001<<5) +const Py_TPFLAGS_HAVE_WEAKREFS = (0x00000001<<6) +const Py_TPFLAGS_HAVE_ITER = (0x00000001<<7) +const Py_TPFLAGS_HAVE_CLASS = (0x00000001<<8) +const Py_TPFLAGS_HAVE_INDEX = (0x00000001<<17) +const Py_TPFLAGS_HAVE_NEWBUFFER = (0x00000001<<21) +const Py_TPFLAGS_STRING_SUBCLASS = (0x00000001<<27) + +# Python 3.0+ has only these: +const Py_TPFLAGS_HEAPTYPE = (0x00000001<<9) +const Py_TPFLAGS_BASETYPE = (0x00000001<<10) +const Py_TPFLAGS_READY = (0x00000001<<12) +const Py_TPFLAGS_READYING = (0x00000001<<13) +const Py_TPFLAGS_HAVE_GC = (0x00000001<<14) +const Py_TPFLAGS_HAVE_VERSION_TAG = (0x00000001<<18) +const Py_TPFLAGS_VALID_VERSION_TAG = (0x00000001<<19) +const Py_TPFLAGS_IS_ABSTRACT = (0x00000001<<20) +const Py_TPFLAGS_INT_SUBCLASS = (0x00000001<<23) +const Py_TPFLAGS_LONG_SUBCLASS = (0x00000001<<24) +const Py_TPFLAGS_LIST_SUBCLASS = (0x00000001<<25) +const Py_TPFLAGS_TUPLE_SUBCLASS = (0x00000001<<26) +const Py_TPFLAGS_BYTES_SUBCLASS = (0x00000001<<27) +const Py_TPFLAGS_UNICODE_SUBCLASS = (0x00000001<<28) +const Py_TPFLAGS_DICT_SUBCLASS = (0x00000001<<29) +const Py_TPFLAGS_BASE_EXC_SUBCLASS = (0x00000001<<30) +const Py_TPFLAGS_TYPE_SUBCLASS = (0x00000001<<31) + +# only use this if we have the stackless extension +const Py_TPFLAGS_HAVE_STACKLESS_EXTENSION = (0x00000003<<15) + +################################################################ +# Mirror of PyNumberMethods in Python object.h + +struct CPyNumberMethods + nb_add::Ptr{Cvoid} + nb_subtract::Ptr{Cvoid} + nb_multiply::Ptr{Cvoid} + nb_remainder::Ptr{Cvoid} + nb_divmod::Ptr{Cvoid} + nb_power::Ptr{Cvoid} + nb_negative::Ptr{Cvoid} + nb_positive::Ptr{Cvoid} + nb_absolute::Ptr{Cvoid} + nb_bool::Ptr{Cvoid} + nb_invert::Ptr{Cvoid} + nb_lshift::Ptr{Cvoid} + nb_rshift::Ptr{Cvoid} + nb_and::Ptr{Cvoid} + nb_xor::Ptr{Cvoid} + nb_or::Ptr{Cvoid} + nb_int::Ptr{Cvoid} + nb_reserved::Ptr{Cvoid} + nb_float::Ptr{Cvoid} + nb_inplace_add::Ptr{Cvoid} + nb_inplace_subtract::Ptr{Cvoid} + nb_inplace_multiply::Ptr{Cvoid} + nb_inplace_remainder::Ptr{Cvoid} + nb_inplace_power::Ptr{Cvoid} + nb_inplace_lshift::Ptr{Cvoid} + nb_inplace_rshift::Ptr{Cvoid} + nb_inplace_and::Ptr{Cvoid} + nb_inplace_xor::Ptr{Cvoid} + nb_inplace_or::Ptr{Cvoid} + nb_floordivide::Ptr{Cvoid} + nb_truedivide::Ptr{Cvoid} + nb_inplace_floordivide::Ptr{Cvoid} + nb_inplace_truedivide::Ptr{Cvoid} + nb_index::Ptr{Cvoid} + nb_matrixmultiply::Ptr{Cvoid} + nb_imatrixmultiply::Ptr{Cvoid} +end + +const CPyNumberMethods_NULL = CPyNumberMethods(C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL) + +################################################################ +# Mirror of PySequenceMethods in Python object.h + +struct CPySequenceMethods + sq_length::Ptr{Cvoid} + sq_concat::Ptr{Cvoid} + sq_repeat::Ptr{Cvoid} + sq_item::Ptr{Cvoid} + was_sq_item::Ptr{Cvoid} + sq_ass_item::Ptr{Cvoid} + was_sq_ass_slice::Ptr{Cvoid} + sq_contains::Ptr{Cvoid} + sq_inplace_concat::Ptr{Cvoid} + sq_inplace_repeat::Ptr{Cvoid} +end + +const CPySequenceMethods_NULL = CPySequenceMethods(C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL) + +################################################################ +# Mirror of PyMappingMethods in Python object.h + +struct CPyMappingMethods + mp_length::Ptr{Cvoid} + mp_subscript::Ptr{Cvoid} + mp_ass_subscript::Ptr{Cvoid} +end + +const CPyMappingMethods_NULL = CPyMappingMethods(C_NULL, C_NULL, C_NULL) + +################################################################ +# Mirror of PyTypeObject in Python object.h +# -- assumes non-debugging Python build (no Py_TRACE_REFS) +# -- most fields can default to 0 except where noted + +struct CPyTypeObject + # PyObject_HEAD (for non-Py_TRACE_REFS build): + ob_refcnt::Int + ob_type::PyPtr + ob_size::Int # PyObject_VAR_HEAD, C_NULL + + # PyTypeObject fields: + tp_name::Ptr{UInt8} # required, should be in format "." + + # warning: these two were Cint for Python <= 2.4 + tp_basicsize::Int # required, = sizeof(instance) + tp_itemsize::Int + + tp_dealloc::Ptr{Cvoid} + tp_print::Ptr{Cvoid} + tp_getattr::Ptr{Cvoid} + tp_setattr::Ptr{Cvoid} + tp_compare::Ptr{Cvoid} + tp_repr::Ptr{Cvoid} + + tp_as_number::Ptr{CPyNumberMethods} + tp_as_sequence::Ptr{CPySequenceMethods} + tp_as_mapping::Ptr{CPyMappingMethods} + + tp_hash::Ptr{Cvoid} + tp_call::Ptr{Cvoid} + tp_str::Ptr{Cvoid} + tp_getattro::Ptr{Cvoid} + tp_setattro::Ptr{Cvoid} + + tp_as_buffer::Ptr{Cvoid} + + tp_flags::Clong # Required, should default to Py_TPFLAGS_DEFAULT + + tp_doc::Ptr{UInt8} # normally set in example code, but may be NULL + + tp_traverse::Ptr{Cvoid} + + tp_clear::Ptr{Cvoid} + + tp_richcompare::Ptr{Cvoid} + + tp_weaklistoffset::Int + + # added in Python 2.2: + tp_iter::Ptr{Cvoid} + tp_iternext::Ptr{Cvoid} + + tp_methods::Ptr{CPyMethodDef} + tp_members::Ptr{CPyMemberDef} + tp_getset::Ptr{CPyGetSetDef} + tp_base::Ptr{Cvoid} + + tp_dict::PyPtr + tp_descr_get::Ptr{Cvoid} + tp_descr_set::Ptr{Cvoid} + tp_dictoffset::Int + + tp_init::Ptr{Cvoid} + tp_alloc::Ptr{Cvoid} + tp_new::Ptr{Cvoid} + tp_free::Ptr{Cvoid} + tp_is_gc::Ptr{Cvoid} + + tp_bases::PyPtr + tp_mro::PyPtr + tp_cache::PyPtr + tp_subclasses::PyPtr + tp_weaklist::PyPtr + tp_del::Ptr{Cvoid} + + # added in Python 2.6: + tp_version_tag::Cuint + + # only used for COUNT_ALLOCS builds of Python + tp_allocs::Int + tp_frees::Int + tp_maxalloc::Int + tp_prev::Ptr{Cvoid} + tp_next::Ptr{Cvoid} +end + +const CPyTypeObject_NULL = CPyTypeObject(0, C_NULL, 0, C_NULL, 0, 0, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, 0, C_NULL, C_NULL, C_NULL, C_NULL, 0, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, 0, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, 0, 0, 0, 0, C_NULL, C_NULL) + +################################################################ +# datetime + +struct CPyDateTime_CAPI + # type objects: + DateType::PyPtr + DateTimeType::PyPtr + TimeType::PyPtr + DeltaType::PyPtr + TZInfoType::PyPtr + + # singletons: + @static if pyversion >= v"3.7" + TimeZone_UTC::PyPtr + end + + # function pointers: + Date_FromDate::Ptr{Cvoid} + DateTime_FromDateAndTime::Ptr{Cvoid} + Time_FromTime::Ptr{Cvoid} + Delta_FromDelta::Ptr{Cvoid} + @static if pyversion >= v"3.7" + TimeZone_FromTimeZone::Ptr{Cvoid} + end + DateTime_FromTimestamp::Ptr{Cvoid} + Date_FromTimestamp::Ptr{Cvoid} +end + +struct CPyDateTime_Delta + # PyObject_HEAD (for non-Py_TRACE_REFS build): + ob_refcnt::Int + ob_type::PyPtr + hashcode::Py_hash_t + days::Cint + seconds::Cint + microseconds::Cint +end + +struct CPyDateTime_Date + ob_refcnt::Int + ob_type::PyPtr + hashcode::Py_hash_t + hastzinfo::Cchar + data::NTuple{4,Cuchar} +end + +struct CPyDateTime_Time + ob_refcnt::Int + ob_type::PyPtr + hashcode::Py_hash_t + hastzinfo::Cchar + data::NTuple{6,Cuchar} + fold::Cuchar + tzinfo::PyPtr +end + +struct CPyDateTime_BaseDateTime + ob_refcnt::Int + ob_type::PyPtr + hashcode::Py_hash_t + hastzinfo::Cchar + data::NTuple{10,Cuchar} +end + +struct CPyDateTime_DateTime + ob_refcnt::Int + ob_type::PyPtr + hashcode::Py_hash_t + hastzinfo::Cchar + data::NTuple{10,Cuchar} + fold::Cuchar + tzinfo::PyPtr +end + diff --git a/src/numpy.jl b/src/numpy.jl index f3d523aa..efdb27aa 100644 --- a/src/numpy.jl +++ b/src/numpy.jl @@ -186,13 +186,13 @@ function NpyArray(a::StridedArray{T}, revdims::Bool) where T<:PYARR_TYPES return PyObject(p, a) end -function PyObject(a::StridedArray{T}) where T<:PYARR_TYPES - try - return NpyArray(a, false) - catch - return array2py(a) # fallback to non-NumPy version - end -end +# function PyObject(a::StridedArray{T}) where T<:PYARR_TYPES +# try +# return NpyArray(a, false) +# catch +# return array2py(a) # fallback to non-NumPy version +# end +# end function PyReverseDims(a::StridedArray{T,N}) where {T<:PYARR_TYPES,N} try @@ -220,8 +220,8 @@ PyReverseDims(a::AbstractArray) ######################################################################### -# transposed arrays can be passed to NumPy without copying -PyObject(a::Union{LinearAlgebra.Adjoint{<:Real},LinearAlgebra.Transpose}) = - PyReverseDims(a.parent) +# # transposed arrays can be passed to NumPy without copying +# PyObject(a::Union{LinearAlgebra.Adjoint{<:Real},LinearAlgebra.Transpose}) = +# PyReverseDims(a.parent) -PyObject(a::LinearAlgebra.Adjoint) = PyObject(Matrix(a)) # non-real arrays require a copy +# PyObject(a::LinearAlgebra.Adjoint) = PyObject(Matrix(a)) # non-real arrays require a copy diff --git a/src/pybuffer.jl b/src/pybuffer.jl index a2cec72d..d83de496 100644 --- a/src/pybuffer.jl +++ b/src/pybuffer.jl @@ -4,32 +4,12 @@ # (thanks to @jakebolewski for his work on this) ############################################################################# -# mirror of Py_buffer struct in Python Include/object.h - -struct Py_buffer - buf::Ptr{Cvoid} - obj::PyPtr - len::Cssize_t - itemsize::Cssize_t - - readonly::Cint - ndim::Cint - format::Ptr{Cchar} - shape::Ptr{Cssize_t} - strides::Ptr{Cssize_t} - suboffsets::Ptr{Cssize_t} - - # some opaque padding fields to account for differences between - # Python versions (the structure changed in Python 2.7 and 3.3) - internal0::Ptr{Cvoid} - internal1::Ptr{Cvoid} - internal2::Ptr{Cvoid} -end +# mirror of CPy_buffer struct in Python Include/object.h mutable struct PyBuffer - buf::Py_buffer + buf::CPy_buffer PyBuffer() = begin - b = new(Py_buffer(C_NULL, PyPtr_NULL, 0, 0, + b = new(CPy_buffer(C_NULL, PyPtr_NULL, 0, 0, 0, 0, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL)) finalizer(pydecref, b) @@ -99,21 +79,6 @@ iscontiguous(b::PyBuffer) = 1 == ccall((@pysym :PyBuffer_IsContiguous), Cint, (Ref{PyBuffer}, Cchar), b, 'A') -############################################################################# -# pybuffer constant values from Include/object.h -const PyBUF_SIMPLE = convert(Cint, 0) -const PyBUF_WRITABLE = convert(Cint, 0x0001) -const PyBUF_FORMAT = convert(Cint, 0x0004) -const PyBUF_ND = convert(Cint, 0x0008) -const PyBUF_STRIDES = convert(Cint, 0x0010) | PyBUF_ND -const PyBUF_C_CONTIGUOUS = convert(Cint, 0x0020) | PyBUF_STRIDES -const PyBUF_F_CONTIGUOUS = convert(Cint, 0x0040) | PyBUF_STRIDES -const PyBUF_ANY_CONTIGUOUS = convert(Cint, 0x0080) | PyBUF_STRIDES -const PyBUF_INDIRECT = convert(Cint, 0x0100) | PyBUF_STRIDES -const PyBUF_ND_STRIDED = Cint(PyBUF_WRITABLE | PyBUF_FORMAT | PyBUF_ND | - PyBUF_STRIDES) -const PyBUF_ND_CONTIGUOUS = PyBUF_ND_STRIDED | PyBUF_ANY_CONTIGUOUS - # construct a PyBuffer from a PyObject, if possible function PyBuffer(o::Union{PyObject,PyPtr}, flags=PyBUF_SIMPLE) return PyBuffer!(PyBuffer(), o, flags) diff --git a/src/pydates.jl b/src/pydates.jl index 14940956..8321700a 100644 --- a/src/pydates.jl +++ b/src/pydates.jl @@ -76,46 +76,46 @@ function init_datetime() Delta_FromDelta[] = PyDateTimeAPI.Delta_FromDelta end -PyObject(d::Dates.Date) = - PyObject(@pycheckn ccall(Date_FromDate[], PyPtr, - (Cint, Cint, Cint, PyPtr), - Dates.year(d), Dates.month(d), Dates.day(d), - DateType[])) - -PyObject(d::Dates.DateTime) = - PyObject(@pycheckn ccall(DateTime_FromDateAndTime[], PyPtr, - (Cint, Cint, Cint, Cint, Cint, Cint, Cint, - PyPtr, PyPtr), - Dates.year(d), Dates.month(d), Dates.day(d), - Dates.hour(d), Dates.minute(d), Dates.second(d), - Dates.millisecond(d) * 1000, - pynothing[], DateTimeType[])) - -PyDelta_FromDSU(days, seconds, useconds) = - PyObject(@pycheckn ccall(Delta_FromDelta[], PyPtr, - (Cint, Cint, Cint, Cint, PyPtr), - days, seconds, useconds, - 1, DeltaType[])) - -PyObject(p::Dates.Day) = PyDelta_FromDSU(Dates.value(p), 0, 0) - -function PyObject(p::Dates.Second) - # normalize to make Cint overflow less likely - s = Dates.value(p) - d = div(s, 86400) - s -= d * 86400 - PyDelta_FromDSU(d, s, 0) -end - -function PyObject(p::Dates.Millisecond) - # normalize to make Cint overflow less likely - ms = Dates.value(p) - s = div(ms, 1000) - ms -= s * 1000 - d = div(s, 86400) - s -= d * 86400 - PyDelta_FromDSU(d, s, ms * 1000) -end +# PyObject(d::Dates.Date) = +# PyObject(@pycheckn ccall(Date_FromDate[], PyPtr, +# (Cint, Cint, Cint, PyPtr), +# Dates.year(d), Dates.month(d), Dates.day(d), +# DateType[])) + +# PyObject(d::Dates.DateTime) = +# PyObject(@pycheckn ccall(DateTime_FromDateAndTime[], PyPtr, +# (Cint, Cint, Cint, Cint, Cint, Cint, Cint, +# PyPtr, PyPtr), +# Dates.year(d), Dates.month(d), Dates.day(d), +# Dates.hour(d), Dates.minute(d), Dates.second(d), +# Dates.millisecond(d) * 1000, +# pynothing[], DateTimeType[])) + +# PyDelta_FromDSU(days, seconds, useconds) = +# PyObject(@pycheckn ccall(Delta_FromDelta[], PyPtr, +# (Cint, Cint, Cint, Cint, PyPtr), +# days, seconds, useconds, +# 1, DeltaType[])) + +# PyObject(p::Dates.Day) = PyDelta_FromDSU(Dates.value(p), 0, 0) + +# function PyObject(p::Dates.Second) +# # normalize to make Cint overflow less likely +# s = Dates.value(p) +# d = div(s, 86400) +# s -= d * 86400 +# PyDelta_FromDSU(d, s, 0) +# end + +# function PyObject(p::Dates.Millisecond) +# # normalize to make Cint overflow less likely +# ms = Dates.value(p) +# s = div(ms, 1000) +# ms -= s * 1000 +# d = div(s, 86400) +# s -= d * 86400 +# PyDelta_FromDSU(d, s, ms * 1000) +# end PyDate_Check(o::PyObject) = pyisinstance(o, DateType[]) PyDateTime_Check(o::PyObject) = pyisinstance(o, DateTimeType[]) @@ -131,37 +131,37 @@ function pydate_query(o::PyObject) end end -function convert(::Type{Dates.DateTime}, o::PyObject) - if PyDate_Check(o) - GC.@preserve o let dt = convert(Ptr{UInt8}, PyPtr(o)) + PyDate_HEAD - if PyDateTime_Check(o) - Dates.DateTime((UInt(unsafe_load(dt,1))<<8)|unsafe_load(dt,2), # Y - unsafe_load(dt,3), unsafe_load(dt,4), # month, day - unsafe_load(dt,5), unsafe_load(dt,6), # hour, minute - unsafe_load(dt,7), # second - div((UInt(unsafe_load(dt,8)) << 16) | - (UInt(unsafe_load(dt,9)) << 8) | - unsafe_load(dt,10), 1000)) # μs ÷ 1000 - else - Dates.DateTime((UInt(unsafe_load(dt,1))<<8)|unsafe_load(dt,2), # Y - unsafe_load(dt,3), unsafe_load(dt,4)) # month, day - end - end - else - throw(ArgumentError("unknown DateTime type $o")) - end -end - -function convert(::Type{Dates.Date}, o::PyObject) - if PyDate_Check(o) - GC.@preserve o let dt = convert(Ptr{UInt8}, PyPtr(o)) + PyDate_HEAD - Dates.Date((UInt(unsafe_load(dt,1)) << 8) | unsafe_load(dt,2), # Y - unsafe_load(dt,3), unsafe_load(dt,4)) # month, day - end - else - throw(ArgumentError("unknown Date type $o")) - end -end +# function convert(::Type{Dates.DateTime}, o::PyObject) +# if PyDate_Check(o) +# GC.@preserve o let dt = convert(Ptr{UInt8}, PyPtr(o)) + PyDate_HEAD +# if PyDateTime_Check(o) +# Dates.DateTime((UInt(unsafe_load(dt,1))<<8)|unsafe_load(dt,2), # Y +# unsafe_load(dt,3), unsafe_load(dt,4), # month, day +# unsafe_load(dt,5), unsafe_load(dt,6), # hour, minute +# unsafe_load(dt,7), # second +# div((UInt(unsafe_load(dt,8)) << 16) | +# (UInt(unsafe_load(dt,9)) << 8) | +# unsafe_load(dt,10), 1000)) # μs ÷ 1000 +# else +# Dates.DateTime((UInt(unsafe_load(dt,1))<<8)|unsafe_load(dt,2), # Y +# unsafe_load(dt,3), unsafe_load(dt,4)) # month, day +# end +# end +# else +# throw(ArgumentError("unknown DateTime type $o")) +# end +# end + +# function convert(::Type{Dates.Date}, o::PyObject) +# if PyDate_Check(o) +# GC.@preserve o let dt = convert(Ptr{UInt8}, PyPtr(o)) + PyDate_HEAD +# Dates.Date((UInt(unsafe_load(dt,1)) << 8) | unsafe_load(dt,2), # Y +# unsafe_load(dt,3), unsafe_load(dt,4)) # month, day +# end +# else +# throw(ArgumentError("unknown Date type $o")) +# end +# end function delta_dsμ(o::PyObject) PyDelta_Check(o) || throw(ArgumentError("$o is not a timedelta instance")) @@ -173,17 +173,17 @@ end # that is not an exact multiple of the resulting unit? For now, # follow the lead of Dates and truncate; see Julia issue #9169. -function convert(::Type{Dates.Millisecond}, o::PyObject) - (d,s,μs) = delta_dsμ(o) - return Dates.Millisecond((86400d + s) * 1000 + div(μs, 1000)) -end +# function convert(::Type{Dates.Millisecond}, o::PyObject) +# (d,s,μs) = delta_dsμ(o) +# return Dates.Millisecond((86400d + s) * 1000 + div(μs, 1000)) +# end -function convert(::Type{Dates.Second}, o::PyObject) - (d,s,μs) = delta_dsμ(o) - return Dates.Second(86400d + s + div(μs, 1000000)) -end +# function convert(::Type{Dates.Second}, o::PyObject) +# (d,s,μs) = delta_dsμ(o) +# return Dates.Second(86400d + s + div(μs, 1000000)) +# end -function convert(::Type{Dates.Day}, o::PyObject) - (d,s,μs) = delta_dsμ(o) - return Dates.Day(d + div(s + div(μs, 1000000), 86400)) -end +# function convert(::Type{Dates.Day}, o::PyObject) +# (d,s,μs) = delta_dsμ(o) +# return Dates.Day(d + div(s + div(μs, 1000000), 86400)) +# end diff --git a/src/pydict.jl b/src/pydict.jl new file mode 100644 index 00000000..f17cc075 --- /dev/null +++ b/src/pydict.jl @@ -0,0 +1,153 @@ +######################################################################### +# PyDict: no-copy wrapping of a Julia object around a Python dictionary + +# we check for "items" attr since PyMapping_Check doesn't do this (it only +# checks for __getitem__) and PyMapping_Check returns true for some +# scipy scalar array members, grrr. +function is_mapping_object(o::PyObject) + pyisinstance(o, @pyglobalobj :PyDict_Type) || + (pyquery((@pyglobal :PyMapping_Check), o) && + ccall((@pysym :PyObject_HasAttrString), Cint, (PyPtr,Ptr{UInt8}), o, "items") == 1) +end + +""" + PyDict(o::PyObject) + PyDict(d::Dict{K,V}) + +This returns a PyDict, which is a no-copy wrapper around a Python dictionary. + +Alternatively, you can specify the return type of a `pycall` as PyDict. +""" +mutable struct PyDict{K,V,isdict} <: AbstractDict{K,V} + o::PyObject + # isdict = true for python dict, otherwise is a generic Mapping object + + function PyDict{K,V,isdict}(o::PyObject) where {K,V,isdict} + if !isdict && !ispynull(o) && !is_mapping_object(o) + throw(ArgumentError("only Dict and Mapping objects can be converted to PyDict")) + end + return new{K,V,isdict}(o) + end +end + +PyDict{K,V}(o::PyObject) where {K,V} = PyDict{K,V,pyisinstance(o, @pyglobalobj :PyDict_Type)}(o) +PyDict{K,V}() where {K,V} = PyDict{K,V,true}(PyObject(@pycheckn ccall((@pysym :PyDict_New), PyPtr, ()))) + +PyDict(o::PyObject) = PyDict{PyAny,PyAny}(o) +PyObject(d::PyDict) = d.o +PyDict() = PyDict{PyAny,PyAny}() +PyDict(d::AbstractDict{K,V}) where {K,V} = PyDict{K,V}(PyObject(d)) +PyDict(d::AbstractDict{Any,Any}) = PyDict{PyAny,PyAny}(PyObject(d)) +PyDict(d::AbstractDict{Any,V}) where {V} = PyDict{PyAny,V}(PyObject(d)) +PyDict(d::AbstractDict{K,Any}) where {K} = PyDict{K,PyAny}(PyObject(d)) +convert(::Type{PyDict}, o::PyObject) = PyDict(o) +convert(::Type{PyDict{K,V}}, o::PyObject) where {K,V} = PyDict{K,V}(o) +unsafe_convert(::Type{PyPtr}, d::PyDict) = PyPtr(d.o) + +haskey(d::PyDict{K,V,true}, key) where {K,V} = 1 == ccall(@pysym(:PyDict_Contains), Cint, (PyPtr, PyPtr), d, PyObject(key)) +keys(::Type{T}, d::PyDict{K,V,true}) where {T,K,V} = convert(Vector{T}, PyObject(@pycheckn ccall((@pysym :PyDict_Keys), PyPtr, (PyPtr,), d))) +values(::Type{T}, d::PyDict{K,V,true}) where {T,K,V} = convert(Vector{T}, PyObject(@pycheckn ccall((@pysym :PyDict_Values), PyPtr, (PyPtr,), d))) + +keys(::Type{T}, d::PyDict{K,V,false}) where {T,K,V} = convert(Vector{T}, pycall(d.o["keys"], PyObject)) +values(::Type{T}, d::PyDict{K,V,false}) where {T,K,V} = convert(Vector{T}, pycall(d.o["values"], PyObject)) +haskey(d::PyDict{K,V,false}, key) where {K,V} = 1 == ccall(@pysym(:PyMapping_HasKey), Cint, (PyPtr, PyPtr), d, PyObject(key)) + +similar(d::PyDict{K,V}) where {K,V} = Dict{pyany_toany(K),pyany_toany(V)}() +eltype(::Type{PyDict{K,V}}) where {K,V} = Pair{pyany_toany(K),pyany_toany(V)} +Base.keytype(::PyDict{K,V}) where {K,V} = pyany_toany(K) +Base.valtype(::PyDict{K,V}) where {K,V} = pyany_toany(V) +Base.keytype(::Type{PyDict{K,V}}) where {K,V} = pyany_toany(K) +Base.valtype(::Type{PyDict{K,V}}) where {K,V} = pyany_toany(V) + +function setindex!(d::PyDict, v, k) + @pycheckz ccall((@pysym :PyObject_SetItem), Cint, (PyPtr, PyPtr, PyPtr), + d, PyObject(k), PyObject(v)) + v +end + +get(d::PyDict{K,V}, k, default) where {K,V} = get(d.o, V, k, default) + +function pop!(d::PyDict{K,V,true}, k) where {K,V} + v = d[k] + @pycheckz ccall(@pysym(:PyDict_DelItem), Cint, (PyPtr, PyPtr), d, PyObject(k)) + return v +end +function pop!(d::PyDict{K,V,false}, k) where {K,V} + v = d[k] + @pycheckz ccall(@pysym(:PyObject_DelItem), Cint, (PyPtr, PyPtr), d, PyObject(k)) + return v +end + +function pop!(d::PyDict, k, default) + try + return pop!(d, k) + catch + return default + end +end + +function delete!(d::PyDict{K,V,true}, k) where {K,V} + e = ccall(@pysym(:PyDict_DelItem), Cint, (PyPtr, PyPtr), d, PyObject(k)) + e == -1 && pyerr_clear() # delete! ignores errors in Julia + return d +end +function delete!(d::PyDict{K,V,false}, k) where {K,V} + e = ccall(@pysym(:PyObject_DelItem), Cint, (PyPtr, PyPtr), d, PyObject(k)) + e == -1 && pyerr_clear() # delete! ignores errors in Julia + return d +end + +function empty!(d::PyDict{K,V,true}) where {K,V} + @pycheck ccall((@pysym :PyDict_Clear), Cvoid, (PyPtr,), d) + return d +end +function empty!(d::PyDict{K,V,false}) where {K,V} + # for generic Mapping items we must delete keys one by one + for k in keys(d) + delete!(d, k) + end + return d +end + +length(d::PyDict{K,V,true}) where {K,V} = @pycheckz ccall(@pysym(:PyDict_Size), Int, (PyPtr,), d) +length(d::PyDict{K,V,false}) where {K,V} = @pycheckz ccall(@pysym(:PyObject_Size), Int, (PyPtr,), d) +isempty(d::PyDict) = length(d) == 0 + + +struct PyDict_Iterator + # arrays to pass key, value, and pos pointers to PyDict_Next + ka::Ref{PyPtr} + va::Ref{PyPtr} + pa::Ref{Int} + i::Int # current position in items list (0-based) + len::Int # length of items list +end + +function Base.iterate(d::PyDict{K,V,true}, itr=PyDict_Iterator(Ref{PyPtr}(), Ref{PyPtr}(), Ref(0), 0, length(d))) where {K,V} + itr.i >= itr.len && return nothing + if 0 == ccall((@pysym :PyDict_Next), Cint, + (PyPtr, Ref{Int}, Ref{PyPtr}, Ref{PyPtr}), + d, itr.pa, itr.ka, itr.va) + error("unexpected end of PyDict_Next") + end + ko = pyincref(itr.ka[]) # PyDict_Next returns + vo = pyincref(itr.va[]) # borrowed ref, so incref + (Pair(convert(K,ko), convert(V,vo)), + PyDict_Iterator(itr.ka, itr.va, itr.pa, itr.i+1, itr.len)) +end + +# Iterator for generic mapping, using Python items iterator. +# Our approach is to wrap an iterator over d.o["items"] +# which necessitates including d.o["items"] in the state. +function _start(d::PyDict{K,V,false}) where {K,V} + d_items = pycall(d.o."items", PyObject) + (d_items, iterate(d_items)) +end +function Base.iterate(d::PyDict{K,V,false}, itr=_start(d)) where {K,V} + d_items, iter_result = itr + iter_result === nothing && return nothing + item, state = iter_result + iter_result = iterate(d_items, state) + (item[1] => item[2], (d_items, iter_result)) +end + diff --git a/src/pyinit.jl b/src/pyinit.jl index 84e20245..0d775c14 100644 --- a/src/pyinit.jl +++ b/src/pyinit.jl @@ -24,44 +24,6 @@ const pyxrange = Ref{PyPtr}(0) ######################################################################### # initialize jlWrapType for pytype.jl -function pyjlwrap_init() - # PyMemberDef stores explicit pointers, hence must be initialized at runtime - empty!(pyjlwrap_members) # for AOT - push!(pyjlwrap_members, PyMemberDef(pyjlwrap_membername, - T_PYSSIZET, sizeof_pyjlwrap_head, READONLY, - pyjlwrap_doc), - PyMemberDef(C_NULL,0,0,0,C_NULL)) - - # all cfunctions must be compiled at runtime - pyjlwrap_dealloc_ptr = @cfunction(pyjlwrap_dealloc, Cvoid, (PyPtr,)) - pyjlwrap_repr_ptr = @cfunction(pyjlwrap_repr, PyPtr, (PyPtr,)) - pyjlwrap_hash_ptr = @cfunction(pyjlwrap_hash, UInt, (PyPtr,)) - pyjlwrap_hash32_ptr = @cfunction(pyjlwrap_hash32, UInt32, (PyPtr,)) - pyjlwrap_call_ptr = @cfunction(pyjlwrap_call, PyPtr, (PyPtr,PyPtr,PyPtr)) - pyjlwrap_getattr_ptr = @cfunction(pyjlwrap_getattr, PyPtr, (PyPtr,PyPtr)) - pyjlwrap_getiter_ptr = @cfunction(pyjlwrap_getiter, PyPtr, (PyPtr,)) - - # detect at runtime whether we are using Stackless Python - try - pyimport("stackless") - Py_TPFLAGS_HAVE_STACKLESS_EXTENSION[] = Py_TPFLAGS_HAVE_STACKLESS_EXTENSION_ - catch - end - - PyTypeObject!(jlWrapType, "PyCall.jlwrap", sizeof(Py_jlWrap)) do t::PyTypeObject - t.tp_flags |= Py_TPFLAGS_BASETYPE - t.tp_members = pointer(pyjlwrap_members); - t.tp_dealloc = pyjlwrap_dealloc_ptr - t.tp_repr = pyjlwrap_repr_ptr - t.tp_call = pyjlwrap_call_ptr - t.tp_getattro = pyjlwrap_getattr_ptr - t.tp_iter = pyjlwrap_getiter_ptr - t.tp_hash = sizeof(Py_hash_t) < sizeof(Int) ? - pyjlwrap_hash32_ptr : pyjlwrap_hash_ptr - t.tp_weaklistoffset = fieldoffset(Py_jlWrap, 3) - end -end - ######################################################################### # Virtual environment support @@ -136,7 +98,7 @@ function __init__() empty!(pycall_gc) empty!(pyexc) empty!(pytype_queries) - empty!(permanent_strings) + # empty!(permanent_strings) # sanity check: in Pkg for Julia 0.7+, the location of Conda can change # if e.g. you checkout Conda master, and we'll need to re-build PyCall @@ -204,6 +166,7 @@ function __init__() copy!(builtin, pyimport(pyversion.major < 3 ? "__builtin__" : "builtins")) copy!(pyproperty, pybuiltin(:property)) + capi_init() pyexc_initialize() # mappings from Julia Exception types to Python exceptions # cache Python None -- PyPtr, not PyObject, to prevent it from diff --git a/src/pyiterator.jl b/src/pyiterator.jl index 85c909aa..f4bfd589 100644 --- a/src/pyiterator.jl +++ b/src/pyiterator.jl @@ -96,55 +96,6 @@ function Base.collect(::Type{T}, o::PyObject) where T end Base.collect(o::PyObject) = collect(Any, o) -######################################################################### -# Iterating over Julia objects in Python - -const jlWrapIteratorType = PyTypeObject() - -# tp_iternext object of a jlwrap_iterator object, similar to PyIter_Next -function pyjlwrap_iternext(self_::PyPtr) - try - iter, iter_result_ref = unsafe_pyjlwrap_to_objref(self_) - iter_result = iter_result_ref[] - if iter_result !== nothing - item, state = iter_result - iter_result_ref[] = iterate(iter, state) - return pyreturn(item) - end - catch e - @pyraise e - end - return PyPtr_NULL -end - -# the tp_iter slot of jlwrap object: like PyObject_GetIter, it -# returns a reference to a new jlwrap_iterator object -function pyjlwrap_getiter(self_::PyPtr) - try - self = unsafe_pyjlwrap_to_objref(self_) - return pystealref!(jlwrap_iterator(self)) - catch e - @pyraise e - end - return PyPtr_NULL -end - -# Given a Julia object o, return a jlwrap_iterator Python iterator object -# that wraps the Julia iteration protocol with the Python iteration protocol. -# Internally, the jlwrap_iterator object stores the tuple (o, Ref(iterate(o))), -# where the Ref is used to store the (element, state)-iterator result tuple -# (which updates during iteration) and also the nothing termination indicator. -function jlwrap_iterator(o::Any) - if jlWrapIteratorType.tp_name == C_NULL # lazily initialize - pyjlwrap_type!(jlWrapIteratorType, "PyCall.jlwrap_iterator") do t - t.tp_iter = @cfunction(pyincref_, PyPtr, (PyPtr,)) # new reference to same object - t.tp_iternext = @cfunction(pyjlwrap_iternext, PyPtr, (PyPtr,)) - end - end - iter_result = iterate(o) - return pyjlwrap_new(jlWrapIteratorType, (o, Ref{Union{Nothing,typeof(iter_result)}}(iterate(o)))) -end - ######################################################################### # Broadcasting: if the object is iterable, return collect(o), and otherwise # return o. diff --git a/src/pyoperators.jl b/src/pyoperators.jl index 6f1c7bf7..49e92b42 100644 --- a/src/pyoperators.jl +++ b/src/pyoperators.jl @@ -58,8 +58,8 @@ for (op,py) in ((:<, Py_LT), (:<=, Py_LE), (:(==), Py_EQ), (:!=, Py_NE), @eval function $op(o1::PyObject, o2::PyObject) if ispynull(o1) || ispynull(o2) return $(py==Py_EQ || py==Py_NE || op==:isless ? :($op(PyPtr(o1), PyPtr(o2))) : false) - elseif is_pyjlwrap(o1) && is_pyjlwrap(o2) - return $op(unsafe_pyjlwrap_to_objref(o1), unsafe_pyjlwrap_to_objref(o2)) + # elseif is_pyjlwrap(o1) && is_pyjlwrap(o2) + # return $op(unsafe_pyjlwrap_load_value(o1), unsafe_pyjlwrap_load_value(o2)) else if $(op == :isless || op == :isequal) return Bool(@pycheckz ccall((@pysym :PyObject_RichCompareBool), Cint, @@ -72,11 +72,11 @@ for (op,py) in ((:<, Py_LT), (:<=, Py_LE), (:(==), Py_EQ), (:!=, Py_NE), end if op != :isequal @eval begin - $op(o1::PyObject, o2::Any) = $op(o1, PyObject(o2)) - $op(o1::Any, o2::PyObject) = $op(PyObject(o1), o2) + $op(o1::PyObject, o2) = $op(o1, PyObject(o2)) + $op(o1, o2::PyObject) = $op(PyObject(o1), o2) end end end # default to false since hash(x) != hash(PyObject(x)) in general -isequal(o1::PyObject, o2::Any) = !ispynull(o1) && is_pyjlwrap(o1) ? isequal(unsafe_pyjlwrap_to_objref(o1), o2) : false -isequal(o1::Any, o2::PyObject) = isequal(o2, o1) +isequal(o1::PyObject, o2) = false +isequal(o1, o2::PyObject) = false diff --git a/src/pytype.jl b/src/pytype.jl index e2df1959..a0191e11 100644 --- a/src/pytype.jl +++ b/src/pytype.jl @@ -4,54 +4,35 @@ # Python expects the PyMethodDef and similar strings to be constants, # so we define anonymous globals to hold them, returning the pointer const permanent_strings = String[] -function gstring_ptr(name::AbstractString, s::AbstractString) +function gstring_ptr(s::AbstractString) g = String(s) push!(permanent_strings, g) unsafe_convert(Ptr{UInt8}, g) end +gstring_ptr(s::Ptr) = convert(Ptr{UInt8}, s) + +gstring_ptr_ornull(s::AbstractString) = + isempty(s) ? NULL_UInt8_Ptr : gstring_ptr(s) +gstring_ptr_ornull(s::Ptr) = gstring_ptr(s) ################################################################ # mirror of Python API types and constants from methodobject.h + + +pymethoddef(name=C_NULL, meth=C_NULL, flags=0) + struct PyMethodDef ml_name::Ptr{UInt8} ml_meth::Ptr{Cvoid} ml_flags::Cint - ml_doc::Ptr{UInt8} # may be NULL + ml_doc::Ptr{UInt8} + function PyMethodDef(name=C_NULL, meth=C_NULL, flags=0, doc=C_NULL) + new(gstring_ptr(name), convert(Ptr{Cvoid}, meth), convert(Cint, flags), gstring_ptr_ornull(doc)) + end end -# A PyCFunction is a C function of the form -# PyObject *func(PyObject *self, PyObject *args) -# or -# PyObject *func(PyObject *self, PyObject *args, PyObject *kwargs) -# The first parameter is the "self" function for method, or -# for module functions it is the module object. The second -# parameter is either a tuple of args (for METH_VARARGS), -# a single arg (for METH_O), or NULL (for METH_NOARGS). func -# must return non-NULL (Py_None is okay) unless there was an -# error, in which case an exception must have been set. - -# ml_flags should be one of: -const METH_VARARGS = 0x0001 # args are a tuple of arguments -const METH_KEYWORDS = 0x0002 # two arguments: the varargs and the kwargs -const METH_NOARGS = 0x0004 # no arguments (NULL argument pointer) -const METH_O = 0x0008 # single argument (not wrapped in tuple) - -# not sure when these are needed: -const METH_CLASS = 0x0010 # for class methods -const METH_STATIC = 0x0020 # for static methods - const NULL_UInt8_Ptr = convert(Ptr{UInt8}, C_NULL) -function PyMethodDef(name::AbstractString, meth::Ptr{Cvoid}, flags::Integer, doc::AbstractString="") - PyMethodDef(gstring_ptr(name, name), - meth, - convert(Cint, flags), - isempty(doc) ? NULL_UInt8_Ptr : gstring_ptr(name, doc)) -end - -# used as sentinel value to end method arrays: -PyMethodDef() = PyMethodDef(NULL_UInt8_Ptr, C_NULL, - convert(Cint, 0), NULL_UInt8_Ptr) ################################################################ # mirror of Python API types and constants from descrobject.h @@ -62,20 +43,11 @@ struct PyGetSetDef set::Ptr{Cvoid} # may be NULL for read-only members doc::Ptr{UInt8} # may be NULL closure::Ptr{Cvoid} # pass-through thunk, may be NULL + function PyGetSetDef(_name=C_NULL, _get=C_NULL, _set=C_NULL, _doc=C_NULL,_closure=C_NULL; name=_name, get=_get, set=_set, doc=_doc, closure=_closure) + new(gstring_ptr(name), convert(Ptr{Cvoid}, get), convert(Ptr{Cvoid}, set), gstring_ptr_ornull(doc), convert(Ptr{Cvoid}, closure)) + end end -# probably should be changed to macro to avoid interpolating into @cfunction: -# (commented out for now since we aren't actually using it) -# function PyGetSetDef(name::AbstractString, get::Function,set::Function, doc::AbstractString="") -# PyGetSetDef(gstring_ptr(name, name), -# @cfunction($get, PyPtr, (PyPtr,Ptr{Cvoid})), -# @cfunction($set, Int, (PyPtr,PyPtr,Ptr{Cvoid})), -# isempty(doc) ? NULL_UInt8_Ptr : gstring_ptr(name, doc), -# C_NULL) -# end - -# used as sentinel value to end attribute arrays: -PyGetSetDef() = PyGetSetDef(NULL_UInt8_Ptr, C_NULL, C_NULL, NULL_UInt8_Ptr, C_NULL) ################################################################ # from Python structmember.h: @@ -87,374 +59,264 @@ struct PyMemberDef offset::Int # warning: was Cint for Python <= 2.4 flags::Cint doc::Ptr{UInt8} - PyMemberDef(name,typ,offset,flags,doc) = - new(unsafe_convert(Ptr{UInt8},name), + function PyMemberDef(name=C_NULL,typ=0,offset=0,flags=0,doc=C_NULL) + new(gstring_ptr(name), convert(Cint,typ), convert(Int,offset), convert(Cint,flags), - unsafe_convert(Ptr{UInt8},doc)) + gstring_ptr_ornull(doc)) + end +end + +################################################################ +# Mirror of PyNumberMethods in Python object.h + +const PyNumberMethods_fields = [ + (:nb_add, Ptr{Cvoid}, C_NULL), + (:nb_subtract, Ptr{Cvoid}, C_NULL), + (:nb_multiply, Ptr{Cvoid}, C_NULL), + (:nb_remainder, Ptr{Cvoid}, C_NULL), + (:nb_divmod, Ptr{Cvoid}, C_NULL), + (:nb_power, Ptr{Cvoid}, C_NULL), + (:nb_negative, Ptr{Cvoid}, C_NULL), + (:nb_positive, Ptr{Cvoid}, C_NULL), + (:nb_absolute, Ptr{Cvoid}, C_NULL), + (:nb_bool, Ptr{Cvoid}, C_NULL), + (:nb_invert, Ptr{Cvoid}, C_NULL), + (:nb_lshift, Ptr{Cvoid}, C_NULL), + (:nb_rshift, Ptr{Cvoid}, C_NULL), + (:nb_and, Ptr{Cvoid}, C_NULL), + (:nb_xor, Ptr{Cvoid}, C_NULL), + (:nb_or, Ptr{Cvoid}, C_NULL), + (:nb_int, Ptr{Cvoid}, C_NULL), + (:nb_reserved, Ptr{Cvoid}, C_NULL), + (:nb_float, Ptr{Cvoid}, C_NULL), + (:nb_inplace_add, Ptr{Cvoid}, C_NULL), + (:nb_inplace_subtract, Ptr{Cvoid}, C_NULL), + (:nb_inplace_multiply, Ptr{Cvoid}, C_NULL), + (:nb_inplace_remainder, Ptr{Cvoid}, C_NULL), + (:nb_inplace_power, Ptr{Cvoid}, C_NULL), + (:nb_inplace_lshift, Ptr{Cvoid}, C_NULL), + (:nb_inplace_rshift, Ptr{Cvoid}, C_NULL), + (:nb_inplace_and, Ptr{Cvoid}, C_NULL), + (:nb_inplace_xor, Ptr{Cvoid}, C_NULL), + (:nb_inplace_or, Ptr{Cvoid}, C_NULL), + (:nb_floordivide, Ptr{Cvoid}, C_NULL), + (:nb_truedivide, Ptr{Cvoid}, C_NULL), + (:nb_inplace_floordivide, Ptr{Cvoid}, C_NULL), + (:nb_inplace_truedivide, Ptr{Cvoid}, C_NULL), + (:nb_index, Ptr{Cvoid}, C_NULL), + (:nb_matrixmultiply, Ptr{Cvoid}, C_NULL), + (:nb_imatrixmultiply, Ptr{Cvoid}, C_NULL), +] + +@eval struct PyNumberMethods + $([:($n::$t) for (n,t,d) in PyNumberMethods_fields]...) + PyNumberMethods(; $([Expr(:kw, n, d) for (n,t,d) in PyNumberMethods_fields]...)) = + new($([:(convert($t, $n)) for (n,t,d) in PyNumberMethods_fields]...)) +end + + +################################################################ +# Mirror of PySequenceMethods in Python object.h + +const PySequenceMethods_fields = [ + (:sq_length, Ptr{Cvoid}, C_NULL), + (:sq_concat, Ptr{Cvoid}, C_NULL), + (:sq_repeat, Ptr{Cvoid}, C_NULL), + (:sq_item, Ptr{Cvoid}, C_NULL), + (:was_sq_item, Ptr{Cvoid}, C_NULL), + (:sq_ass_item, Ptr{Cvoid}, C_NULL), + (:was_sq_ass_slice, Ptr{Cvoid}, C_NULL), + (:sq_contains, Ptr{Cvoid}, C_NULL), + (:sq_inplace_concat, Ptr{Cvoid}, C_NULL), + (:sq_inplace_repeat, Ptr{Cvoid}, C_NULL), +] + +@eval struct PySequenceMethods + $([:($n :: $t) for (n,t,d) in PySequenceMethods_fields]...) + PySequenceMethods(; $([Expr(:kw, n, d) for (n,t,d) in PySequenceMethods_fields]...)) = + new($([:(convert($t, $n)) for (n,t,d) in PySequenceMethods_fields]...)) end -# types: -const T_SHORT =0 -const T_INT =1 -const T_LONG =2 -const T_FLOAT =3 -const T_DOUBLE =4 -const T_STRING =5 -const T_OBJECT =6 -const T_CHAR =7 -const T_BYTE =8 -const T_UBYTE =9 -const T_USHORT =10 -const T_UINT =11 -const T_ULONG =12 -const T_STRING_INPLACE =13 -const T_BOOL =14 -const T_OBJECT_EX =16 -const T_LONGLONG =17 # added in Python 2.5 -const T_ULONGLONG =18 # added in Python 2.5 -const T_PYSSIZET =19 # added in Python 2.6 -const T_NONE =20 # added in Python 3.0 - -# flags: -const READONLY = 1 -const READ_RESTRICTED = 2 -const PY_WRITE_RESTRICTED = 4 -const RESTRICTED = (READ_RESTRICTED | PY_WRITE_RESTRICTED) ################################################################ -# type-flag constants, from Python object.h: - -# Python 2.7 -const Py_TPFLAGS_HAVE_GETCHARBUFFER = (0x00000001<<0) -const Py_TPFLAGS_HAVE_SEQUENCE_IN = (0x00000001<<1) -const Py_TPFLAGS_GC = 0 # was sometimes (0x00000001<<2) in Python <= 2.1 -const Py_TPFLAGS_HAVE_INPLACEOPS = (0x00000001<<3) -const Py_TPFLAGS_CHECKTYPES = (0x00000001<<4) -const Py_TPFLAGS_HAVE_RICHCOMPARE = (0x00000001<<5) -const Py_TPFLAGS_HAVE_WEAKREFS = (0x00000001<<6) -const Py_TPFLAGS_HAVE_ITER = (0x00000001<<7) -const Py_TPFLAGS_HAVE_CLASS = (0x00000001<<8) -const Py_TPFLAGS_HAVE_INDEX = (0x00000001<<17) -const Py_TPFLAGS_HAVE_NEWBUFFER = (0x00000001<<21) -const Py_TPFLAGS_STRING_SUBCLASS = (0x00000001<<27) - -# Python 3.0+ has only these: -const Py_TPFLAGS_HEAPTYPE = (0x00000001<<9) -const Py_TPFLAGS_BASETYPE = (0x00000001<<10) -const Py_TPFLAGS_READY = (0x00000001<<12) -const Py_TPFLAGS_READYING = (0x00000001<<13) -const Py_TPFLAGS_HAVE_GC = (0x00000001<<14) -const Py_TPFLAGS_HAVE_VERSION_TAG = (0x00000001<<18) -const Py_TPFLAGS_VALID_VERSION_TAG = (0x00000001<<19) -const Py_TPFLAGS_IS_ABSTRACT = (0x00000001<<20) -const Py_TPFLAGS_INT_SUBCLASS = (0x00000001<<23) -const Py_TPFLAGS_LONG_SUBCLASS = (0x00000001<<24) -const Py_TPFLAGS_LIST_SUBCLASS = (0x00000001<<25) -const Py_TPFLAGS_TUPLE_SUBCLASS = (0x00000001<<26) -const Py_TPFLAGS_BYTES_SUBCLASS = (0x00000001<<27) -const Py_TPFLAGS_UNICODE_SUBCLASS = (0x00000001<<28) -const Py_TPFLAGS_DICT_SUBCLASS = (0x00000001<<29) -const Py_TPFLAGS_BASE_EXC_SUBCLASS = (0x00000001<<30) -const Py_TPFLAGS_TYPE_SUBCLASS = (0x00000001<<31) - -# only use this if we have the stackless extension -const Py_TPFLAGS_HAVE_STACKLESS_EXTENSION_ = (0x00000003<<15) +# Mirror of PyMappingMethods in Python object.h + +const PyMappingMethods_fields = [ + (:mp_length, Ptr{Cvoid}, C_NULL), + (:mp_subscript, Ptr{Cvoid}, C_NULL), + (:mp_ass_subscript, Ptr{Cvoid}, C_NULL), +] + +@eval struct PyMappingMethods + $([:($n :: $t) for (n,t,d) in PyMappingMethods_fields]...) + PyMappingMethods(; $([Expr(:kw, n, d) for (n,t,d) in PyMappingMethods_fields]...)) = + new($([:(convert($t, $n)) for (n,t,d) in PyMappingMethods_fields]...)) +end ################################################################ # Mirror of PyTypeObject in Python object.h # -- assumes non-debugging Python build (no Py_TRACE_REFS) # -- most fields can default to 0 except where noted -const sizeof_PyObject_HEAD = sizeof(Int) + sizeof(PyPtr) -const sizeof_pyjlwrap_head = sizeof_PyObject_HEAD + sizeof(PyPtr) +PyTypeObject_defaultflags() = + pyversion.major >= 3 ? + (Py_TPFLAGS_HAVE_STACKLESS_EXTENSION[] | + Py_TPFLAGS_HAVE_VERSION_TAG) : + (Py_TPFLAGS_HAVE_GETCHARBUFFER | + Py_TPFLAGS_HAVE_SEQUENCE_IN | + Py_TPFLAGS_HAVE_INPLACEOPS | + Py_TPFLAGS_HAVE_RICHCOMPARE | + Py_TPFLAGS_HAVE_WEAKREFS | + Py_TPFLAGS_HAVE_ITER | + Py_TPFLAGS_HAVE_CLASS | + Py_TPFLAGS_HAVE_STACKLESS_EXTENSION[] | + Py_TPFLAGS_HAVE_INDEX) -mutable struct PyTypeObject +const PyTypeObject_fields = [ # PyObject_HEAD (for non-Py_TRACE_REFS build): - ob_refcnt::Int - ob_type::PyPtr - ob_size::Int # PyObject_VAR_HEAD + (:ob_refcnt, Int, 0), + (:ob_type, PyPtr, C_NULL), + (:ob_size, Int, 0), # PyObject_VAR_HEAD, C_NULL # PyTypeObject fields: - tp_name::Ptr{UInt8} # required, should be in format "." + (:tp_name, Ptr{UInt8}, C_NULL), # required, should be in format "." # warning: these two were Cint for Python <= 2.4 - tp_basicsize::Int # required, = sizeof(instance) - tp_itemsize::Int + (:tp_basicsize, Int, 0), # required, = sizeof(instance) + (:tp_itemsize, Int, 0), - tp_dealloc::Ptr{Cvoid} - tp_print::Ptr{Cvoid} - tp_getattr::Ptr{Cvoid} - tp_setattr::Ptr{Cvoid} - tp_compare::Ptr{Cvoid} - tp_repr::Ptr{Cvoid} + (:tp_dealloc, Ptr{Cvoid}, C_NULL), + (:tp_print, Ptr{Cvoid}, C_NULL), + (:tp_getattr, Ptr{Cvoid}, C_NULL), + (:tp_setattr, Ptr{Cvoid}, C_NULL), + (:tp_compare, Ptr{Cvoid}, C_NULL), + (:tp_repr, Ptr{Cvoid}, C_NULL), - tp_as_number::Ptr{Cvoid} - tp_as_sequence::Ptr{Cvoid} - tp_as_mapping::Ptr{Cvoid} + (:tp_as_number, Ptr{PyNumberMethods}, C_NULL), + (:tp_as_sequence, Ptr{PySequenceMethods}, C_NULL), + (:tp_as_mapping, Ptr{PyMappingMethods}, C_NULL), - tp_hash::Ptr{Cvoid} - tp_call::Ptr{Cvoid} - tp_str::Ptr{Cvoid} - tp_getattro::Ptr{Cvoid} - tp_setattro::Ptr{Cvoid} + (:tp_hash, Ptr{Cvoid}, C_NULL), + (:tp_call, Ptr{Cvoid}, C_NULL), + (:tp_str, Ptr{Cvoid}, C_NULL), + (:tp_getattro, Ptr{Cvoid}, C_NULL), + (:tp_setattro, Ptr{Cvoid}, C_NULL), - tp_as_buffer::Ptr{Cvoid} + (:tp_as_buffer, Ptr{Cvoid}, C_NULL), - tp_flags::Clong # Required, should default to Py_TPFLAGS_DEFAULT + (:tp_flags, Clong, 0), # Required, should default to Py_TPFLAGS_DEFAULT - tp_doc::Ptr{UInt8} # normally set in example code, but may be NULL + (:tp_doc, Ptr{UInt8}, C_NULL), # normally set in example code, but may be NULL - tp_traverse::Ptr{Cvoid} + (:tp_traverse, Ptr{Cvoid}, C_NULL), - tp_clear::Ptr{Cvoid} + (:tp_clear, Ptr{Cvoid}, C_NULL), - tp_richcompare::Ptr{Cvoid} + (:tp_richcompare, Ptr{Cvoid}, C_NULL), - tp_weaklistoffset::Int + (:tp_weaklistoffset, Int, 0), # added in Python 2.2: - tp_iter::Ptr{Cvoid} - tp_iternext::Ptr{Cvoid} - - tp_methods::Ptr{PyMethodDef} - tp_members::Ptr{PyMemberDef} - tp_getset::Ptr{PyGetSetDef} - tp_base::Ptr{Cvoid} - - tp_dict::PyPtr - tp_descr_get::Ptr{Cvoid} - tp_descr_set::Ptr{Cvoid} - tp_dictoffset::Int - - tp_init::Ptr{Cvoid} - tp_alloc::Ptr{Cvoid} - tp_new::Ptr{Cvoid} - tp_free::Ptr{Cvoid} - tp_is_gc::Ptr{Cvoid} - - tp_bases::PyPtr - tp_mro::PyPtr - tp_cache::PyPtr - tp_subclasses::PyPtr - tp_weaklist::PyPtr - tp_del::Ptr{Cvoid} + (:tp_iter, Ptr{Cvoid}, C_NULL), + (:tp_iternext, Ptr{Cvoid}, C_NULL), + + (:tp_methods, Ptr{PyMethodDef}, C_NULL), + (:tp_members, Ptr{PyMemberDef}, C_NULL), + (:tp_getset, Ptr{PyGetSetDef}, C_NULL), + (:tp_base, Ptr{Cvoid}, C_NULL), + + (:tp_dict, PyPtr, C_NULL), + (:tp_descr_get, Ptr{Cvoid}, C_NULL), + (:tp_descr_set, Ptr{Cvoid}, C_NULL), + (:tp_dictoffset, Int, 0), + + (:tp_init, Ptr{Cvoid}, C_NULL), + (:tp_alloc, Ptr{Cvoid}, C_NULL), + (:tp_new, Ptr{Cvoid}, C_NULL), + (:tp_free, Ptr{Cvoid}, C_NULL), + (:tp_is_gc, Ptr{Cvoid}, C_NULL), + + (:tp_bases, PyPtr, C_NULL), + (:tp_mro, PyPtr, C_NULL), + (:tp_cache, PyPtr, C_NULL), + (:tp_subclasses, PyPtr, C_NULL), + (:tp_weaklist, PyPtr, C_NULL), + (:tp_del, Ptr{Cvoid}, C_NULL), # added in Python 2.6: - tp_version_tag::Cuint + (:tp_version_tag, Cuint, 0), # only used for COUNT_ALLOCS builds of Python - tp_allocs::Int - tp_frees::Int - tp_maxalloc::Int - tp_prev::Ptr{Cvoid} - tp_next::Ptr{Cvoid} - - # Julia-specific fields, after the end of the Python structure: - - # save the tp_name Julia string so that it is not garbage-collected - tp_name_save # This is a gc slot that is never read from - - function PyTypeObject() - new(0,C_NULL,0, - C_NULL, - 0, 0, - C_NULL,C_NULL,C_NULL,C_NULL,C_NULL,C_NULL, # tp_dealloc ... - C_NULL,C_NULL,C_NULL, # tp_as_number... - C_NULL,C_NULL,C_NULL,C_NULL,C_NULL, # tp_hash ... - C_NULL, # tp_as_buffer - 0, - C_NULL, # tp_doc - C_NULL, # tp_traverse, - C_NULL, # tp_clear - C_NULL, # tp_richcompare - 0, # tp_weaklistoffset - C_NULL,C_NULL, # tp_iter, tp_iternext - C_NULL,C_NULL,C_NULL,C_NULL, # tp_methods... - C_NULL,C_NULL,C_NULL,0, # tp_dict... - C_NULL,C_NULL,C_NULL,C_NULL,C_NULL, # tp_init ... - C_NULL,C_NULL,C_NULL,C_NULL,C_NULL,C_NULL, # tp_bases... - 0, # tp_version_tag - 0,0,0,C_NULL,C_NULL, # tp_allocs... - "") + (:tp_allocs, Int, 0), + (:tp_frees, Int, 0), + (:tp_maxalloc, Int, 0), + (:tp_prev, Ptr{Cvoid}, C_NULL), + (:tp_next, Ptr{Cvoid}, C_NULL), +] + +@eval mutable struct PyTypeObject + $([:($n :: $t) for (n,t,d) in PyTypeObject_fields]...) + # cache of julia objects referenced by this type, to prevent them being garbage-collected + jl_cache::Dict{Symbol,Any} + + function PyTypeObject(; unsafe_null=false, opts...) + t = new($([:(convert($t, $d)) for (n,t,d) in PyTypeObject_fields]...), Dict{Symbol,Any}()) + unsafe_null ? t : PyTypeObject_init!(t; opts...) end - PyTypeObject(name::AbstractString, basicsize::Integer, init::Function) = - PyTypeObject!(PyTypeObject(), name, basicsize, init) end -# Often, PyTypeObject instances are global constants, which we initialize -# to 0 via PyTypeObject() and then initialize at runtime via PyTypeObject! -function PyTypeObject!(init::Function, t::PyTypeObject, name::AbstractString, basicsize::Integer) - t.tp_basicsize = convert(Int, basicsize) - - # figure out Py_TPFLAGS_DEFAULT, depending on Python version - t.tp_flags = # Py_TPFLAGS_DEFAULT = - pyversion.major >= 3 ? - (Py_TPFLAGS_HAVE_STACKLESS_EXTENSION[] | - Py_TPFLAGS_HAVE_VERSION_TAG) : - (Py_TPFLAGS_HAVE_GETCHARBUFFER | - Py_TPFLAGS_HAVE_SEQUENCE_IN | - Py_TPFLAGS_HAVE_INPLACEOPS | - Py_TPFLAGS_HAVE_RICHCOMPARE | - Py_TPFLAGS_HAVE_WEAKREFS | - Py_TPFLAGS_HAVE_ITER | - Py_TPFLAGS_HAVE_CLASS | - Py_TPFLAGS_HAVE_STACKLESS_EXTENSION[] | - Py_TPFLAGS_HAVE_INDEX) - - # Emulate the rooting behavior of a ccall: - name_save = Base.cconvert(Ptr{UInt8}, name) - t.tp_name_save = name_save - t.tp_name = unsafe_convert(Ptr{UInt8}, name_save) - - init(t) # initialize any other fields as needed - if t.tp_new == C_NULL - t.tp_new = @pyglobal :PyType_GenericNew +function PyTypeObject_init!(t::PyTypeObject; opts...) + for (k, x) in pairs(opts) + setproperty!(t, k, x) end + t.tp_name == C_NULL && error("required: tp_name") + t.tp_basicsize == 0 && !haskey(opts, :tp_basicsize) && error("required: tp_basicsize") + t.tp_flags == 0 && !haskey(opts, :tp_flags) && (t.tp_flags = PyTypeObject_defaultflags()) + t.tp_new == C_NULL && !haskey(opts, :tp_new) && (t.tp_new = @pyglobal(:PyType_GenericNew)) @pycheckz ccall((@pysym :PyType_Ready), Cint, (Ref{PyTypeObject},), t) ccall((@pysym :Py_IncRef), Cvoid, (Any,), t) return t end -################################################################ -# Wrap a Python type around a Julia Any object - -struct Py_jlWrap - # PyObject_HEAD (for non-Py_TRACE_REFS build): - ob_refcnt::Int - ob_type::PyPtr - - ob_weakrefs::PyPtr - jl_value::Any -end - -# destructor for jlwrap instance, assuming it was created with pyjlwrap_new -function pyjlwrap_dealloc(o::PyPtr) - p = convert(Ptr{PyPtr}, o) - if unsafe_load(p, 3) != PyPtr_NULL - ccall((@pysym :PyObject_ClearWeakRefs), Cvoid, (PyPtr,), o) - end - delete!(pycall_gc, o) - return nothing -end - -unsafe_pyjlwrap_to_objref(o::Union{PyPtr, PyObject}) = - GC.@preserve o unsafe_pointer_to_objref(unsafe_load(convert(Ptr{Ptr{Cvoid}}, PyPtr(o)), 4)) - -function pyjlwrap_repr(o::PyPtr) - try - return pyreturn(o != C_NULL ? string("") - : "") - catch e - @pyraise e - return PyPtr_NULL - end -end - -function pyjlwrap_hash(o::PyPtr) - h = hash(unsafe_pyjlwrap_to_objref(o)) - # Python hashes are not permitted to return -1!! - return h == reinterpret(UInt, -1) ? pysalt::UInt : h::UInt -end - -# 32-bit hash on 64-bit machines, needed for Python < 3.2 with Windows -const pysalt32 = 0xb592cd9b # hash("PyCall") % UInt32 -function pyjlwrap_hash32(o::PyPtr) - h = ccall(:int64to32hash, UInt32, (UInt64,), - hash(unsafe_pyjlwrap_to_objref(o))) - # Python hashes are not permitted to return -1!! - return h == reinterpret(UInt32, Int32(-1)) ? pysalt32 : h::UInt32 -end - -docstring(x) = string(Docs.doc(x)) - -# this function emulates standard attributes of Python functions, -# where possible. -function pyjlwrap_getattr(self_::PyPtr, attr__::PyPtr) - attr_ = PyObject(attr__) # don't need pyincref because of finally clause below - try - f = unsafe_pyjlwrap_to_objref(self_) - attr = convert(String, attr_) - if attr in ("__name__","func_name") - return pystealref!(PyObject(string(f))) - elseif attr in ("__doc__", "func_doc") - return pystealref!(PyObject(docstring(f))) - elseif attr in ("__module__","__defaults__","func_defaults","__closure__","func_closure") - return pystealref!(PyObject(nothing)) - elseif startswith(attr, "__") - # TODO: handle __code__/func_code (issue #268) - return ccall(@pysym(:PyObject_GenericGetAttr), PyPtr, (PyPtr,PyPtr), self_, attr__) - else - fidx = Base.fieldindex(typeof(f), Symbol(attr), false) - if fidx != 0 - return pyreturn(getfield(f, fidx)) - else - return ccall(@pysym(:PyObject_GenericGetAttr), PyPtr, (PyPtr,PyPtr), self_, attr__) - end +function Base.setproperty!(t::PyTypeObject, k::Symbol, x) + if k == :tp_name && x isa AbstractString + z = t.jl_cache[k] = Base.cconvert(Ptr{UInt8}, x) + setfield!(t, k, unsafe_convert(Ptr{UInt8}, z)) + elseif k == :tp_as_number && x isa PyNumberMethods + z = t.jl_cache[k] = Ref(x) + setfield!(t, k, unsafe_convert(Ptr{PyNumberMethods}, z)) + elseif k == :tp_as_sequence && x isa PySequenceMethods + z = t.jl_cache[k] = Ref(x) + setfield!(t, k, unsafe_convert(Ptr{PySequenceMethods}, z)) + elseif k == :tp_as_mapping && x isa PyMappingMethods + z = t.jl_cache[k] = Ref(x) + setfield!(t, k, unsafe_convert(Ptr{PyMappingMethods}, z)) + elseif k == :tp_members && x isa AbstractVector{PyMemberDef} + z = t.jl_cache[k] = push!(copy(convert(Vector{PyMemberDef}, x)), PyMemberDef()) + setfield!(t, k, pointer(z)) + elseif k == :tp_methods && x isa AbstractVector{PyMethodDef} + z = t.jl_cache[k] = push!(copy(convert(Vector{PyMethodDef}, x)), PyMethodDef()) + setfield!(t, k, pointer(z)) + elseif k == :tp_getset && x isa AbstractVector{PyGetSetDef} + z = t.jl_cache[k] = push!(copy(convert(Vector{PyGetSetDef}, x)), PyGetSetDef()) + setfield!(t, k, pointer(z)) + elseif k == :tp_dict && x isa NamedTuple + d = t.jl_cache[k] = PyObject(@pycheckn ccall(@pysym(:PyDict_New), PyPtr, ())) + for (k, v) in pairs(x) + @pycheckz ccall(@pysym(:PyDict_SetItemString), Cint, (PyPtr, Cstring, PyPtr), d, string(k), PyObject(v)) end - catch e - @pyraise e - finally - setfield!(attr_, :o, PyPtr_NULL) # don't decref - end - return PyPtr_NULL -end - -# constant strings (must not be gc'ed) for pyjlwrap_members -const pyjlwrap_membername = "jl_value" -const pyjlwrap_doc = "Julia jl_value_t* (Any object)" -# other pointer-containing constants that need to be initialized at runtime -const pyjlwrap_members = PyMemberDef[] -const jlWrapType = PyTypeObject() # initialized by pyjlwrap_init in __init__ -const Py_TPFLAGS_HAVE_STACKLESS_EXTENSION = Ref(0x00000000) - -# use this to create a new jlwrap type, with init to set up custom members -function pyjlwrap_type!(init::Function, to::PyTypeObject, name::AbstractString) - sz = sizeof(Py_jlWrap) + sizeof(PyPtr) # must be > base type - PyTypeObject!(to, name, sz) do t::PyTypeObject - # We want tp_base to be a pointer to the C-like PyTypeObject struct. - # This is equivalent to the jl_value_t* in Julia (see JuliaLang/julia#31473). - t.tp_base = pointer_from_objref(jlWrapType) - ccall((@pysym :Py_IncRef), Cvoid, (Ref{PyTypeObject},), jlWrapType) - init(t) - end -end - -pyjlwrap_type(init::Function, name::AbstractString) = - pyjlwrap_type!(init, PyTypeObject(), name) - -# Given a jlwrap type, create a new instance (and save value for gc) -function pyjlwrap_new(pyT::PyTypeObject, value::Any) - o = PyObject(@pycheckn ccall((@pysym :_PyObject_New), - PyPtr, (Ref{PyTypeObject},), pyT)) - p = convert(Ptr{Ptr{Cvoid}}, PyPtr(o)) - if isimmutable(value) - # It is undefined to call `pointer_from_objref` on immutable objects. - # The compiler is free to return basically anything since the boxing is not - # significant at all. - # Below is a well defined way to get a pointer (`ptr`) and an object that defines - # the lifetime of the pointer `ref`. - ref = Ref{Any}(value) - pycall_gc[PyPtr(o)] = ref - ptr = unsafe_load(Ptr{Ptr{Cvoid}}(pointer_from_objref(ref))) + setfield!(t, k, PyPtr(d)) + elseif k == :tp_base && x isa PyTypeObject + t.jl_cache[k] = x + setfield!(t, k, pointer_from_objref(x)) else - pycall_gc[PyPtr(o)] = value - ptr = pointer_from_objref(value) + setfield!(t, k, convert(fieldtype(PyTypeObject, k), x)) end - unsafe_store!(p, C_NULL, 3) - unsafe_store!(p, ptr, 4) - return o end -function pyjlwrap_new(x::Any) - pyjlwrap_new(jlWrapType, x) -end - -is_pyjlwrap(o::PyObject) = jlWrapType.tp_new != C_NULL && ccall((@pysym :PyObject_IsInstance), Cint, (PyPtr, Ref{PyTypeObject}), o, jlWrapType) == 1 - -################################################################ -# Fallback conversion: if we don't have a better conversion function, -# just wrap the Julia object in a Python object +unsafe_pytype(o::PyPtr) = + convert(Ptr{PyTypeObject}, unsafe_load(o).ob_type) -PyObject(x::Any) = pyjlwrap_new(x) +PyObject(t::PyTypeObject) = pyincref(convert(PyPtr, pointer_from_objref(t))) diff --git a/src/pyvector.jl b/src/pyvector.jl new file mode 100644 index 00000000..b81666bb --- /dev/null +++ b/src/pyvector.jl @@ -0,0 +1,69 @@ +######################################################################### +# PyVector: no-copy wrapping of a Julia object around a Python sequence + +""" + PyVector(o::PyObject) + +This returns a PyVector object, which is a wrapper around an arbitrary Python list or sequence object. + +Alternatively, `PyVector` can be used as the return type for a `pycall` that returns a sequence object (including tuples). +""" +mutable struct PyVector{T} <: AbstractVector{T} + o::PyObject + function PyVector{T}(o::PyObject) where T + if ispynull(o) + throw(ArgumentError("cannot make PyVector from NULL PyObject")) + end + new{T}(o) + end +end + +PyVector(o::PyObject) = PyVector{PyAny}(o) +PyObject(a::PyVector) = a.o +convert(::Type{PyVector}, o::PyObject) = PyVector(o) +convert(::Type{PyVector{T}}, o::PyObject) where {T} = PyVector{T}(o) +unsafe_convert(::Type{PyPtr}, a::PyVector) = PyPtr(a.o) +PyVector(a::PyVector) = a +PyVector(a::AbstractVector{T}) where {T} = PyVector{T}(array2py(a)) + +# when a PyVector is copied it is converted into an ordinary Julia Vector +similar(a::PyVector, T, dims::Dims) = Array{T}(dims) +similar(a::PyVector{T}) where {T} = similar(a, pyany_toany(T), size(a)) +similar(a::PyVector{T}, dims::Dims) where {T} = similar(a, pyany_toany(T), dims) +similar(a::PyVector{T}, dims::Int...) where {T} = similar(a, pyany_toany(T), dims) +eltype(::PyVector{T}) where {T} = pyany_toany(T) +eltype(::Type{PyVector{T}}) where {T} = pyany_toany(T) + +size(a::PyVector) = (length(a.o),) + +getindex(a::PyVector) = getindex(a, 1) +getindex(a::PyVector{T}, i::Integer) where {T} = convert(T, PyObject(@pycheckn ccall((@pysym :PySequence_GetItem), PyPtr, (PyPtr, Int), a, i-1))) + +setindex!(a::PyVector, v) = setindex!(a, v, 1) +function setindex!(a::PyVector, v, i::Integer) + @pycheckz ccall((@pysym :PySequence_SetItem), Cint, (PyPtr, Int, PyPtr), a, i-1, PyObject(v)) + v +end + +summary(a::PyVector{T}) where {T} = string(Base.dims2string(size(a)), " ", + string(pyany_toany(T)), " PyVector") + +splice!(a::PyVector, i::Integer) = splice!(a.o, i) +function splice!(a::PyVector{T}, indices::AbstractVector{I}) where {T,I<:Integer} + v = pyany_toany(T)[a[i] for i in indices] + for i in sort(indices, rev=true) + @pycheckz ccall((@pysym :PySequence_DelItem), Cint, (PyPtr, Int), a, i-1) + end + v +end +pop!(a::PyVector) = pop!(a.o) +popfirst!(a::PyVector) = popfirst!(a.o) +empty!(a::PyVector) = empty!(a.o) + +# only works for List subtypes: +push!(a::PyVector, item) = push!(a.o, item) +insert!(a::PyVector, i::Integer, item) = insert!(a.o, i, item) +pushfirst!(a::PyVector, item) = pushfirst!(a.o, item) +prepend!(a::PyVector, items) = prepend!(a.o, items) +append!(a::PyVector{T}, items) where {T} = PyVector{T}(append!(a.o, items)) + diff --git a/src/startup.jl b/src/startup.jl index 7f7b2e91..8b515757 100644 --- a/src/startup.jl +++ b/src/startup.jl @@ -122,10 +122,10 @@ if libpython == nothing :(cglobal($(esc(name)))) end macro pyglobalobj(name) - :(cglobal($(esc(name)), PyObject_struct)) + :(cglobal($(esc(name)), CPyObject)) end macro pyglobalobjptr(name) - :(unsafe_load(cglobal($(esc(name)), Ptr{PyObject_struct}))) + :(unsafe_load(cglobal($(esc(name)), Ptr{CPyObject}))) end else macro pysym(func) @@ -135,9 +135,9 @@ else :(cglobal(($(esc(name)), libpython))) end macro pyglobalobj(name) - :(cglobal(($(esc(name)), libpython), PyObject_struct)) + :(cglobal(($(esc(name)), libpython), CPyObject)) end macro pyglobalobjptr(name) - :(unsafe_load(cglobal(($(esc(name)), libpython), Ptr{PyObject_struct}))) + :(unsafe_load(cglobal(($(esc(name)), libpython), Ptr{CPyObject}))) end end