Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

WIP: Add support for controlling recursion and parametric types #7

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

WIP: Add support for controlling recursion and parametric types #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 99 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,105 @@ probably slower than an ordinary array, since a new object must be heap
allocated every time the StructOfArrays is indexed. In practice, StructsOfArrays
works best with `isbits` immutables such as `Complex{T}`.

## Advanced Usage

When embedding a StructOfArrays in a larger, data structure, it can be useful
to automatically compute the SoA type corresponding to a regular arrays type.
This can be accomplished using the `similar` function:

```
struct Vectors
x::similar(StructOfArrays, Vector{Complex{Float64}})
y::similar(StructOfArrays, Vector{Complex{Float64}})
end
# equivalent in layout to
struct Vectors
x_real::Vector{Float64}
x_imag::Vector{Float64}
y_real::Vector{Float64}
y_imag::Vector{Float64}
end
```

Note that this feature also works with parameterized types. However, if
later a composite type is substituted for type type parameter, it will not
be unpacked into separate arrays:

```
struct Vector3D{T}
x::T
y::T
z::T
end
struct Container{T}
soa::similar(StructOfArrays, Vector{Vector3D{T}})
end
# Container{Float64} is equivalent to
struct ContainerFloat64
soa_x::Vector{Float64}
soa_y::Vector{Float64}
soa_z::Vector{Float64}
end
# Container{Complex{Float64}} is equivalent to
struct ContainerFloat64
soa_x::Vector{Complex{Float64}}
soa_y::Vector{Complex{Float64}}
soa_z::Vector{Complex{Float64}}
end
# Note that this is different from similar(StructOfArrays, Vector{Vector3D{Complex{Float64}}}), which would expand to
struct ContainerFloat64
soa_x_real::Vector{Float64}
soa_x_imag::Vector{Float64}
soa_y_real::Vector{Float64}
soa_y_imag::Vector{Float64}
soa_y_real::Vector{Float64}
soa_y_imag::Vector{Float64}
end
```

This behavior was chosen to accomodate julia's handling of parameterize element,
types. If future versions of julia expand these capabilities, the default behavior
may need to be revisited.

Lastly, note that it is possible to choose control the recursion explicitly,
by providing a type (as the second type parameters) whose subtypes should be
considered leaves for the purpose of recursion. E.g.:
```
struct Bundle
x::Complex{Float64}
y::Complex{Int64}
z::Rational{Int64}
end
# Consider
A = StructOfArrays{Bundle, Any}(2,2)
# Since all types are <: Any, no recusion will occur, and the SoaA is equivalent to
struct SoAAny
x::Matrix{Complex{Float64}}
y::Matrix{Complex{Int64}}
z::Matrix{Rational{Int64}}
end
# Next, let's say we want to have separate SoA for the complex values. Consider
B = StructOfArrays{Bundle, Complex}(2,2)
# which will be equivalent to
struct SoAComplex
x_real::Matrix{Float64}
x_imag::Matrix{Float64}
y_real::Matrix{Int64}
y_imag::Matrix{Int64}
z::Matrix{Rational{Int64}}
end
# Lastly it is of course possible to specify a union:
C = StructOfArrays{Bundle, Union{Complex{Float64},Rational}}(2,2)
struct SoAUnion
x_real::Matrix{Float64}
x_imag::Matrix{Float64}
y::Matrox{Complex{Int64}}
z_num::Matrix{Int64}
z_den::Matrix{Int64}
end
```


## Benchmark

```julia
Expand Down
152 changes: 112 additions & 40 deletions src/StructsOfArrays.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,40 +1,97 @@
__precompile__()
module StructsOfArrays

using Compat

export StructOfArrays

immutable StructOfArrays{T,N,U<:Tuple} <: AbstractArray{T,N}
immutable StructOfArrays{T,Leaves,N,U<:Tuple} <: AbstractArray{T,N}
arrays::U
StructOfArrays{T,Leaves,N,U}(arrays::U) where {T,Leaves,N,U} = new{T,Leaves,N,U}(arrays)
end

function gather_eltypes(T, visited = Set{Type}())
(!isleaftype(T) || T.mutable) && throw(ArgumentError("can only create an StructOfArrays of leaf type immutables"))
function gather_eltypes(T, Leaves=Union{}, visited = Set{Type}(); allow_typevar=false)
@assert !isa(T, UnionAll)
T.mutable && throw(ArgumentError("can not create StructOfArray for a mutable type"))
isempty(T.types) && throw(ArgumentError("cannot create an StructOfArrays of an empty or bitstype"))
types = Type[]
types = Any[]
typevars = TypeVar[]
push!(visited, T)
for S in T.types
sizeof(S) == 0 && continue
(S in visited) && throw(ArgumentError("Recursive types are not allowed for SoA conversion"))
if isempty(S.types)
if isa(S, TypeVar)
!allow_typevar && throw(ArgumentError("TypeVars are not allowed when constructing an SoA array"))
push!(typevars, S)
elseif isa(S, UnionAll)
throw(ArgumentError("Interior UnionAlls are not allowed"))
elseif isleaftype(S) && S.size == 0
continue
end
if isa(S, DataType) && S.name.wrapper == Vararg
n = S.parameters[2]
elT = S.parameters[1]
isa(n, Integer) || throw(ArgumentError("Tuple are only supported with definite lengths"))
isa(elT, TypeVar) && push!(typevars, elT)
if isa(elT, TypeVar) || isempty(eT.types) || elT <: Leaves
append!(types, [elT for i = 1:n])
else
ntypes, ntypevars = gather_eltypes(S, Leaves, copy(visited); allow_typevar=allow_typevar)
for i = 1:n
append!(types, ntypes)
end
append!(typevars, ntypevars)
end
elseif isa(S, TypeVar) || isempty(S.types) || S <: Leaves
push!(types, S)
else
append!(types, gather_eltypes(S, copy(visited)))
ntypes, ntypevars = gather_eltypes(S, Leaves, copy(visited); allow_typevar=allow_typevar)
append!(types, ntypes)
append!(typevars, ntypevars)
end
end
types
types, typevars
end

@generated function StructOfArrays{T}(::Type{T}, dims::Integer...)
N = length(dims)
types = gather_eltypes(T)
arrtuple = Tuple{[Array{S,N} for S in types]...}
:(StructOfArrays{T,$N,$arrtuple}(($([:(Array{$(S)}(dims)) for S in types]...),)))
@generated function StructOfArrays{T,Leaves,N,arrtuple}(dims::Tuple{Vararg{Integer}}) where {T,Leaves,N,arrtuple}
Expr(:new, StructOfArrays{T,Leaves,N,arrtuple},
Expr(:tuple,
(:($arr(dims)) for arr in arrtuple.parameters)...))
end

@generated function StructOfArrays{T,Leaves}(dims::Tuple{Vararg{Integer}}) where {T,Leaves}
types = gather_eltypes(T, Leaves)[1]
arrtuple = Expr(:curly, Tuple, [:(Array{$S,length(dims)}) for S in types]...)
:(StructOfArrays{T,Leaves,length(dims),$arrtuple}(dims...))
end
StructOfArrays(T::Type, dims::Tuple{Vararg{Integer}}) = StructOfArrays(T, dims...)

@generated function StructOfArrays{T}(dims::Tuple{Vararg{Integer}}) where {T}
types, typevars = gather_eltypes(T)
@assert isempty(typevars)
arrtuple = Expr(:curly, Tuple, [:(Array{$S,length(dims)}) for S in types]...)
:(StructOfArrays{T,Union{},length(dims),$arrtuple}(($([:(Array{$(S)}(dims)) for S in types]...),)))
end
StructOfArrays(T::Type, dims::Integer...) = StructOfArrays{T}(dims)
StructOfArrays(T::Type, dims::Tuple{Vararg{Integer}}) = StructOfArrays{T}(dims)
StructOfArrays{T}(dims::Integer...) where {T} = StructOfArrays{T}(dims)
StructOfArrays{T,Leaves}(dims::Integer...) where {T,Leaves} = StructOfArrays{T,Leaves}(dims)
StructOfArrays{T,Leaves,N,U}(dims::Integer...) where {T,Leaves,N,U} = StructOfArrays{T,Leaves,N,U}(dims)
StructOfArrays{T,<:Any,N}() where {T,N} = StructOfArrays{T}((0 for i=1:N)...)
StructOfArrays{T,Leaves,N,U}() where {T,Leaves,N,U} = StructOfArrays{T,Leaves,N,U}((0 for i=1:N)...)

@compat Base.IndexStyle{T<:StructOfArrays}(::Type{T}) = IndexLinear()

function Base.similar{T,N}(::Type{<:StructOfArrays}, ::Type{<:AbstractArray{T,N}})
uT = Base.unwrap_unionall(T)
types, typevars = gather_eltypes(uT, allow_typevar = true)
Leaves = Union{typevars...}
arrtuple = Tuple{[Array{S,N} for S in types]...}
Base.rewrap_unionall(StructOfArrays{uT, Leaves, N, arrtuple}, T)
end

function Base.similar{T,N}(::Type{StructOfArrays}, A::AbstractArray{T,N})
StructOfArrays{T}(size(A))
end

@generated function Base.similar{T}(A::StructOfArrays, ::Type{T}, dims::Dims)
if isbits(T) && length(T.types) > 1
:(StructOfArrays(T, dims))
Expand All @@ -53,49 +110,64 @@ Base.convert{T,N}(::Type{StructOfArrays}, A::AbstractArray{T,N}) =
Base.size(A::StructOfArrays) = size(A.arrays[1])
Base.size(A::StructOfArrays, d) = size(A.arrays[1], d)

function generate_getindex(T, arraynum)
members = Expr[]
for S in T.types
function generate_elementwise(leaff, recursef, combinef, state, T, Leaves, arraynum=1)
for (el,S) in enumerate(T.types)
sizeof(S) == 0 && push!(members, :($(S())))
if isempty(S.types)
push!(members, :(A.arrays[$arraynum][i...]))
if isempty(S.types) || S <: Leaves
leaff(arraynum, el, state)
arraynum += 1
else
member, arraynum = generate_getindex(S, arraynum)
push!(members, member)
nstate, arraynum = generate_elementwise(leaff, recursef, combinef, recursef(S, arraynum, el, state), S, Leaves, arraynum)
combinef(S, arraynum, el, state, nstate)
end
end
Expr(:new, T, members...), arraynum
state, arraynum
end

@generated function Base.getindex{T}(A::StructOfArrays{T}, i::Integer...)
strct, _ = generate_getindex(T, 1)
Expr(:block, Expr(:meta, :inline), strct)
@generated function Base.getindex{T,Leaves}(A::StructOfArrays{T,Leaves}, i::Integer...)
leaf(arraynum, el, members) = push!(members, :(A.arrays[$arraynum][i...]))
recursef(S, arraynum, el, state) = Expr[]
combinef(S, arraynum, el, members, eltmems) = push!(members, Expr(:new, S, eltmems...))
mems, _ = generate_elementwise(leaf, recursef, combinef, Expr[], T, Leaves)
Expr(:block, Expr(:meta, :inline, :propagate_inbounds), Expr(:new, T, mems...))
end

function generate_setindex(T, x, arraynum)
s = gensym()
exprs = Expr[:($s = $x)]
for (el,S) in enumerate(T.types)
sizeof(S) == 0 && push!(members, :($(S())))
if isempty(S.types)
push!(exprs, :(A.arrays[$arraynum][i...] = getfield($s, $el)))
arraynum += 1
else
nexprs, arraynum = generate_setindex(S, :(getfield($s, $el)), arraynum)
append!(exprs, nexprs)
end
function setindex_recusion(exprs)
function recursef(S, arraynum, el, state)
s = gensym()
push!(exprs, :($s = getfield($state, $el)))
s
end
end

no_combinef(args...) = nothing

@generated function Base.setindex!{T, Leaves}(A::StructOfArrays{T, Leaves}, x, i::Integer...)
exprs = Expr[]
function leaf(arraynum, el, state)
push!(exprs, :(A.arrays[$arraynum][i...] = getfield($state, $el)))
end
generate_elementwise(leaf, setindex_recusion(exprs), no_combinef, :v, T, Leaves)
exprs = Expr(:block, exprs...)
quote
$(Expr(:meta, :inline, :propagate_inbounds))
v = convert(T, x)
$exprs
x
end
exprs, arraynum
end

@generated function Base.setindex!{T}(A::StructOfArrays{T}, x, i::Integer...)
exprs = Expr(:block, generate_setindex(T, :x, 1)[1]...)
@generated function Base.push!{T, Leaves}(A::StructOfArrays{T, Leaves}, x)
exprs = Expr[]
leaf(arraynum, el, state) = push!(exprs, :(push!(A.arrays[$arraynum],getfield($state,$el))))
generate_elementwise(leaf, setindex_recusion(exprs), no_combinef, :v, T, Leaves)
exprs = Expr(:block, exprs...)
quote
$(Expr(:meta, :inline))
v = convert(T, x)
$exprs
x
end
end

end # module
Loading