Skip to content

Commit

Permalink
make map and collect more general and uniform by adding iterator …
Browse files Browse the repository at this point in the history 
…traits

make HasEltype and HasLength the default

this is a possible fix for #15342

type-accumulating Dict constructor

some tests for type accumulation in `map` and `Dict`
  • Loading branch information
@JeffBezanson
JeffBezanson committed Mar 28, 2016
1 parent 2103b17 commit a2ce230
Show file tree
Hide file tree
Showing 21 changed files with 243 additions and 220 deletions.
152 changes: 5 additions & 147 deletions base/abstractarray.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1086,34 +1086,6 @@ foreach(f) = (f(); nothing)
foreach(f, itr) = (for x in itr; f(x); end; nothing)
foreach(f, itrs...) = (for z in zip(itrs...); f(z...); end; nothing)

# generic map on any iterator
function map(f, iters...)
result = []
len = length(iters)
states = [start(iters[idx]) for idx in 1:len]
nxtvals = cell(len)
cont = true
for idx in 1:len
if done(iters[idx], states[idx])
cont = false
break
end
end
while cont
for idx in 1:len
nxtvals[idx],states[idx] = next(iters[idx], states[idx])
end
push!(result, f(nxtvals...))
for idx in 1:len
if done(iters[idx], states[idx])
cont = false
break
end
end
end
result
end

## map over arrays ##

## transform any set of dimensions
Expand Down Expand Up @@ -1174,39 +1146,6 @@ function mapslices(f, A::AbstractArray, dims::AbstractVector)
return R
end


# using promote_type
function promote_to!{T,F}(f::F, offs, dest::AbstractArray{T}, A::AbstractArray)
# map to dest array, checking the type of each result. if a result does not
# match, do a type promotion and re-dispatch.
for i = offs:length(A) #Fixme iter
@inbounds Ai = A[i]
el = f(Ai)
S = typeof(el)
if S === T || S <: T
@inbounds dest[i] = el::T
else
R = promote_type(T, S)
if R !== T
new = similar(dest, R)
copy!(new,1, dest,1, i-1)
new[i] = el
return promote_to!(f, i+1, new, A)
end
@inbounds dest[i] = el
end
end
return dest
end

function map_promote(f, A::AbstractArray)
if isempty(A); return similar(A, Bottom); end
first = f(A[1])
dest = similar(A, typeof(first))
dest[1] = first
return promote_to!(f, 2, dest, A)
end

# These are needed because map(eltype, As) is not inferrable
promote_eltype_op(::Any) = (@_pure_meta; Bottom)
promote_eltype_op{T}(op, ::AbstractArray{T}) = (@_pure_meta; promote_op(op, T))
Expand All @@ -1225,39 +1164,11 @@ function map!{F}(f::F, dest::AbstractArray, A::AbstractArray)
return dest
end

function map_to!{T,F}(f::F, offs, st, dest::AbstractArray{T}, A)
# map to dest array, checking the type of each result. if a result does not
# match, widen the result type and re-dispatch.
i = offs
while !done(A, st)
@inbounds Ai, st = next(A, st)
el = f(Ai)
S = typeof(el)
if S === T || S <: T
@inbounds dest[i] = el::T
i += 1
else
R = typejoin(T, S)
new = similar(dest, R)
copy!(new,1, dest,1, i-1)
@inbounds new[i] = el
return map_to!(f, i+1, st, new, A)
end
end
return dest
end
# map on collections
map(f, A::Union{AbstractArray,AbstractSet,Associative}) = collect_similar(A, Generator(f,A))

function map(f, A::AbstractArray)
if isempty(A)
return isa(f,Type) ? similar(A,f) : similar(A)
end
st = start(A)
A1, st = next(A, st)
first = f(A1)
dest = similar(A, typeof(first))
dest[1] = first
return map_to!(f, 2, st, dest, A)
end
# default to returning an Array for `map` on general iterators
map(f, A) = collect(Generator(f,A))

## 2 argument
function map!{F}(f::F, dest::AbstractArray, A::AbstractArray, B::AbstractArray)
Expand All @@ -1267,34 +1178,6 @@ function map!{F}(f::F, dest::AbstractArray, A::AbstractArray, B::AbstractArray)
return dest
end

function map_to!{T,F}(f::F, offs, dest::AbstractArray{T}, A::AbstractArray, B::AbstractArray)
for i = offs:length(A) #Fixme iter
@inbounds Ai, Bi = A[i], B[i]
el = f(Ai, Bi)
S = typeof(el)
if (S !== T) && !(S <: T)
R = typejoin(T, S)
new = similar(dest, R)
copy!(new,1, dest,1, i-1)
@inbounds new[i] = el
return map_to!(f, i+1, new, A, B)
end
@inbounds dest[i] = el::T
end
return dest
end

function map(f, A::AbstractArray, B::AbstractArray)
shp = promote_shape(size(A),size(B))
if prod(shp) == 0
return similar(A, promote_type(eltype(A),eltype(B)), shp)
end
first = f(A[1], B[1])
dest = similar(A, typeof(first), shp)
dest[1] = first
return map_to!(f, 2, dest, A, B)
end

## N argument

ith_all(i, ::Tuple{}) = ()
Expand All @@ -1310,32 +1193,7 @@ end

map!{F}(f::F, dest::AbstractArray, As::AbstractArray...) = map_n!(f, dest, As)

function map_to_n!{T,F}(f::F, offs, dest::AbstractArray{T}, As)
for i = offs:length(As[1])
el = f(ith_all(i, As)...)
S = typeof(el)
if (S !== T) && !(S <: T)
R = typejoin(T, S)
new = similar(dest, R)
copy!(new,1, dest,1, i-1)
@inbounds new[i] = el
return map_to_n!(f, i+1, new, As)
end
@inbounds dest[i] = el::T
end
return dest
end

function map(f, As::AbstractArray...)
shape = mapreduce(size, promote_shape, As)
if prod(shape) == 0
return similar(As[1], promote_eltype(As...), shape)
end
first = f(map(a->a[1], As)...)
dest = similar(As[1], typeof(first), shape)
dest[1] = first
return map_to_n!(f, 2, dest, As)
end
map(f, iters...) = collect(Generator(f, iters...))

# multi-item push!, unshift! (built on top of type-specific 1-item version)
# (note: must not cause a dispatch loop when 1-item case is not defined)
Expand Down
100 changes: 82 additions & 18 deletions base/array.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,35 +199,99 @@ convert{T,n,S}(::Type{Array{T,n}}, x::AbstractArray{S,n}) = copy!(Array(T, size(

promote_rule{T,n,S}(::Type{Array{T,n}}, ::Type{Array{S,n}}) = Array{promote_type(T,S),n}

## copying iterators to containers

# make a collection similar to `c` and appropriate for collecting `itr`
_similar_for(c::AbstractArray, T, itr, ::SizeUnknown) = similar(c, T, 0)
_similar_for(c::AbstractArray, T, itr, ::HasLength) = similar(c, T, Int(length(itr)::Integer))
_similar_for(c::AbstractArray, T, itr, ::HasShape) = similar(c, T, convert(Dims,size(itr)))
_similar_for(c, T, itr, isz) = similar(c, T)

"""
collect(element_type, collection)
Return an array of type `Array{element_type,1}` of all items in a collection.
"""
function collect{T}(::Type{T}, itr)
if applicable(length, itr)
# when length() isn't defined this branch might pollute the
# type of the other.
a = Array(T,length(itr)::Integer)
i = 0
for x in itr
a[i+=1] = x
end
else
a = Array(T,0)
for x in itr
push!(a,x)
end
end
return a
end
collect{T}(::Type{T}, itr) = collect(Generator(T, itr))

"""
collect(collection)
Return an array of all items in a collection. For associative collections, returns Pair{KeyType, ValType}.
"""
collect(itr) = collect(eltype(itr), itr)
collect(itr) = _collect(1:1 #= Array =#, itr, iteratoreltype(itr), iteratorsize(itr))

collect_similar(cont, itr) = _collect(cont, itr, iteratoreltype(itr), iteratorsize(itr))

_collect(cont, itr, ::HasEltype, isz::Union{HasLength,HasShape}) =
copy!(_similar_for(cont, eltype(itr), itr, isz), itr)

function _collect(cont, itr, ::HasEltype, isz::SizeUnknown)
a = _similar_for(cont, eltype(itr), itr, isz)
for x in itr
push!(a,x)
end
return a
end

_collect(c, itr, ::EltypeUnknown, isz::SizeUnknown) = grow_to!(_similar_for(c, Union{}, itr, isz), itr)

function _collect(c, itr, ::EltypeUnknown, isz::Union{HasLength,HasShape})
st = start(itr)
if done(itr,st)
return _similar_for(c, Union{}, itr, isz)
end
v1, st = next(itr, st)
collect_to_with_first!(_similar_for(c, typeof(v1), itr, isz), v1, itr, st)
end

function collect_to_with_first!(dest::AbstractArray, v1, itr, st)
dest[1] = v1
return collect_to!(dest, itr, 2, st)
end

function collect_to_with_first!(dest, v1, itr, st)
push!(dest, v1)
return grow_to!(dest, itr, st)
end

function collect_to!{T}(dest::AbstractArray{T}, itr, offs, st)
# collect to dest array, checking the type of each result. if a result does not
# match, widen the result type and re-dispatch.
i = offs
while !done(itr, st)
el, st = next(itr, st)
S = typeof(el)
if S === T || S <: T
@inbounds dest[i] = el::T
i += 1
else
R = typejoin(T, S)
new = similar(dest, R)
copy!(new,1, dest,1, i-1)
@inbounds new[i] = el
return collect_to!(new, itr, i+1, st)
end
end
return dest
end

function grow_to!(dest, itr, st = start(itr))
T = eltype(dest)
while !done(itr, st)
el, st = next(itr, st)
S = typeof(el)
if S === T || S <: T
push!(dest, el::T)
else
new = similar(dest, typejoin(T, S))
copy!(new, dest)
push!(new, el)
return grow_to!(new, itr, st)
end
end
return dest
end

## Iteration ##
start(A::Array) = 1
Expand Down
2 changes: 2 additions & 0 deletions base/channels.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,3 +95,5 @@ function done(c::Channel, state::Ref)
end
end
next{T}(c::Channel{T}, state) = (v=get(state[]); state[]=nothing; (v, state))

iteratorsize{C<:Channel}(::Type{C}) = SizeUnknown()
45 changes: 29 additions & 16 deletions base/dict.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@

# generic operations on associative collections

abstract Associative{K,V}

const secret_table_token = :__c782dbf1cf4d6a2e5e3865d7e95634f2e09b5902__

haskey(d::Associative, k) = in(k,keys(d))
Expand Down Expand Up @@ -225,6 +223,16 @@ function merge!(d::Associative, others::Associative...)
end
return d
end

# very similar to `merge!`, but accepts any iterable and extends code
# that would otherwise only use `copy!` with arrays.
function copy!(dest::Union{Associative,AbstractSet}, src)
for x in src
push!(dest, x)
end
return dest
end

keytype{K,V}(::Type{Associative{K,V}}) = K
keytype(a::Associative) = keytype(typeof(a))
keytype{A<:Associative}(::Type{A}) = keytype(supertype(A))
Expand Down Expand Up @@ -449,19 +457,6 @@ copy(d::Dict) = Dict(d)

const AnyDict = Dict{Any,Any}

# TODO: this can probably be simplified using `eltype` as a THT (Tim Holy trait)
Dict{K,V}(kv::Tuple{Vararg{Tuple{K,V}}}) = Dict{K,V}(kv)
Dict{K }(kv::Tuple{Vararg{Tuple{K,Any}}}) = Dict{K,Any}(kv)
Dict{V }(kv::Tuple{Vararg{Tuple{Any,V}}}) = Dict{Any,V}(kv)
Dict{K,V}(kv::Tuple{Vararg{Pair{K,V}}}) = Dict{K,V}(kv)
Dict{K }(kv::Tuple{Vararg{Pair{K}}}) = Dict{K,Any}(kv)
Dict{V }(kv::Tuple{Vararg{Pair{TypeVar(:K),V}}}) = Dict{Any,V}(kv)
Dict( kv::Tuple{Vararg{Pair}}) = Dict{Any,Any}(kv)

Dict{K,V}(kv::AbstractArray{Tuple{K,V}}) = Dict{K,V}(kv)
Dict{K,V}(kv::AbstractArray{Pair{K,V}}) = Dict{K,V}(kv)
Dict{K,V}(kv::Associative{K,V}) = Dict{K,V}(kv)

Dict{K,V}(ps::Pair{K,V}...) = Dict{K,V}(ps)
Dict{K }(ps::Pair{K}...,) = Dict{K,Any}(ps)
Dict{V }(ps::Pair{TypeVar(:K),V}...,) = Dict{Any,V}(ps)
Expand All @@ -482,9 +477,27 @@ end

dict_with_eltype{K,V}(kv, ::Type{Tuple{K,V}}) = Dict{K,V}(kv)
dict_with_eltype{K,V}(kv, ::Type{Pair{K,V}}) = Dict{K,V}(kv)
dict_with_eltype(kv, t) = Dict{Any,Any}(kv)
dict_with_eltype(kv, t) = grow_to!(Dict{Union{},Union{}}(), kv)

# this is a special case due to (1) allowing both Pairs and Tuples as elements,
# and (2) Pair being invariant. a bit annoying.
function grow_to!{K,V}(dest::Associative{K,V}, itr, st = start(itr))
while !done(itr, st)
(k,v), st = next(itr, st)
if isa(k,K) && isa(v,V)
dest[k] = v
else
new = similar(dest, Pair{typejoin(K,typeof(k)), typejoin(V,typeof(v))})
copy!(new, dest)
new[k] = v
return grow_to!(new, itr, st)
end
end
return dest
end

similar{K,V}(d::Dict{K,V}) = Dict{K,V}()
similar{K,V}(d::Dict, ::Type{Pair{K,V}}) = Dict{K,V}()

# conversion between Dict types
function convert{K,V}(::Type{Dict{K,V}},d::Associative)
Expand Down
3 changes: 3 additions & 0 deletions base/essentials.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ typealias Callable Union{Function,DataType}

const Bottom = Union{}

abstract AbstractSet{T}
abstract Associative{K,V}

# The real @inline macro is not available until after array.jl, so this
# internal macro splices the meta Expr directly into the function body.
macro _inline_meta()
Expand Down
Loading

0 comments on commit a2ce230

Please sign in to comment.