Merge pull request #18839 from JuliaLang/jb/itertools

RFC: add `Base.Iterators` module

NEWS.md

@@ -49,6 +49,10 @@ Library improvements
     One way of doing this is by adding `ENV["JULIA_INFO_COLOR"] = :blue` to the `.juliarc.jl` file.
     For more information regarding customizing colors in the REPL, see this [manual section]( http://docs.julialang.org/en/latest/manual/interacting-with-julia/#customizing-colors).
 
+  * Iteration utilities that wrap iterators and return other iterators (`enumerate`, `zip`, `rest`,
+    `countfrom`, `take`, `drop`, `cycle`, `repeated`, `product`, `flatten`, `partition`) have been
+    moved to the module `Base.Iterators` ([#18839]).
+
 Compiler/Runtime improvements
 -----------------------------
 

base/REPLCompletions.jl

@@ -376,10 +376,10 @@ function bslash_completions(string, pos)
         # return possible matches; these cannot be mixed with regular
         # Julian completions as only latex / emoji symbols contain the leading \
         if startswith(s, "\\:") # emoji
-            emoji_names = filter(k -> startswith(k, s), keys(emoji_symbols))
+            emoji_names = Iterators.filter(k -> startswith(k, s), keys(emoji_symbols))
             return (true, (sort!(collect(emoji_names)), slashpos:pos, true))
         else # latex
-            latex_names = filter(k -> startswith(k, s), keys(latex_symbols))
+            latex_names = Iterators.filter(k -> startswith(k, s), keys(latex_symbols))
             return (true, (sort!(collect(latex_names)), slashpos:pos, true))
         end
     end

base/asyncmap.jl

@@ -1,5 +1,6 @@
 # This file is a part of Julia. License is MIT: http://julialang.org/license
 
+using Base.Iterators.Enumerate
 
 """
     AsyncCollector(f, results, c...; ntasks=0) -> iterator

base/broadcast.jl

@@ -226,7 +226,7 @@ end
             else
                 R = typejoin(eltype(B), S)
                 new = similar(B, R)
-                for II in take(iter, count)
+                for II in Iterators.take(iter, count)
                     new[II] = B[II]
                 end
                 new[I] = V

base/client.jl

@@ -23,7 +23,7 @@ end
 # Create a docstring with an automatically generated list
 # of colors.
 const possible_formatting_symbols = [:normal, :bold]
-available_text_colors = collect(filter(x -> !isa(x, Integer), keys(text_colors)))
+available_text_colors = collect(Iterators.filter(x -> !isa(x, Integer), keys(text_colors)))
 available_text_colors = cat(1,
     sort(intersect(available_text_colors, possible_formatting_symbols), rev=true),
     sort(setdiff(  available_text_colors, possible_formatting_symbols)))

base/dates/Dates.jl

@@ -4,6 +4,8 @@ module Dates
 
 importall ..Base.Operators
 
+using Base.Iterators
+
 include("types.jl")
 include("periods.jl")
 include("accessors.jl")

base/deprecated.jl

@@ -1023,4 +1023,14 @@ end))
 
 @deprecate is (===)
 
+@deprecate_binding Filter    Iterators.Filter
+@deprecate_binding Zip       Iterators.Zip
+@deprecate filter(flt, itr)  Iterators.filter(flt, itr)
+@deprecate_binding rest      Iterators.rest
+@deprecate_binding countfrom Iterators.countfrom
+@deprecate_binding take      Iterators.take
+@deprecate_binding drop      Iterators.drop
+@deprecate_binding cycle     Iterators.cycle
+@deprecate_binding repeated  Iterators.repeated
+
 # End deprecations scheduled for 0.6

base/exports.jl

@@ -23,6 +23,7 @@ export
     Docs,
     Markdown,
     Threads,
+    Iterators,
 
 # Types
     AbstractChannel,
@@ -61,7 +62,6 @@ export
     Enumerate,
     Factorization,
     FileMonitor,
-    Filter,
     FloatRange,
     Future,
     Hermitian,
@@ -124,7 +124,6 @@ export
     VersionNumber,
     WeakKeyDict,
     WorkerConfig,
-    Zip,
 
 # Ccall types
     Cchar,
@@ -958,16 +957,11 @@ export
 
 # iteration
     done,
-    enumerate,
     next,
     start,
+
+    enumerate,  # re-exported from Iterators
     zip,
-    rest,
-    countfrom,
-    take,
-    drop,
-    cycle,
-    repeated,
 
 # object identity and equality
     copy,

base/generator.jl

@@ -55,11 +55,6 @@ result, and algorithms that resize their result incrementally.
 iteratorsize(x) = iteratorsize(typeof(x))
 iteratorsize(::Type) = HasLength()  # HasLength is the default
 
-and_iteratorsize{T}(isz::T, ::T) = isz
-and_iteratorsize(::HasLength, ::HasShape) = HasLength()
-and_iteratorsize(::HasShape, ::HasLength) = HasLength()
-and_iteratorsize(a, b) = SizeUnknown()
-
 abstract IteratorEltype
 immutable EltypeUnknown <: IteratorEltype end
 immutable HasEltype <: IteratorEltype end
@@ -81,9 +76,6 @@ values.
 iteratoreltype(x) = iteratoreltype(typeof(x))
 iteratoreltype(::Type) = HasEltype()  # HasEltype is the default
 
-and_iteratoreltype{T}(iel::T, ::T) = iel
-and_iteratoreltype(a, b) = EltypeUnknown()
-
 iteratorsize{T<:AbstractArray}(::Type{T}) = HasShape()
 iteratorsize{I,F}(::Type{Generator{I,F}}) = iteratorsize(I)
 length(g::Generator) = length(g.iter)

base/iterator.jl → base/iterators.jl

@@ -1,5 +1,13 @@
 # This file is a part of Julia. License is MIT: http://julialang.org/license
 
+module Iterators
+
+import Base: start, done, next, isempty, length, size, eltype, iteratorsize, iteratoreltype, indices, ndims
+
+using Base: tuple_type_cons, SizeUnknown, HasLength, HasShape, IsInfinite, EltypeUnknown, HasEltype, OneTo
+
+export enumerate, zip, rest, countfrom, take, drop, cycle, repeated, product, flatten, partition
+
 _min_length(a, b, ::IsInfinite, ::IsInfinite) = min(length(a),length(b)) # inherit behaviour, error
 _min_length(a, b, A, ::IsInfinite) = length(a)
 _min_length(a, b, ::IsInfinite, B) = length(b)
@@ -10,6 +18,14 @@ _diff_length(a, b, ::IsInfinite, ::IsInfinite) = 0
 _diff_length(a, b, ::IsInfinite, B) = length(a) # inherit behaviour, error
 _diff_length(a, b, A, B) = max(length(a)-length(b), 0)
 
+and_iteratorsize{T}(isz::T, ::T) = isz
+and_iteratorsize(::HasLength, ::HasShape) = HasLength()
+and_iteratorsize(::HasShape, ::HasLength) = HasLength()
+and_iteratorsize(a, b) = SizeUnknown()
+
+and_iteratoreltype{T}(iel::T, ::T) = iel
+and_iteratoreltype(a, b) = EltypeUnknown()
+
 # enumerate
 
 immutable Enumerate{I}
@@ -238,42 +254,6 @@ rest_iteratorsize(::IsInfinite) = IsInfinite()
 iteratorsize{I,S}(::Type{Rest{I,S}}) = rest_iteratorsize(iteratorsize(I))
 
 
-"""
-    head_and_tail(c, n) -> head, tail
-
-Returns `head`: the first `n` elements of `c`;
-and `tail`: an iterator over the remaining elements.
-
-```jldoctest
-julia> a = 1:10
-1:10
-
-julia> b, c = Base.head_and_tail(a, 3)
-([1,2,3],Base.Rest{UnitRange{Int64},Int64}(1:10,4))
-
-julia> collect(c)
-7-element Array{Any,1}:
-  4
-  5
-  6
-  7
-  8
-  9
- 10
-```
-"""
-function head_and_tail(c, n)
-    head = Vector{eltype(c)}(n)
-    s = start(c)
-    i = 0
-    while i < n && !done(c, s)
-        i += 1
-        head[i], s = next(c, s)
-    end
-    return resize!(head, i), rest(c, s)
-end
-
-
 # Count -- infinite counting
 
 immutable Count{S<:Number}
@@ -677,7 +657,7 @@ end
 
 
 """
-    partition(collection, n) -> iterator
+    partition(collection, n)
 
 Iterate over a collection `n` elements at a time.
 
@@ -723,3 +703,5 @@ function next(itr::PartitionIterator, state)
     end
     return resize!(v, i), state
 end
+
+end

base/multidimensional.jl

@@ -381,7 +381,7 @@ end
 # and ensure the value to set is either an AbstractArray or a Repeated scalar
 # before redispatching to the _unsafe_batchsetindex!
 _iterable(v::AbstractArray) = v
-_iterable(v) = repeated(v)
+_iterable(v) = Iterators.repeated(v)
 @inline function _setindex!{T,N}(l::LinearIndexing, A::AbstractArray{T,N}, x, J::Vararg{Union{Real,AbstractArray,Colon},N})
     @boundscheck checkbounds(A, J...)
     _unsafe_setindex!(l, A, x, J...)

base/pkg/resolve.jl

@@ -66,7 +66,7 @@ function sanity_check(deps::Dict{String,Dict{VersionNumber,Available}},
 
     vers = Array{Tuple{String,VersionNumber,VersionNumber}}(0)
     for (p,d) in deps, vn in keys(d)
-        lvns = VersionNumber[filter(vn2->(vn2>vn), keys(d))...]
+        lvns = VersionNumber[Iterators.filter(vn2->(vn2>vn), keys(d))...]
         nvn = isempty(lvns) ? typemax(VersionNumber) : minimum(lvns)
         push!(vers, (p,vn,nvn))
     end

base/pmap.jl

@@ -24,7 +24,7 @@ function pgenerate(p::WorkerPool, f, c)
         return AsyncGenerator(f, c; ntasks=()->nworkers(p))
     end
     batches = batchsplit(c, min_batch_count = length(p) * 3)
-    return flatten(AsyncGenerator(remote(p, b -> asyncmap(f, b)), batches))
+    return Iterators.flatten(AsyncGenerator(remote(p, b -> asyncmap(f, b)), batches))
 end
 pgenerate(p::WorkerPool, f, c1, c...) = pgenerate(p, a->f(a...), zip(c1, c...))
 pgenerate(f, c) = pgenerate(default_worker_pool(), f, c)
@@ -133,7 +133,7 @@ function pmap(p::AbstractWorkerPool, f, c; distributed=true, batch_size=1, on_er
             f = wrap_on_error(f, (x,e)->BatchProcessingError(x,e); capture_data=true)
         end
         f = wrap_batch(f, p, on_error)
-        results = collect(flatten(AsyncGenerator(f, batches; ntasks=()->nworkers(p))))
+        results = collect(Iterators.flatten(AsyncGenerator(f, batches; ntasks=()->nworkers(p))))
         if (on_error !== nothing) || (retry_n > 0)
             process_batch_errors!(p, f_orig, results, on_error, retry_on, retry_n, retry_max_delay)
         end
@@ -213,6 +213,40 @@ function process_batch_errors!(p, f, results, on_error, retry_on, retry_n, retry
     nothing
 end
 
+"""
+    head_and_tail(c, n) -> head, tail
+
+Returns `head`: the first `n` elements of `c`;
+and `tail`: an iterator over the remaining elements.
+
+```jldoctest
+julia> a = 1:10
+1:10
+
+julia> b, c = Base.head_and_tail(a, 3)
+([1,2,3],Base.Iterators.Rest{UnitRange{Int64},Int64}(1:10,4))
+
+julia> collect(c)
+7-element Array{Any,1}:
+  4
+  5
+  6
+  7
+  8
+  9
+ 10
+```
+"""
+function head_and_tail(c, n)
+    head = Vector{eltype(c)}(n)
+    s = start(c)
+    i = 0
+    while i < n && !done(c, s)
+        i += 1
+        head[i], s = next(c, s)
+    end
+    return resize!(head, i), Iterators.rest(c, s)
+end
 
 """
     batchsplit(c; min_batch_count=1, max_batch_size=100) -> iterator
@@ -231,14 +265,14 @@ function batchsplit(c; min_batch_count=1, max_batch_size=100)
     end
 
     # Split collection into batches, then peek at the first few batches
-    batches = partition(c, max_batch_size)
+    batches = Iterators.partition(c, max_batch_size)
     head, tail = head_and_tail(batches, min_batch_count)
 
     # If there are not enough batches, use a smaller batch size
     if length(head) < min_batch_count
         batch_size = max(1, div(sum(length, head), min_batch_count))
-        return partition(collect(flatten(head)), batch_size)
+        return Iterators.partition(collect(Iterators.flatten(head)), batch_size)
     end
 
-    return flatten((head, tail))
+    return Iterators.flatten((head, tail))
 end

base/promotion.jl

@@ -226,7 +226,7 @@ if isdefined(Core, :Inference)
     end
     function _promote_op(op, R::ANY, S::ANY)
         F = typeof(a -> op(a...))
-        G = Tuple{Generator{Zip2{Tuple{R},Tuple{S}},F}}
+        G = Tuple{Generator{Iterators.Zip2{Tuple{R},Tuple{S}},F}}
         return Core.Inference.return_type(first, G)
     end
 else

base/show.jl

@@ -156,7 +156,7 @@ function is_exported_from_stdlib(name::Symbol, mod::Module)
         end
         mod = parent
     end
-    return isexported(mod, name) && isdefined(mod, name) && getfield(mod, name) === orig
+    return isexported(mod, name) && isdefined(mod, name) && !isdeprecated(mod, name) && getfield(mod, name) === orig
 end
 
 function show(io::IO, f::Function)

base/sysimg.jl

@@ -129,7 +129,9 @@ include("intset.jl")
 include("associative.jl")
 include("dict.jl")
 include("set.jl")
-include("iterator.jl")
+include("iterators.jl")
+using .Iterators: zip, enumerate
+using .Iterators: Flatten, product  # for generators
 
 # Definition of StridedArray
 typealias StridedReshapedArray{T,N,A<:DenseArray} ReshapedArray{T,N,A}