add IterTools module around extra iterator functionality

Makefile

@@ -198,7 +198,6 @@ CORE_SRCS := $(addprefix $(JULIAHOME)/, \
 		base/inference.jl \
 		base/int.jl \
 		base/intset.jl \
-		base/iterator.jl \
 		base/nofloat_hashing.jl \
 		base/number.jl \
 		base/operators.jl \

NEWS.md

@@ -49,6 +49,10 @@ Library improvements
 
   * `max`, `min`, and related functions (`minmax`, `maximum`, `minimum`, `extrema`) now return `NaN` for `NaN` arguments ([#12563]).
 
+  * Iteration utilities that wrap iterators and return other iterators (`enumerate`, `zip`, `rest`,
+    `countfrom`, `take`, `drop`, `cycle`, `repeated`, `product`, `flatten`, `partition`) have been
+    moved to the module `Base.IterTools` ([#18839]).
+
 Compiler/Runtime improvements
 -----------------------------
 

base/REPLCompletions.jl

@@ -376,10 +376,10 @@ function bslash_completions(string, pos)
         # return possible matches; these cannot be mixed with regular
         # Julian completions as only latex / emoji symbols contain the leading \
         if startswith(s, "\\:") # emoji
-            emoji_names = filter(k -> startswith(k, s), keys(emoji_symbols))
+            emoji_names = IterTools.filter(k -> startswith(k, s), keys(emoji_symbols))
             return (true, (sort!(collect(emoji_names)), slashpos:pos, true))
         else # latex
-            latex_names = filter(k -> startswith(k, s), keys(latex_symbols))
+            latex_names = IterTools.filter(k -> startswith(k, s), keys(latex_symbols))
             return (true, (sort!(collect(latex_names)), slashpos:pos, true))
         end
     end

base/asyncmap.jl

@@ -1,5 +1,6 @@
 # This file is a part of Julia. License is MIT: http://julialang.org/license
 
+using Base.IterTools.Enumerate
 
 """
     AsyncCollector(f, results, c...; ntasks=0) -> iterator

base/broadcast.jl

@@ -226,7 +226,7 @@ end
             else
                 R = typejoin(eltype(B), S)
                 new = similar(B, R)
-                for II in take(iter, count)
+                for II in IterTools.take(iter, count)
                     new[II] = B[II]
                 end
                 new[I] = V

base/client.jl

@@ -23,7 +23,7 @@ end
 # Create a docstring with an automatically generated list
 # of colors.
 const possible_formatting_symbols = [:normal, :bold]
-available_text_colors = collect(filter(x -> !isa(x, Integer), keys(text_colors)))
+available_text_colors = collect(IterTools.filter(x -> !isa(x, Integer), keys(text_colors)))
 available_text_colors = cat(1,
     sort(intersect(available_text_colors, possible_formatting_symbols), rev=true),
     sort(setdiff(  available_text_colors, possible_formatting_symbols)))

base/coreimg.jl

@@ -60,7 +60,6 @@ include("reduce.jl")
 ## core structures
 include("intset.jl")
 include("dict.jl")
-include("iterator.jl")
 
 # core docsystem
 include("docs/core.jl")

base/dates/Dates.jl

@@ -4,6 +4,8 @@ module Dates
 
 importall ..Base.Operators
 
+using Base.IterTools
+
 include("types.jl")
 include("periods.jl")
 include("accessors.jl")

base/deprecated.jl

@@ -1019,4 +1019,14 @@ eval(Multimedia, :(macro textmime(mime)
     end
 end))
 
+@deprecate_binding Filter    IterTools.Filter
+@deprecate_binding Zip       IterTools.Zip
+@deprecate filter(flt, itr)  IterTools.filter(flt, itr)
+@deprecate_binding rest      IterTools.rest
+@deprecate_binding countfrom IterTools.countfrom
+@deprecate_binding take      IterTools.take
+@deprecate_binding drop      IterTools.drop
+@deprecate_binding cycle     IterTools.cycle
+@deprecate_binding repeated  IterTools.repeated
+
 # End deprecations scheduled for 0.6

base/essentials.jl

@@ -63,6 +63,12 @@ function tuple_type_tail(T::DataType)
     return Tuple{argtail(T.parameters...)...}
 end
 
+tuple_type_cons{S}(::Type{S}, ::Type{Union{}}) = Union{}
+function tuple_type_cons{S,T<:Tuple}(::Type{S}, ::Type{T})
+    @_pure_meta
+    Tuple{S, T.parameters...}
+end
+
 isvarargtype(t::ANY) = isa(t, DataType) && is((t::DataType).name, Vararg.name)
 isvatuple(t::DataType) = (n = length(t.parameters); n > 0 && isvarargtype(t.parameters[n]))
 unwrapva(t::ANY) = isvarargtype(t) ? t.parameters[1] : t
@@ -231,3 +237,5 @@ function vector_any(xs::ANY...)
     end
     a
 end
+
+isempty(itr) = done(itr, start(itr))

base/exports.jl

@@ -23,6 +23,7 @@ export
     Docs,
     Markdown,
     Threads,
+    IterTools,
 
 # Types
     AbstractChannel,
@@ -61,7 +62,6 @@ export
     Enumerate,
     Factorization,
     FileMonitor,
-    Filter,
     FloatRange,
     Future,
     Hermitian,
@@ -124,7 +124,6 @@ export
     VersionNumber,
     WeakKeyDict,
     WorkerConfig,
-    Zip,
 
 # Ccall types
     Cchar,
@@ -959,16 +958,11 @@ export
 
 # iteration
     done,
-    enumerate,
     next,
     start,
+
+    enumerate,  # re-exported from IterTools
     zip,
-    rest,
-    countfrom,
-    take,
-    drop,
-    cycle,
-    repeated,
 
 # object identity and equality
     copy,

base/generator.jl

@@ -55,11 +55,6 @@ result, and algorithms that resize their result incrementally.
 iteratorsize(x) = iteratorsize(typeof(x))
 iteratorsize(::Type) = HasLength()  # HasLength is the default
 
-and_iteratorsize{T}(isz::T, ::T) = isz
-and_iteratorsize(::HasLength, ::HasShape) = HasLength()
-and_iteratorsize(::HasShape, ::HasLength) = HasLength()
-and_iteratorsize(a, b) = SizeUnknown()
-
 abstract IteratorEltype
 immutable EltypeUnknown <: IteratorEltype end
 immutable HasEltype <: IteratorEltype end
@@ -81,9 +76,6 @@ values.
 iteratoreltype(x) = iteratoreltype(typeof(x))
 iteratoreltype(::Type) = HasEltype()  # HasEltype is the default
 
-and_iteratoreltype{T}(iel::T, ::T) = iel
-and_iteratoreltype(a, b) = EltypeUnknown()
-
 iteratorsize{T<:AbstractArray}(::Type{T}) = HasShape()
 iteratorsize{I,F}(::Type{Generator{I,F}}) = iteratorsize(I)
 length(g::Generator) = length(g.iter)

base/inference.jl

@@ -1017,7 +1017,8 @@ function abstract_apply(af::ANY, fargs, aargtypes::Vector{Any}, vtypes::VarTable
 end
 
 function pure_eval_call(f::ANY, argtypes::ANY, atype::ANY, vtypes::VarTable, sv::InferenceState)
-    for a in drop(argtypes,1)
+    for i = 2:length(argtypes)
+        a = argtypes[i]
         if !(isa(a,Const) || isconstType(a,false))
             return false
         end
@@ -1053,7 +1054,7 @@ function pure_eval_call(f::ANY, argtypes::ANY, atype::ANY, vtypes::VarTable, sv:
         return false
     end
 
-    args = Any[ isa(a,Const) ? a.val : a.parameters[1] for a in drop(argtypes,1) ]
+    args = Any[ (a=argtypes[i]; isa(a,Const) ? a.val : a.parameters[1]) for i in 2:length(argtypes) ]
     try
         return abstract_eval_constant(f(args...))
     catch

base/iterator.jl → base/itertools.jl

@@ -1,6 +1,14 @@
 # This file is a part of Julia. License is MIT: http://julialang.org/license
 
-isempty(itr) = done(itr, start(itr))
+module IterTools
+
+global Filter
+
+import Base: start, done, next, isempty, length, size, eltype, iteratorsize, iteratoreltype, indices, ndims
+
+using Base: tuple_type_cons, SizeUnknown, HasLength, HasShape, IsInfinite, EltypeUnknown, HasEltype, OneTo
+
+export enumerate, zip, rest, countfrom, take, drop, cycle, repeated, product, flatten, partition
 
 _min_length(a, b, ::IsInfinite, ::IsInfinite) = min(length(a),length(b)) # inherit behaviour, error
 _min_length(a, b, A, ::IsInfinite) = length(a)
@@ -12,6 +20,14 @@ _diff_length(a, b, ::IsInfinite, ::IsInfinite) = 0
 _diff_length(a, b, ::IsInfinite, B) = length(a) # inherit behaviour, error
 _diff_length(a, b, A, B) = max(length(a)-length(b), 0)
 
+and_iteratorsize{T}(isz::T, ::T) = isz
+and_iteratorsize(::HasLength, ::HasShape) = HasLength()
+and_iteratorsize(::HasShape, ::HasLength) = HasLength()
+and_iteratorsize(a, b) = SizeUnknown()
+
+and_iteratoreltype{T}(iel::T, ::T) = iel
+and_iteratoreltype(a, b) = EltypeUnknown()
+
 # enumerate
 
 immutable Enumerate{I}
@@ -143,11 +159,6 @@ zip(a, b, c...) = Zip(a, zip(b, c...))
 length(z::Zip) = _min_length(z.a, z.z, iteratorsize(z.a), iteratorsize(z.z))
 size(z::Zip) = promote_shape(size(z.a), size(z.z))
 indices(z::Zip) = promote_shape(indices(z.a), indices(z.z))
-tuple_type_cons{S}(::Type{S}, ::Type{Union{}}) = Union{}
-function tuple_type_cons{S,T<:Tuple}(::Type{S}, ::Type{T})
-    @_pure_meta
-    Tuple{S, T.parameters...}
-end
 eltype{I,Z}(::Type{Zip{I,Z}}) = tuple_type_cons(eltype(I), eltype(Z))
 @inline start(z::Zip) = tuple(start(z.a), start(z.z))
 @inline function next(z::Zip, st)
@@ -245,42 +256,6 @@ rest_iteratorsize(::IsInfinite) = IsInfinite()
 iteratorsize{I,S}(::Type{Rest{I,S}}) = rest_iteratorsize(iteratorsize(I))
 
 
-"""
-    head_and_tail(c, n) -> head, tail
-
-Returns `head`: the first `n` elements of `c`;
-and `tail`: an iterator over the remaining elements.
-
-```jldoctest
-julia> a = 1:10
-1:10
-
-julia> b, c = Base.head_and_tail(a, 3)
-([1,2,3],Base.Rest{UnitRange{Int64},Int64}(1:10,4))
-
-julia> collect(c)
-7-element Array{Any,1}:
-  4
-  5
-  6
-  7
-  8
-  9
- 10
-```
-"""
-function head_and_tail(c, n)
-    head = Vector{eltype(c)}(n)
-    s = start(c)
-    i = 0
-    while i < n && !done(c, s)
-        i += 1
-        head[i], s = next(c, s)
-    end
-    return resize!(head, i), rest(c, s)
-end
-
-
 # Count -- infinite counting
 
 immutable Count{S<:Number}
@@ -684,7 +659,7 @@ end
 
 
 """
-    partition(collection, n) -> iterator
+    partition(collection, n)
 
 Iterate over a collection `n` elements at a time.
 
@@ -730,3 +705,5 @@ function next(itr::PartitionIterator, state)
     end
     return resize!(v, i), state
 end
+
+end

base/multidimensional.jl

@@ -381,7 +381,7 @@ end
 # and ensure the value to set is either an AbstractArray or a Repeated scalar
 # before redispatching to the _unsafe_batchsetindex!
 _iterable(v::AbstractArray) = v
-_iterable(v) = repeated(v)
+_iterable(v) = IterTools.repeated(v)
 @inline function _setindex!{T,N}(l::LinearIndexing, A::AbstractArray{T,N}, x, J::Vararg{Union{Real,AbstractArray,Colon},N})
     @boundscheck checkbounds(A, J...)
     _unsafe_setindex!(l, A, x, J...)

base/pkg/resolve.jl

@@ -66,7 +66,7 @@ function sanity_check(deps::Dict{String,Dict{VersionNumber,Available}},
 
     vers = Array{Tuple{String,VersionNumber,VersionNumber}}(0)
     for (p,d) in deps, vn in keys(d)
-        lvns = VersionNumber[filter(vn2->(vn2>vn), keys(d))...]
+        lvns = VersionNumber[IterTools.filter(vn2->(vn2>vn), keys(d))...]
         nvn = isempty(lvns) ? typemax(VersionNumber) : minimum(lvns)
         push!(vers, (p,vn,nvn))
     end

base/pmap.jl

@@ -213,6 +213,40 @@ function process_batch_errors!(p, f, results, on_error, retry_on, retry_n, retry
     nothing
 end
 
+"""
+    head_and_tail(c, n) -> head, tail
+
+Returns `head`: the first `n` elements of `c`;
+and `tail`: an iterator over the remaining elements.
+
+```jldoctest
+julia> a = 1:10
+1:10
+
+julia> b, c = Base.head_and_tail(a, 3)
+([1,2,3],Base.IterTools.Rest{UnitRange{Int64},Int64}(1:10,4))
+
+julia> collect(c)
+7-element Array{Any,1}:
+  4
+  5
+  6
+  7
+  8
+  9
+ 10
+```
+"""
+function head_and_tail(c, n)
+    head = Vector{eltype(c)}(n)
+    s = start(c)
+    i = 0
+    while i < n && !done(c, s)
+        i += 1
+        head[i], s = next(c, s)
+    end
+    return resize!(head, i), IterTools.rest(c, s)
+end
 
 """
     batchsplit(c; min_batch_count=1, max_batch_size=100) -> iterator
@@ -231,13 +265,13 @@ function batchsplit(c; min_batch_count=1, max_batch_size=100)
     end
 
     # Split collection into batches, then peek at the first few batches
-    batches = partition(c, max_batch_size)
+    batches = IterTools.partition(c, max_batch_size)
     head, tail = head_and_tail(batches, min_batch_count)
 
     # If there are not enough batches, use a smaller batch size
     if length(head) < min_batch_count
         batch_size = max(1, div(sum(length, head), min_batch_count))
-        return partition(collect(flatten(head)), batch_size)
+        return IterTools.partition(collect(flatten(head)), batch_size)
     end
 
     return flatten((head, tail))

base/promotion.jl

@@ -226,7 +226,7 @@ if isdefined(Core, :Inference)
     end
     function _promote_op(op, R::ANY, S::ANY)
         F = typeof(a -> op(a...))
-        G = Tuple{Generator{Zip2{Tuple{R},Tuple{S}},F}}
+        G = Tuple{Generator{IterTools.Zip2{Tuple{R},Tuple{S}},F}}
         return Core.Inference.return_type(first, G)
     end
 else

base/sysimg.jl

@@ -128,7 +128,9 @@ include("bitarray.jl")
 include("intset.jl")
 include("dict.jl")
 include("set.jl")
-include("iterator.jl")
+include("itertools.jl")
+using .IterTools: zip, enumerate
+using .IterTools: Flatten, product  # for generators
 
 # Definition of StridedArray
 typealias StridedReshapedArray{T,N,A<:DenseArray} ReshapedArray{T,N,A}