From 6221277774d53cb0e55e9c978ae0aeb55565798d Mon Sep 17 00:00:00 2001 From: Lorenzo Sindoni Date: Fri, 31 Aug 2018 07:40:37 +0200 Subject: [PATCH] Changes for Julia 0.7/1.0 (#45) * Modified code to support Julia 0.7 - 1.0 * added dependencies * Removed Iterators * Added version check for initial imports * Fixed deprecation warning (Iterators) * Addressed commets raised by ararslan * Removed VERSION checks and code for < v"0.7.0" * Update .travis.yml Co-authored-by: Anand Bisen --- .travis.yml | 5 ++-- REQUIRE | 2 +- src/MLBase.jl | 5 ++-- src/classification.jl | 26 +++++++++---------- src/crossval.jl | 59 +++++++++++++++++++++--------------------- src/modeltune.jl | 2 +- src/perfeval.jl | 27 ++++++++++--------- src/utils.jl | 18 ++++++------- test/classification.jl | 19 +++++++------- test/crossval.jl | 2 +- test/modeltune.jl | 3 +-- test/perfeval.jl | 6 ++--- test/utils.jl | 6 ++--- 13 files changed, 90 insertions(+), 90 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4432602..ae42cd8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,8 @@ os: - linux - osx julia: - - 0.6 + - 0.7 + - 1.0 - nightly notifications: email: false @@ -12,4 +13,4 @@ notifications: # - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi # - julia -e 'Pkg.clone(pwd()); Pkg.build("MLBase"); Pkg.test("MLBase")' after_success: - - julia -e 'cd(Pkg.dir("MLBase")); Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder())' + - julia -e 'using Pkg; Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder())' diff --git a/REQUIRE b/REQUIRE index 4a27dad..cec87c1 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,4 +1,4 @@ -julia 0.6 +julia 0.7 Reexport StatsBase 0.6.9- IterTools diff --git a/src/MLBase.jl b/src/MLBase.jl index 7797b4e..6cebcd1 100644 --- a/src/MLBase.jl +++ b/src/MLBase.jl @@ -2,12 +2,14 @@ module MLBase using Reexport using IterTools + using Random @reexport using StatsBase import Base: length, show, keys, precision, length, getindex - import Base: start, next, done + import Base: iterate import Base.Order: lt, Ordering, ForwardOrdering, ReverseOrdering, Forward, Reverse import StatsBase: RealVector, IntegerVector, RealMatrix, IntegerMatrix, RealArray + import IterTools: product export @@ -77,4 +79,3 @@ module MLBase include("deprecates.jl") end - diff --git a/src/classification.jl b/src/classification.jl index da2d33c..7025a44 100644 --- a/src/classification.jl +++ b/src/classification.jl @@ -32,7 +32,8 @@ end classify!(r::IntegerVector, x::RealMatrix) = classify!(r, x, Forward) -classify(x::RealMatrix, ord::Ordering) = classify!(Array{Int}(size(x,2)), x, ord) +# - this one throws a deprecation +classify(x::RealMatrix, ord::Ordering) = classify!(Array{Int}(undef, size(x,2)), x, ord) classify(x::RealMatrix) = classify(x, Forward) # classify with score(s) @@ -65,13 +66,13 @@ function classify_withscores!(r::IntegerVector, s::RealVector, x::RealMatrix, or return (r, s) end -classify_withscores!(r::IntegerVector, s::RealVector, x::RealMatrix) = +classify_withscores!(r::IntegerVector, s::RealVector, x::RealMatrix) = classify_withscores!(r, s, x, Forward) function classify_withscores(x::RealMatrix{T}, ord::Ordering) where T<:Real n = size(x, 2) - r = Array{Int}(n) - s = Array{T}(n) + r = Array{Int}(undef, n) + s = Array{T}(undef, n) return classify_withscores!(r, s, x, ord) end @@ -80,7 +81,7 @@ classify_withscores(x::RealMatrix{T}) where {T<:Real} = classify_withscores(x, F # classify with threshold -classify(x::RealVector, t::Real, ord::Ordering) = +classify(x::RealVector, t::Real, ord::Ordering) = ((k, v) = classify_withscore(x, ord); ifelse(lt(ord, v, t), 0, k)) classify(x::RealVector, t::Real) = classify(x, t, Forward) @@ -97,8 +98,8 @@ end classify!(r::IntegerVector, x::RealMatrix, t::Real) = classify!(r, x, t, Forward) -classify(x::RealMatrix, t::Real, ord::Ordering) = classify!(Array{Int}(size(x,2)), x, t, ord) -classify(x::RealMatrix, t::Real) = classify(x, t, Forward) +classify(x::RealMatrix, t::Real, ord::Ordering) = classify!(Array{Int}(undef, size(x,2)), x, t, ord) +classify(x::RealMatrix, t::Real) = classify(x, t, Forward) ## label map @@ -109,7 +110,7 @@ struct LabelMap{K} function LabelMap{K}(vs, v2i) where K length(vs) == length(v2i) || throw(DimensionMismatch("lengths of vs and v2i mismatch")) - new(vs,v2i) + new(vs,v2i) end end @@ -143,18 +144,18 @@ end # use a map to encode discrete values into labels labelencode(lmap::LabelMap{T}, x) where {T} = lmap.v2i[convert(T, x)] -labelencode(lmap::LabelMap{T}, xs::AbstractArray{T}) where {T} = +labelencode(lmap::LabelMap{T}, xs::AbstractArray{T}) where {T} = reshape(Int[labelencode(lmap, x) for x in xs], size(xs)) # decode the label to the associated discrete value labeldecode(lmap::LabelMap{T}, y::Int) where {T} = lmap.vs[y] -labeldecode(lmap::LabelMap{T}, ys::AbstractArray{Int}) where {T} = +labeldecode(lmap::LabelMap{T}, ys::AbstractArray{Int}) where {T} = reshape(T[labeldecode(lmap, y) for y in ys], size(ys)) ## group labels function groupindices(k::Int, xs::IntegerVector; warning::Bool=true) - gs = Array{Vector{Int}}(k) + gs = Array{Vector{Int}}(undef, k) for i = 1:k gs[i] = Int[] end @@ -176,7 +177,7 @@ end function groupindices(lmap::LabelMap{T}, xs::AbstractArray{T}) where T k = length(lmap) - gs = Array{Vector{Int}}(k) + gs = Array{Vector{Int}}(undef, k) for i = 1:k gs[i] = Int[] end @@ -187,4 +188,3 @@ function groupindices(lmap::LabelMap{T}, xs::AbstractArray{T}) where T end return gs end - diff --git a/src/crossval.jl b/src/crossval.jl index 203d231..b30fba2 100644 --- a/src/crossval.jl +++ b/src/crossval.jl @@ -4,7 +4,7 @@ abstract type CrossValGenerator end -# K-fold +# K-fold struct Kfold <: CrossValGenerator permseq::Vector{Int} @@ -20,15 +20,20 @@ end length(c::Kfold) = c.k struct KfoldState - i::Int # the i-th of the subset + i::Int # the i-th of the subset s::Int # starting index e::Int # ending index end -start(c::Kfold) = KfoldState(1, 1, round.(Integer,c.coeff)) -next(c::Kfold, s::KfoldState) = - (i = s.i+1; (setdiff(1:length(c.permseq), c.permseq[s.s:s.e]), KfoldState(i, s.e+1, round.(Integer,c.coeff * i)))) -done(c::Kfold, s::KfoldState) = (s.i > c.k) +# A version check allows to maintain compatibility with earlier versions +function Base.iterate(c::Kfold, state::KfoldState=KfoldState(1, 1, round.(Integer, c.coeff))) + i, s, e = state.i, state.s, state.e + (i > c.k) && return nothing + i += 1 + sd = setdiff(1:length(c.permseq), c.permseq[s:e]) + kst = KfoldState(i, e + 1, round.(Integer, c.coeff * i)) + return (i; (sd, kst)) +end # Stratified K-fold @@ -52,22 +57,21 @@ end length(c::StratifiedKfold) = c.k -start(c::StratifiedKfold) = 1 -function next(c::StratifiedKfold, s::Int) +function Base.iterate(c::StratifiedKfold, s::Int=1) + (s > c.k) && return nothing r = Int[] for (permseq, coeff) in zip(c.permseqs, c.coeffs) a, b = round.(Integer, [s-1, s] .* coeff) append!(r, view(permseq, a+1:b)) end - setdiff(1:c.n, r), s+1 + return setdiff(1:c.n, r), s+1 end -done(c::StratifiedKfold, s::Int) = (s > c.k) # LOOCV (Leave-one-out cross-validation) function leave_one_out(n::Int, i::Int) @assert 1 <= i <= n - x = Array{Int}(n-1) + x = Array{Int}(undef, n - 1) for j = 1:i-1 x[j] = j end @@ -83,10 +87,10 @@ end length(c::LOOCV) = c.n -start(c::LOOCV) = 1 -next(c::LOOCV, s::Int) = (leave_one_out(c.n, s), s+1) -done(c::LOOCV, s::Int) = (s > c.n) - +function iterate(c::LOOCV, s::Int=1) + (s > c.n) && return nothing + return (leave_one_out(c.n, s), s + 1) +end # Repeated random sub-sampling @@ -98,9 +102,10 @@ end length(c::RandomSub) = c.k -start(c::RandomSub) = 1 -next(c::RandomSub, s::Int) = (sort!(sample(1:c.n, c.sn; replace=false)), s+1) -done(c::RandomSub, s::Int) = (s > c.k) +function iterate(c::RandomSub, s::Int=1) + (s > c.k) && return nothing + return (sort!(sample(1:c.n, c.sn; replace=false)), s+1) +end # Stratified repeated random sub-sampling @@ -134,16 +139,15 @@ end length(c::StratifiedRandomSub) = c.k -start(c::StratifiedRandomSub) = 1 -function next(c::StratifiedRandomSub, s::Int) - idxs = Array{Int}(0) +function iterate(c::StratifiedRandomSub, s::Int=1) + (s > c.k) && return nothing + idxs = Array{Int}(undef, 0) sizehint!(idxs, c.sn) for (stratum_sn, stratum_idxs) in zip(c.sns, c.idxs) append!(idxs, sample(stratum_idxs, stratum_sn, replace=false)) end - (sort!(idxs), s+1) + return (sort!(idxs), s + 1) end -done(c::StratifiedRandomSub, s::Int) = (s > c.k) ## Cross validation algorithm # @@ -151,7 +155,7 @@ done(c::StratifiedRandomSub, s::Int) = (s > c.k) # # model = estfun(train_inds) # -# it takes as input the indices of +# it takes as input the indices of # the samples for training, and returns # a trained model. # @@ -172,7 +176,7 @@ done(c::StratifiedRandomSub, s::Int) = (s > c.k) # function cross_validate(estfun::Function, evalfun::Function, n::Int, gen) best_model = nothing - best_score = NaN + best_score = NaN best_inds = Int[] first = true @@ -186,8 +190,5 @@ function cross_validate(estfun::Function, evalfun::Function, n::Int, gen) return scores end -cross_validate(estfun::Function, evalfun::Function, n::Integer, gen) = +cross_validate(estfun::Function, evalfun::Function, n::Integer, gen) = cross_validate(estfun, evalfun, n, gen, Forward) - - - diff --git a/src/modeltune.jl b/src/modeltune.jl index 7ecce56..9085ced 100644 --- a/src/modeltune.jl +++ b/src/modeltune.jl @@ -16,7 +16,7 @@ function gridtune(estfun::Function, # model estimation t = 0 first = true local best_score, best_model, best_cfg - for cf in product(pvals...) + for cf in Iterators.product(pvals...) t += 1 m = estfun(cf...) if m == nothing diff --git a/src/perfeval.jl b/src/perfeval.jl index b26d0d4..4f521e2 100644 --- a/src/perfeval.jl +++ b/src/perfeval.jl @@ -67,14 +67,14 @@ function counthits(gt::IntegerVector, rklst::IntegerMatrix, ks::IntegerVector) end -hitrate(gt::IntegerVector, rklst::IntegerMatrix, k::Integer) = +hitrate(gt::IntegerVector, rklst::IntegerMatrix, k::Integer) = (counthits(gt, rklst, k) / length(gt))::Float64 function hitrates(gt::IntegerVector, rklst::IntegerMatrix, ks::IntegerVector) n = length(gt) h = counthits(gt, rklst, ks) nk = length(ks) - r = Array{Float64}(nk) + r = Array{Float64}(undef, nk) for i = 1:nk r[i] = h[i] / n end @@ -178,7 +178,7 @@ length(v::BinaryThresPredVec) = length(v.scores) getindex(v::BinaryThresPredVec, i::Integer) = !lt(v.ord, v.scores[i], v.thres) # compute roc numbers based on scores & threshold -roc(gt::IntegerVector, scores::RealVector, t::Real, ord::Ordering) = +roc(gt::IntegerVector, scores::RealVector, t::Real, ord::Ordering) = _roc(gt, BinaryThresPredVec(scores, t, ord)) roc(gt::IntegerVector, scores::RealVector, thres::Real) = @@ -211,7 +211,7 @@ length(v::ThresPredVec) = length(v.preds) getindex(v::ThresPredVec, i::Integer) = ifelse(lt(v.ord, v.scores[i], v.thres), 0, v.preds[i]) # compute roc numbers based on predictions & scores & threshold -roc(gt::IntegerVector, preds::Tuple{PV,SV}, t::Real, ord::Ordering) where {PV<:IntegerVector,SV<:RealVector} = +roc(gt::IntegerVector, preds::Tuple{PV,SV}, t::Real, ord::Ordering) where {PV<:IntegerVector,SV<:RealVector} = _roc(gt, ThresPredVec(preds..., t, ord)) roc(gt::IntegerVector, preds::Tuple{PV,SV}, thres::Real) where {PV<:IntegerVector,SV<:RealVector} = @@ -246,10 +246,10 @@ end find_thresbin(x::Real, thresholds::RealVector) = find_thresbin(x, thresholds, Forward) -lin_thresholds(scores::RealArray, n::Integer, ord::ForwardOrdering) = +lin_thresholds(scores::RealArray, n::Integer, ord::ForwardOrdering) = ((s0, s1) = extrema(scores); intv = (s1 - s0) / (n-1); s0:intv:s1) -lin_thresholds(scores::RealArray, n::Integer, ord::ReverseOrdering{ForwardOrdering}) = +lin_thresholds(scores::RealArray, n::Integer, ord::ReverseOrdering{ForwardOrdering}) = ((s0, s1) = extrema(scores); intv = (s0 - s1) / (n-1); s1:intv:s0) # roc for binary predictions @@ -278,7 +278,7 @@ function roc(gt::IntegerVector, scores::RealVector, thresholds::RealVector, ord: end # produce results - r = Array{ROCNums{Int}}(nt) + r = Array{ROCNums{Int}}(undef, nt) fn = 0 tn = 0 @inbounds for i = 1:nt @@ -293,7 +293,7 @@ end roc(gt::IntegerVector, scores::RealVector, thresholds::RealVector) = roc(gt, scores, thresholds, Forward) -roc(gt::IntegerVector, scores::RealVector, n::Integer, ord::Ordering) = +roc(gt::IntegerVector, scores::RealVector, n::Integer, ord::Ordering) = roc(gt, scores, lin_thresholds(scores, n, ord), ord) roc(gt::IntegerVector, scores::RealVector, n::Integer) = roc(gt, scores, n, Forward) @@ -341,7 +341,7 @@ function roc( end # produce results - r = Array{ROCNums{Int}}(nt) + r = Array{ROCNums{Int}}(undef, nt) fn = 0 tn = 0 @inbounds for i = 1:nt @@ -357,15 +357,14 @@ end roc(gt::IntegerVector, preds::Tuple{PV,SV}, thresholds::RealVector) where {PV<:IntegerVector, SV<:RealVector} = roc(gt, preds, thresholds, Forward) -roc(gt::IntegerVector, preds::Tuple{PV,SV}, n::Integer, ord::Ordering) where {PV<:IntegerVector, SV<:RealVector} = +roc(gt::IntegerVector, preds::Tuple{PV,SV}, n::Integer, ord::Ordering) where {PV<:IntegerVector, SV<:RealVector} = roc(gt, preds, lin_thresholds(preds[2],n,ord), ord) -roc(gt::IntegerVector, preds::Tuple{PV,SV}, n::Integer) where {PV<:IntegerVector, SV<:RealVector} = +roc(gt::IntegerVector, preds::Tuple{PV,SV}, n::Integer) where {PV<:IntegerVector, SV<:RealVector} = roc(gt, preds, n, Forward) -roc(gt::IntegerVector, preds::Tuple{PV,SV}, ord::Ordering) where {PV<:IntegerVector, SV<:RealVector} = +roc(gt::IntegerVector, preds::Tuple{PV,SV}, ord::Ordering) where {PV<:IntegerVector, SV<:RealVector} = roc(gt, preds, 100, ord) -roc(gt::IntegerVector, preds::Tuple{PV,SV}) where {PV<:IntegerVector, SV<:RealVector} = +roc(gt::IntegerVector, preds::Tuple{PV,SV}) where {PV<:IntegerVector, SV<:RealVector} = roc(gt, preds, Forward) - diff --git a/src/utils.jl b/src/utils.jl index 396acce..a0f3e99 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -4,12 +4,12 @@ function repeach(x::AbstractVector{T}, n::Integer) where T k = length(x) - r = Array{T}(k * n) + r = Array{T}(undef, k * n) p = 0 @inbounds for i = 1:k xi = x[i] for j = 1:n - r[p += 1] = xi + r[p += 1] = xi end end return r @@ -18,7 +18,7 @@ end function repeach(x::AbstractVector{T}, ns::IntegerVector) where T k = length(x) length(ns) == k || throw(DimensionMismatch("length(ns) should be equal to k.")) - r = Array{T}(sum(ns)) + r = Array{T}(undef, sum(ns)) p = 0 @inbounds for i = 1:k xi = x[i] @@ -35,7 +35,7 @@ end function repeachcol(x::DenseArray{T,2}, n::Integer) where T m = size(x, 1) k = size(x, 2) - r = Array{T}(m, k * n) + r = Array{T}(undef, m, k * n) p = 0 @inbounds for i = 1:k xi = view(x, :, i) @@ -66,7 +66,7 @@ end function repeachrow(x::DenseArray{T,2}, n::Integer) where T k = size(x, 1) m = size(x, 2) - r = Array{T}(k * n, m) + r = Array{T}(undef, k * n, m) p = 0 @inbounds for icol = 1:m p = 0 @@ -83,7 +83,7 @@ end function repeachrow(x::DenseArray{T,2}, ns::IntegerVector) where T k = size(x, 1) m = size(x, 2) - r = Array{T}(sum(ns), m) + r = Array{T}(undef, sum(ns), m) @inbounds for icol = 1:m p = 0 for i = 1:k @@ -100,9 +100,9 @@ end ## return the unique values of A and a vector of vectors of indexes to reconstruct ## the original array -function unique_inverse(A) - out = Array{eltype(A)}(0) - out_idx = Array{Vector{Int}}(0) +function unique_inverse(A::AbstractArray) + out = Array{eltype(A)}(undef, 0) + out_idx = Array{Vector{Int}}(undef, 0) seen = Dict{eltype(A), Int}() for (idx, x) in enumerate(A) if !in(x, keys(seen)) diff --git a/test/classification.jl b/test/classification.jl index 831c064..5ddb7f6 100644 --- a/test/classification.jl +++ b/test/classification.jl @@ -1,15 +1,15 @@ # Tests of intstats.jl using MLBase -using Base.Test +using Test # classify ss = rand(8, 50) for i = 1:size(ss,2) ss_i = ss[:,i] - kmax = indmax(ss_i) - kmin = indmin(ss_i) + kmax = argmax(ss_i) + kmin = argmin(ss_i) vmax = ss_i[kmax] vmin = ss_i[kmin] @@ -26,13 +26,13 @@ for i = 1:size(ss,2) @test classify_withscore(ss_i, Reverse) == (kmin, ss_i[kmin]) end -rmax = Int[indmax(ss[:,i]) for i = 1:size(ss,2)] -rmin = Int[indmin(ss[:,i]) for i = 1:size(ss,2)] -vmax = ss[sub2ind(size(ss), rmax, 1:size(ss,2))] -vmin = ss[sub2ind(size(ss), rmin, 1:size(ss,2))] +rmax = Int[argmax(ss[:,i]) for i = 1:size(ss,2)] +rmin = Int[argmin(ss[:,i]) for i = 1:size(ss,2)] +vmax = ss[LinearIndices(size(ss))[CartesianIndex.(rmax, 1:size(ss,2))]] +vmin = ss[LinearIndices(size(ss))[CartesianIndex.(rmin, 1:size(ss,2))]] -trmax = copy(rmax); trmax[vmax .< 0.8] = 0 -trmin = copy(rmin); trmin[vmin .> 0.2] = 0 +trmax = copy(rmax); trmax[vmax .< 0.8] .= 0 +trmin = copy(rmin); trmin[vmin .> 0.2] .= 0 @test classify(ss) == rmax @test classify(ss, Forward) == rmax @@ -62,4 +62,3 @@ gs = Any[[1,2,5,8],[3,4,6],[7]] @test groupindices(3, labels) == gs @test groupindices(lmap, xs) == gs - diff --git a/test/crossval.jl b/test/crossval.jl index ba40960..a93712f 100644 --- a/test/crossval.jl +++ b/test/crossval.jl @@ -1,5 +1,5 @@ using MLBase -using Base.Test +using Test ## Kfold diff --git a/test/modeltune.jl b/test/modeltune.jl index 2c8afdc..911ef42 100644 --- a/test/modeltune.jl +++ b/test/modeltune.jl @@ -1,5 +1,5 @@ using MLBase -using Base.Test +using Test ## gridtune @@ -25,4 +25,3 @@ p2 = ("y", [:a, :b]) @test rmodel == 1.0 @test rcfg == (2, :a) @test rscore == 2.0 - diff --git a/test/perfeval.jl b/test/perfeval.jl index 61ba21b..d8dc12e 100644 --- a/test/perfeval.jl +++ b/test/perfeval.jl @@ -1,9 +1,9 @@ using MLBase -using Base.Test +using Test import StatsBase -import StatsBase: harmmean +import StatsBase: harmmean ## correctrate & errorrate @@ -29,7 +29,7 @@ rs = [1 2 2 1 3 2 1 1 3 3; @test counthits(gt, rs, 1:3) == [3, 8, 8] @test counthits(gt, rs, [2, 4]) == [8, 10] -@test counthits(gt, rs, 1:2:5) == [3, 8, 10] +@test counthits(gt, rs, 1:2:5) == [3, 8, 10] @test [hitrate(gt, rs, k) for k=1:5] ≈ [0.3, 0.8, 0.8, 1.0, 1.0] @test hitrates(gt, rs, 1:3) ≈ [0.3, 0.8, 0.8] diff --git a/test/utils.jl b/test/utils.jl index 881212b..272d0cc 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -1,7 +1,7 @@ # Tests of intstats.jl using MLBase -using Base.Test +using Test # repeach @@ -25,6 +25,6 @@ a = rand(3, 4) a = [:a, :a, :b, :c, :b, :a] ui = MLBase.unique_inverse(a) @test isa(ui, Tuple{Vector{Symbol}, Vector{Vector{Int}}}) -b = Array{Symbol}(mapreduce(length, +, ui[2])) -for (obj, idx) in zip(ui...) b[idx] = obj end +b = Array{Symbol}(undef, mapreduce(length, +, ui[2])) +for (obj, idx) in zip(ui...) b[idx] .= obj end @test a == b