From 3ab73916a375bc41513ae89d000b9c0b6bde821d Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Fri, 22 May 2015 09:31:32 -0400 Subject: [PATCH] Speed up scalar BitArray indexing by ~25% This works around issue #9974 for BitArray indexing. BitArrays use inlined helper functions, `unsafe_bit(get|set)index`, to do the dirty work of picking bits out of the chunks array. Previously, these helpers took the array of chunks as an argument, but that array needs a GC root since BitArrays are mutable. This changes those helper functions to work on whole BitArray itself, which enables an optimization to avoid that root (since the chunks array is only accessed as an argument to `arrayref`, which is special-cased). The ~25% performance gain is for unsafe_getindex; the difference isn't quite as big for getindex (only ~10%) since there's still a GC root for the BoundsError. That can also be avoided, but I'd rather make that change more systematically (as `checkbounds`) with #10525 or a subset thereof. --- base/bitarray.jl | 33 +++++++++++++-------------------- base/broadcast.jl | 7 +++---- base/multidimensional.jl | 7 +++---- 3 files changed, 19 insertions(+), 28 deletions(-) diff --git a/base/bitarray.jl b/base/bitarray.jl index 14cb014aa2cb8..a44ee60803d7c 100644 --- a/base/bitarray.jl +++ b/base/bitarray.jl @@ -292,9 +292,8 @@ end convert{T,N}(::Type{Array{T}}, B::BitArray{N}) = convert(Array{T,N},B) function convert{T,N}(::Type{Array{T,N}}, B::BitArray{N}) A = Array(T, size(B)) - Bc = B.chunks @inbounds for i = 1:length(A) - A[i] = unsafe_bitgetindex(Bc, i) + A[i] = unsafe_bitgetindex(B, i) end return A end @@ -341,16 +340,16 @@ bitpack{T,N}(A::AbstractArray{T,N}) = convert(BitArray{N}, A) ## Indexing: getindex ## -@inline function unsafe_bitgetindex(Bc::Vector{UInt64}, i::Int) +@inline function unsafe_bitgetindex(B::BitArray, i::Int) i1, i2 = get_chunks_id(i) u = UInt64(1) << i2 - @inbounds r = (Bc[i1] & u) != 0 + @inbounds r = (B.chunks[i1] & u) != 0 return r end @inline function getindex(B::BitArray, i::Int) 1 <= i <= length(B) || throw(BoundsError(B, i)) - return unsafe_bitgetindex(B.chunks, i) + return unsafe_bitgetindex(B, i) end getindex(B::BitArray, i::Real) = getindex(B, to_index(i)) @@ -358,19 +357,17 @@ getindex(B::BitArray, i::Real) = getindex(B, to_index(i)) getindex(B::BitArray) = getindex(B, 1) # 0d bitarray -getindex(B::BitArray{0}) = unsafe_bitgetindex(B.chunks, 1) +getindex(B::BitArray{0}) = unsafe_bitgetindex(B, 1) function getindex{T<:Real}(B::BitArray, I::AbstractVector{T}) X = BitArray(length(I)) lB = length(B) - Xc = X.chunks - Bc = B.chunks ind = 1 for i in I # faster X[ind] = B[i] j = to_index(i) 1 <= j <= lB || throw(BoundsError(B, j)) - unsafe_bitsetindex!(Xc, unsafe_bitgetindex(Bc, j), ind) + unsafe_bitsetindex!(X, unsafe_bitgetindex(B, j), ind) ind += 1 end return X @@ -388,13 +385,11 @@ for IT in [AbstractVector{Bool}, AbstractArray{Bool}] checkbounds(B, I) n = sum(I) X = BitArray(n) - Xc = X.chunks - Bc = B.chunks ind = 1 for i = 1:length(I) if $idxop(I, i) # faster X[ind] = B[i] - unsafe_bitsetindex!(Xc, unsafe_bitgetindex(Bc, i), ind) + unsafe_bitsetindex!(X, unsafe_bitgetindex(B, i), ind) ind += 1 end end @@ -405,14 +400,14 @@ end ## Indexing: setindex! ## -@inline function unsafe_bitsetindex!(Bc::Array{UInt64}, x::Bool, i::Int) +@inline function unsafe_bitsetindex!(B::BitArray, x::Bool, i::Int) i1, i2 = get_chunks_id(i) u = UInt64(1) << i2 @inbounds begin if x - Bc[i1] |= u + B.chunks[i1] |= u else - Bc[i1] &= ~u + B.chunks[i1] &= ~u end end end @@ -421,7 +416,7 @@ setindex!(B::BitArray, x) = setindex!(B, convert(Bool,x), 1) function setindex!(B::BitArray, x::Bool, i::Int) 1 <= i <= length(B) || throw(BoundsError(B, i)) - unsafe_bitsetindex!(B.chunks, x, i) + unsafe_bitsetindex!(B, x, i) return B end @@ -453,10 +448,9 @@ end quote checkbounds(B, I) y = convert(Bool, x) - Bc = B.chunks for i = 1:length(I) # faster I[i] && B[i] = y - $idxop(I, i) && unsafe_bitsetindex!(Bc, y, i) + $idxop(I, i) && unsafe_bitsetindex!(B, y, i) end return B end @@ -499,12 +493,11 @@ end idxop = I <: Array{Bool} ? :unsafe_getindex : :getindex quote checkbounds(B, I) - Bc = B.chunks c = 1 for i = 1:length(I) if $idxop(I, i) # faster B[i] = X[c] - unsafe_bitsetindex!(Bc, convert(Bool, X[c]), i) + unsafe_bitsetindex!(B, convert(Bool, X[c]), i) c += 1 end end diff --git a/base/broadcast.jl b/base/broadcast.jl index 740b506be5301..b316d3021bc6b 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -417,11 +417,11 @@ end (.^)(A::BitArray, B::AbstractArray{Bool}) = (B .<= A) (.^)(A::AbstractArray{Bool}, B::AbstractArray{Bool}) = (B .<= A) -function bitcache_pow{T}(Ac::Vector{UInt64}, B::Array{T}, l::Int, ind::Int, C::Vector{Bool}) +function bitcache_pow{T}(A::BitArray, B::Array{T}, l::Int, ind::Int, C::Vector{Bool}) left = l - ind + 1 @inbounds begin for j = 1:min(bitcache_size, left) - C[j] = unsafe_bitgetindex(Ac, ind) ^ B[ind] + C[j] = unsafe_bitgetindex(A, ind) ^ B[ind] ind += 1 end C[left+1:bitcache_size] = false @@ -438,13 +438,12 @@ function (.^){T<:Integer}(A::BitArray, B::Array{T}) F = BitArray(shape) l = length(F) l == 0 && return F - Ac = A.chunks Fc = F.chunks C = Array(Bool, bitcache_size) ind = 1 cind = 1 for i = 1:div(l + bitcache_size - 1, bitcache_size) - ind = bitcache_pow(Ac, B, l, ind, C) + ind = bitcache_pow(A, B, l, ind, C) dumpbitcache(Fc, cind, C) cind += bitcache_chunks end diff --git a/base/multidimensional.jl b/base/multidimensional.jl index fb211e3b29334..2837e6ad76976 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -235,11 +235,11 @@ using .IteratorsMD end end -@inline unsafe_getindex(v::BitArray, ind::Int) = Base.unsafe_bitgetindex(v.chunks, ind) +@inline unsafe_getindex(v::BitArray, ind::Int) = Base.unsafe_bitgetindex(v, ind) @inline unsafe_setindex!{T}(v::Array{T}, x::T, ind::Int) = (@inbounds v[ind] = x; v) @inline unsafe_setindex!{T}(v::AbstractArray{T}, x::T, ind::Int) = (v[ind] = x; v) -@inline unsafe_setindex!(v::BitArray, x::Bool, ind::Int) = (Base.unsafe_bitsetindex!(v.chunks, x, ind); v) +@inline unsafe_setindex!(v::BitArray, x::Bool, ind::Int) = (Base.unsafe_bitsetindex!(v, x, ind); v) @inline unsafe_setindex!{T}(v::AbstractArray{T}, x::T, ind::Real) = unsafe_setindex!(v, x, to_index(ind)) # Version that uses cartesian indexing for src @@ -640,7 +640,6 @@ end quote @nexprs $N d->(I_d = I[d]) X = BitArray(index_shape($(Isplat...))) - Xc = X.chunks stride_1 = 1 @nexprs $N d->(stride_{d+1} = stride_d * size(B, d)) @@ -649,7 +648,7 @@ end @nloops($N, i, d->I_d, d->(offset_{d-1} = offset_d + (i_d-1)*stride_d), # PRE begin - unsafe_bitsetindex!(Xc, B[offset_0], ind) + unsafe_bitsetindex!(X, B[offset_0], ind) ind += 1 end) return X