Array resizing fixes and clean up

* Move `memcpy` and `memmove` from julia to C (this is dangerous especially
  for `ptrarray`)
* Make sure resizing a ptrarray always have the new memory cleared
* Make sure resizing a byte array always have the implicit NUL byte
* Use array flag to conditionally copy the array when there's no implicit NUL
  byte in the array.
* Add more test for shared array resizing and implicit NUL byte check

Author: yuyichao

base/array.jl

@@ -38,6 +38,8 @@ end
 ## copy ##
 
 function unsafe_copy!{T}(dest::Ptr{T}, src::Ptr{T}, n)
+    # Do not use this to copy data between arrays.
+    # It can't be made safe no matter how carefully you checked.
     ccall(:memmove, Ptr{Void}, (Ptr{Void}, Ptr{Void}, UInt),
           dest, src, n*sizeof(T))
     return dest
@@ -65,11 +67,7 @@ end
 
 copy!{T}(dest::Array{T}, src::Array{T}) = copy!(dest, 1, src, 1, length(src))
 
-function copy(a::Array)
-    b = similar(a)
-    ccall(:memcpy, Ptr{Void}, (Ptr{Void}, Ptr{Void}, UInt), b, a, sizeof(a))
-    return b
-end
+copy{T<:Array}(a::T) = ccall(:jl_array_copy, Ref{T}, (Any,), a)
 
 function reinterpret{T,S}(::Type{T}, a::Array{S,1})
     nel = Int(div(length(a)*sizeof(S),sizeof(T)))
@@ -120,14 +118,14 @@ end
 ## Constructors ##
 
 similar(a::Array, T::Type, dims::Dims) = Array{T}(dims)
-similar{T}(a::Array{T,1})              = Array{T}(size(a,1))
-similar{T}(a::Array{T,2})              = Array{T}(size(a,1), size(a,2))
+similar{T<:Array}(a::T)                =
+    ccall(:jl_array_similar, Ref{T}, (Any,), a)
+similar{T}(a::Array{T}, m::Int)        =
+    ccall(:jl_array_similar_1d, Ref{Vector{T}}, (Any, Any, Int), Vector{T}, a, m)
+similar{T}(a::Array{T,1}, S::Type)     = Vector{S}(size(a,1))
+similar{T}(a::Array{T,2}, S::Type)     = Matrix{S}(size(a,1), size(a,2))
 similar{T}(a::Array{T,1}, dims::Dims)  = Array{T}(dims)
-similar{T}(a::Array{T,1}, m::Int)      = Array{T}(m)
-similar{T}(a::Array{T,1}, S::Type)     = Array{S}(size(a,1))
 similar{T}(a::Array{T,2}, dims::Dims)  = Array{T}(dims)
-similar{T}(a::Array{T,2}, m::Int)      = Array{T}(m)
-similar{T}(a::Array{T,2}, S::Type)     = Array{S}(size(a,1), size(a,2))
 
 # T[x...] constructs Array{T,1}
 function getindex(T::Type, vals...)
@@ -387,66 +385,13 @@ setindex!{T, N}(A::Array{T, N}, x::Number, ::Vararg{Colon, N}) = fill!(A, x)
 
 # efficiently grow an array
 
-function _growat!(a::Vector, i::Integer, delta::Integer)
-    n = length(a)
-    if i < div(n,2)
-        _growat_beg!(a, i, delta)
-    else
-        _growat_end!(a, i, delta)
-    end
-    return a
-end
-
-function _growat_beg!(a::Vector, i::Integer, delta::Integer)
-    ccall(:jl_array_grow_beg, Void, (Any, UInt), a, delta)
-    if i > 1
-        ccall(:memmove, Ptr{Void}, (Ptr{Void}, Ptr{Void}, Csize_t),
-              pointer(a, 1), pointer(a, 1+delta), (i-1)*elsize(a))
-    end
-    return a
-end
-
-function _growat_end!(a::Vector, i::Integer, delta::Integer)
-    ccall(:jl_array_grow_end, Void, (Any, UInt), a, delta)
-    n = length(a)
-    if n >= i+delta
-        ccall(:memmove, Ptr{Void}, (Ptr{Void}, Ptr{Void}, Csize_t),
-              pointer(a, i+delta), pointer(a, i), (n-i-delta+1)*elsize(a))
-    end
-    return a
-end
+_growat!(a::Vector, i::Integer, delta::Integer) =
+    ccall(:jl_array_grow_at, Void, (Any, Int, UInt), a, i - 1, delta)
 
 # efficiently delete part of an array
 
-function _deleteat!(a::Vector, i::Integer, delta::Integer)
-    n = length(a)
-    last = i+delta-1
-    if i-1 < n-last
-        _deleteat_beg!(a, i, delta)
-    else
-        _deleteat_end!(a, i, delta)
-    end
-    return a
-end
-
-function _deleteat_beg!(a::Vector, i::Integer, delta::Integer)
-    if i > 1
-        ccall(:memmove, Ptr{Void}, (Ptr{Void}, Ptr{Void}, Csize_t),
-              pointer(a, 1+delta), pointer(a, 1), (i-1)*elsize(a))
-    end
-    ccall(:jl_array_del_beg, Void, (Any, UInt), a, delta)
-    return a
-end
-
-function _deleteat_end!(a::Vector, i::Integer, delta::Integer)
-    n = length(a)
-    if n >= i+delta
-        ccall(:memmove, Ptr{Void}, (Ptr{Void}, Ptr{Void}, Csize_t),
-              pointer(a, i), pointer(a, i+delta), (n-i-delta+1)*elsize(a))
-    end
-    ccall(:jl_array_del_end, Void, (Any, UInt), a, delta)
-    return a
-end
+_deleteat!(a::Vector, i::Integer, delta::Integer) =
+    ccall(:jl_array_del_at, Void, (Any, Int, UInt), a, i - 1, delta)
 
 ## Dequeue functionality ##
 
@@ -526,34 +471,20 @@ function shift!(a::Vector)
 end
 
 function insert!{T}(a::Array{T,1}, i::Integer, item)
-    if !(1 <= i <= length(a)+1)
-        throw(BoundsError())
-    end
-    if i == length(a)+1
-        return push!(a, item)
-    end
-    item = convert(T, item)
+    # Throw convert error before changing the shape of the array
+    _item = convert(T, item)
     _growat!(a, i, 1)
-    a[i] = item
+    # _growat! already did bound check
+    @inbounds a[i] = _item
     return a
 end
 
-function deleteat!(a::Vector, i::Integer)
-    if !(1 <= i <= length(a))
-        throw(BoundsError())
-    end
-    return _deleteat!(a, i, 1)
-end
+deleteat!(a::Vector, i::Integer) = (_deleteat!(a, i, 1); a)
 
 function deleteat!{T<:Integer}(a::Vector, r::UnitRange{T})
     n = length(a)
-    isempty(r) && return a
-    f = first(r)
-    l = last(r)
-    if !(1 <= f && l <= n)
-        throw(BoundsError())
-    end
-    return _deleteat!(a, f, length(r))
+    isempty(r) || _deleteat!(a, first(r), length(r))
+    return a
 end
 
 function deleteat!(a::Vector, inds)
@@ -620,18 +551,9 @@ function splice!{T<:Integer}(a::Vector, r::UnitRange{T}, ins=_default_splice)
 
     if m < d
         delta = d - m
-        if f-1 < n-l
-            _deleteat_beg!(a, f, delta)
-        else
-            _deleteat_end!(a, l-delta+1, delta)
-        end
+        _deleteat!(a, (f - 1 < n - l) ? f : (l - delta + 1), delta)
     elseif m > d
-        delta = m - d
-        if f-1 < n-l
-            _growat_beg!(a, f, delta)
-        else
-            _growat_end!(a, l+1, delta)
-        end
+        _growat!(a, (f - 1 < n - l) ? f : (l + 1), m - d)
     end
 
     k = 1
@@ -684,27 +606,8 @@ function reverse!(v::AbstractVector, s=1, n=length(v))
     return v
 end
 
-function vcat{T}(arrays::Vector{T}...)
-    n = 0
-    for a in arrays
-        n += length(a)
-    end
-    arr = Array{T}(n)
-    ptr = pointer(arr)
-    offset = 0
-    if isbits(T)
-        elsz = sizeof(T)
-    else
-        elsz = Core.sizeof(Ptr{Void})
-    end
-    for a in arrays
-        nba = length(a)*elsz
-        ccall(:memcpy, Ptr{Void}, (Ptr{Void}, Ptr{Void}, UInt),
-              ptr+offset, a, nba)
-        offset += nba
-    end
-    return arr
-end
+vcat{T}(arrays::Vector{T}...) =
+    ccall(:jl_array_vcat_vectors, Ref{Vector{T}}, (Any,), arrays)
 
 function hcat{T}(V::Vector{T}...)
     height = length(V[1])

base/c.jl

@@ -80,15 +80,19 @@ unsafe_wrap(::Type{String}, p::Cstring, len::Integer, own::Bool=false) =
 unsafe_string(s::Cstring) = unsafe_string(convert(Ptr{UInt8}, s))
 
 # convert strings to String etc. to pass as pointers
-cconvert(::Type{Cstring}, s::AbstractString) = String(s)
+cconvert(::Type{Cstring}, s::String) =
+    ccall(:jl_array_cconvert_cstring, Ref{Vector{UInt8}},
+          (Vector{UInt8},), s.data)
+cconvert(::Type{Cstring}, s::AbstractString) =
+    cconvert(Cstring, String(s)::String)
 cconvert(::Type{Cwstring}, s::AbstractString) = wstring(s)
 
 containsnul(p::Ptr, len) =
     C_NULL != ccall(:memchr, Ptr{Cchar}, (Ptr{Cchar}, Cint, Csize_t), p, 0, len)
 containsnul(s::String) = containsnul(unsafe_convert(Ptr{Cchar}, s), sizeof(s))
 containsnul(s::AbstractString) = '\0' in s
 
-function unsafe_convert(::Type{Cstring}, s::String)
+function unsafe_convert(::Type{Cstring}, s::Vector{UInt8})
     p = unsafe_convert(Ptr{Cchar}, s)
     if containsnul(p, sizeof(s))
         throw(ArgumentError("embedded NULs are not allowed in C strings: $(repr(s))"))

base/libgit2.jl

@@ -297,7 +297,7 @@ function clone{P<:AbstractPayload}(repo_url::AbstractString, repo_path::Abstract
                remote_cb::Ptr{Void} = C_NULL,
                payload::Nullable{P}=Nullable{AbstractPayload}())
     # setup clone options
-    lbranch = Base.cconvert(Cstring, String(branch))
+    lbranch = Base.cconvert(Cstring, branch)
     fetch_opts=FetchOptions(callbacks = RemoteCallbacks(credentials_cb(), payload))
     clone_opts = CloneOptions(
                 bare = Cint(isbare),

base/libgit2/strarray.jl

@@ -2,7 +2,11 @@
 
 function StrArrayStruct{T<:AbstractString}(strs::T...)
     strcount = length(strs)
-    map(s->Base.unsafe_convert(Cstring, String(s)), strs) # check for null-strings
+    for s in strs
+        if Base.containsnul(s)
+            throw("embedded NULs are not allowed in C strings: $(repr(s))")
+        end
+    end
     sa_strings = convert(Ptr{Cstring}, Libc.malloc(sizeof(Cstring) * strcount))
     for i=1:strcount
         len = length(strs[i])

base/precompile.jl

@@ -154,10 +154,7 @@ precompile(Libc.TmStruct, (Float64,))
 precompile(Base.VersionNumber, (Int, Int, Int, Tuple{}, Tuple{String}))
 precompile(Base._atexit, ())
 precompile(Base._deleteat!, (Array{UInt8, 1}, Int, Int))
-precompile(Base._deleteat_beg!, (Array{UInt8, 1}, Int, Int))
-precompile(Base._deleteat_end!, (Array{UInt8, 1}, Int, Int))
-precompile(Base._growat_beg!, (Array{UInt8, 1}, Int, Int))
-precompile(Base._growat_end!, (Array{UInt8, 1}, Int, Int))
+precompile(Base._growat!, (Array{UInt8, 1}, Int, Int))
 precompile(Base._setindex!, (Base.Dict{Symbol, Any}, Base.LineEdit.Prompt, Symbol, Int))
 precompile(Base._setindex!, (Dict{Any, Any}, Base.LineEdit.PromptState, Base.LineEdit.Prompt, Int))
 precompile(Base._setindex!, (Dict{Any, Any}, Bool, WeakRef, Int))