Merge pull request #16893 from JuliaLang/yyc/gc/array

Array resizing fixes and clean up

Author: yuyichao

base/array.jl

@@ -36,6 +36,8 @@ end
 ## copy ##
 
 function unsafe_copy!{T}(dest::Ptr{T}, src::Ptr{T}, n)
+    # Do not use this to copy data between pointer arrays.
+    # It can't be made safe no matter how carefully you checked.
     ccall(:memmove, Ptr{Void}, (Ptr{Void}, Ptr{Void}, UInt),
           dest, src, n*sizeof(T))
     return dest
@@ -45,9 +47,8 @@ function unsafe_copy!{T}(dest::Array{T}, doffs, src::Array{T}, soffs, n)
     if isbits(T)
         unsafe_copy!(pointer(dest, doffs), pointer(src, soffs), n)
     else
-        for i=0:n-1
-            @inbounds arrayset(dest, src[i+soffs], i+doffs)
-        end
+        ccall(:jl_array_ptr_copy, Void, (Any, Ptr{Void}, Any, Ptr{Void}, Int),
+              dest, pointer(dest, doffs), src, pointer(src, soffs), n)
     end
     return dest
 end
@@ -63,11 +64,7 @@ end
 
 copy!{T}(dest::Array{T}, src::Array{T}) = copy!(dest, 1, src, 1, length(src))
 
-function copy(a::Array)
-    b = similar(a)
-    ccall(:memcpy, Ptr{Void}, (Ptr{Void}, Ptr{Void}, UInt), b, a, sizeof(a))
-    return b
-end
+copy{T<:Array}(a::T) = ccall(:jl_array_copy, Ref{T}, (Any,), a)
 
 function reinterpret{T,S}(::Type{T}, a::Array{S,1})
     nel = Int(div(length(a)*sizeof(S),sizeof(T)))
@@ -389,66 +386,13 @@ setindex!{T, N}(A::Array{T, N}, x::Number, ::Vararg{Colon, N}) = fill!(A, x)
 
 # efficiently grow an array
 
-function _growat!(a::Vector, i::Integer, delta::Integer)
-    n = length(a)
-    if i < div(n,2)
-        _growat_beg!(a, i, delta)
-    else
-        _growat_end!(a, i, delta)
-    end
-    return a
-end
-
-function _growat_beg!(a::Vector, i::Integer, delta::Integer)
-    ccall(:jl_array_grow_beg, Void, (Any, UInt), a, delta)
-    if i > 1
-        ccall(:memmove, Ptr{Void}, (Ptr{Void}, Ptr{Void}, Csize_t),
-              pointer(a, 1), pointer(a, 1+delta), (i-1)*elsize(a))
-    end
-    return a
-end
-
-function _growat_end!(a::Vector, i::Integer, delta::Integer)
-    ccall(:jl_array_grow_end, Void, (Any, UInt), a, delta)
-    n = length(a)
-    if n >= i+delta
-        ccall(:memmove, Ptr{Void}, (Ptr{Void}, Ptr{Void}, Csize_t),
-              pointer(a, i+delta), pointer(a, i), (n-i-delta+1)*elsize(a))
-    end
-    return a
-end
+_growat!(a::Vector, i::Integer, delta::Integer) =
+    ccall(:jl_array_grow_at, Void, (Any, Int, UInt), a, i - 1, delta)
 
 # efficiently delete part of an array
 
-function _deleteat!(a::Vector, i::Integer, delta::Integer)
-    n = length(a)
-    last = i+delta-1
-    if i-1 < n-last
-        _deleteat_beg!(a, i, delta)
-    else
-        _deleteat_end!(a, i, delta)
-    end
-    return a
-end
-
-function _deleteat_beg!(a::Vector, i::Integer, delta::Integer)
-    if i > 1
-        ccall(:memmove, Ptr{Void}, (Ptr{Void}, Ptr{Void}, Csize_t),
-              pointer(a, 1+delta), pointer(a, 1), (i-1)*elsize(a))
-    end
-    ccall(:jl_array_del_beg, Void, (Any, UInt), a, delta)
-    return a
-end
-
-function _deleteat_end!(a::Vector, i::Integer, delta::Integer)
-    n = length(a)
-    if n >= i+delta
-        ccall(:memmove, Ptr{Void}, (Ptr{Void}, Ptr{Void}, Csize_t),
-              pointer(a, i), pointer(a, i+delta), (n-i-delta+1)*elsize(a))
-    end
-    ccall(:jl_array_del_end, Void, (Any, UInt), a, delta)
-    return a
-end
+_deleteat!(a::Vector, i::Integer, delta::Integer) =
+    ccall(:jl_array_del_at, Void, (Any, Int, UInt), a, i - 1, delta)
 
 ## Dequeue functionality ##
 
@@ -528,34 +472,20 @@ function shift!(a::Vector)
 end
 
 function insert!{T}(a::Array{T,1}, i::Integer, item)
-    if !(1 <= i <= length(a)+1)
-        throw(BoundsError())
-    end
-    if i == length(a)+1
-        return push!(a, item)
-    end
-    item = convert(T, item)
+    # Throw convert error before changing the shape of the array
+    _item = convert(T, item)
     _growat!(a, i, 1)
-    a[i] = item
+    # _growat! already did bound check
+    @inbounds a[i] = _item
     return a
 end
 
-function deleteat!(a::Vector, i::Integer)
-    if !(1 <= i <= length(a))
-        throw(BoundsError())
-    end
-    return _deleteat!(a, i, 1)
-end
+deleteat!(a::Vector, i::Integer) = (_deleteat!(a, i, 1); a)
 
 function deleteat!{T<:Integer}(a::Vector, r::UnitRange{T})
     n = length(a)
-    isempty(r) && return a
-    f = first(r)
-    l = last(r)
-    if !(1 <= f && l <= n)
-        throw(BoundsError())
-    end
-    return _deleteat!(a, f, length(r))
+    isempty(r) || _deleteat!(a, first(r), length(r))
+    return a
 end
 
 function deleteat!(a::Vector, inds)
@@ -622,18 +552,9 @@ function splice!{T<:Integer}(a::Vector, r::UnitRange{T}, ins=_default_splice)
 
     if m < d
         delta = d - m
-        if f-1 < n-l
-            _deleteat_beg!(a, f, delta)
-        else
-            _deleteat_end!(a, l-delta+1, delta)
-        end
+        _deleteat!(a, (f - 1 < n - l) ? f : (l - delta + 1), delta)
     elseif m > d
-        delta = m - d
-        if f-1 < n-l
-            _growat_beg!(a, f, delta)
-        else
-            _growat_end!(a, l+1, delta)
-        end
+        _growat!(a, (f - 1 < n - l) ? f : (l + 1), m - d)
     end
 
     k = 1
@@ -693,17 +614,22 @@ function vcat{T}(arrays::Vector{T}...)
     end
     arr = Array{T}(n)
     ptr = pointer(arr)
-    offset = 0
     if isbits(T)
-        elsz = sizeof(T)
+        elsz = Core.sizeof(T)
     else
         elsz = Core.sizeof(Ptr{Void})
     end
     for a in arrays
-        nba = length(a)*elsz
-        ccall(:memcpy, Ptr{Void}, (Ptr{Void}, Ptr{Void}, UInt),
-              ptr+offset, a, nba)
-        offset += nba
+        na = length(a)
+        nba = na * elsz
+        if isbits(T)
+            ccall(:memcpy, Ptr{Void}, (Ptr{Void}, Ptr{Void}, UInt),
+                  ptr, a, nba)
+        else
+            ccall(:jl_array_ptr_copy, Void, (Any, Ptr{Void}, Any, Ptr{Void}, Int),
+                  arr, ptr, a, pointer(a), na)
+        end
+        ptr += nba
     end
     return arr
 end

base/c.jl

@@ -46,17 +46,6 @@ if !is_windows()
     end
 end
 
-# C NUL-terminated string pointers; these can be used in ccall
-# instead of Ptr{Cchar} and Ptr{Cwchar_t}, respectively, to enforce
-# a check for embedded NUL chars in the string (to avoid silent truncation).
-if Int === Int64
-    bitstype 64 Cstring
-    bitstype 64 Cwstring
-else
-    bitstype 32 Cstring
-    bitstype 32 Cwstring
-end
-
 # construction from typed pointers
 convert{T<:Union{Int8,UInt8}}(::Type{Cstring}, p::Ptr{T}) = box(Cstring, p)
 convert(::Type{Cwstring}, p::Ptr{Cwchar_t}) = box(Cwstring, p)
@@ -80,20 +69,27 @@ unsafe_wrap(::Type{String}, p::Cstring, len::Integer, own::Bool=false) =
 unsafe_string(s::Cstring) = unsafe_string(convert(Ptr{UInt8}, s))
 
 # convert strings to String etc. to pass as pointers
-cconvert(::Type{Cstring}, s::AbstractString) = String(s)
+cconvert(::Type{Cstring}, s::String) =
+    ccall(:jl_array_cconvert_cstring, Ref{Vector{UInt8}},
+          (Vector{UInt8},), s.data)
+cconvert(::Type{Cstring}, s::AbstractString) =
+    cconvert(Cstring, String(s)::String)
 
 function cconvert(::Type{Cwstring}, s::AbstractString)
     v = transcode(Cwchar_t, String(s).data)
     !isempty(v) && v[end] == 0 || push!(v, 0)
     return v
 end
 
+eltype(::Type{Cstring}) = UInt8
+eltype(::Type{Cwstring}) = Cwchar_t
+
 containsnul(p::Ptr, len) =
     C_NULL != ccall(:memchr, Ptr{Cchar}, (Ptr{Cchar}, Cint, Csize_t), p, 0, len)
 containsnul(s::String) = containsnul(unsafe_convert(Ptr{Cchar}, s), sizeof(s))
 containsnul(s::AbstractString) = '\0' in s
 
-function unsafe_convert(::Type{Cstring}, s::String)
+function unsafe_convert(::Type{Cstring}, s::Vector{UInt8})
     p = unsafe_convert(Ptr{Cchar}, s)
     containsnul(p, sizeof(s)) &&
         throw(ArgumentError("embedded NULs are not allowed in C strings: $(repr(s))"))

base/libgit2/libgit2.jl

@@ -297,7 +297,7 @@ function clone{P<:AbstractPayload}(repo_url::AbstractString, repo_path::Abstract
                remote_cb::Ptr{Void} = C_NULL,
                payload::Nullable{P}=Nullable{AbstractPayload}())
     # setup clone options
-    lbranch = Base.cconvert(Cstring, String(branch))
+    lbranch = Base.cconvert(Cstring, branch)
     fetch_opts=FetchOptions(callbacks = RemoteCallbacks(credentials_cb(), payload))
     clone_opts = CloneOptions(
                 bare = Cint(isbare),

base/libgit2/oid.jl

@@ -28,7 +28,7 @@ function Oid(id::AbstractString)
               (Ptr{Oid}, Ptr{UInt8}, Csize_t), oid_ptr, bstr, len)
     else
         ccall((:git_oid_fromstrp, :libgit2), Cint,
-              (Ptr{Oid}, Ptr{Cchar}), oid_ptr, bstr)
+              (Ptr{Oid}, Cstring), oid_ptr, bstr)
     end
     err != 0 && return Oid()
     return oid_ptr[]
@@ -46,7 +46,7 @@ function Oid(repo::GitRepo, ref_name::AbstractString)
     isempty(repo) && return Oid()
     oid_ptr  = Ref(Oid())
     @check ccall((:git_reference_name_to_id, :libgit2), Cint,
-                    (Ptr{Oid}, Ptr{Void}, Ptr{UInt8}),
+                    (Ptr{Oid}, Ptr{Void}, Cstring),
                      oid_ptr, repo.ptr, ref_name)
     return oid_ptr[]
 end

base/libgit2/strarray.jl

@@ -2,7 +2,11 @@
 
 function StrArrayStruct{T<:AbstractString}(strs::T...)
     strcount = length(strs)
-    map(s->Base.unsafe_convert(Cstring, String(s)), strs) # check for null-strings
+    for s in strs
+        if Base.containsnul(s)
+            throw("embedded NULs are not allowed in C strings: $(repr(s))")
+        end
+    end
     sa_strings = convert(Ptr{Cstring}, Libc.malloc(sizeof(Cstring) * strcount))
     for i=1:strcount
         len = length(strs[i])

base/loading.jl

@@ -171,7 +171,7 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, t
         end
     elseif node == myid()
         myid() == 1 && recompile_stale(mod, path_to_try)
-        restored = ccall(:jl_restore_incremental, Any, (Ptr{UInt8},), path_to_try)
+        restored = ccall(:jl_restore_incremental, Any, (Cstring,), path_to_try)
     else
         content = remotecall_fetch(open, node, read, path_to_try)
         restored = _include_from_serialized(content)
@@ -378,8 +378,8 @@ end
 # remote/parallel load
 
 include_string(txt::String, fname::String) =
-    ccall(:jl_load_file_string, Any, (Ptr{UInt8},Csize_t,Ptr{UInt8},Csize_t),
-          txt, sizeof(txt), fname, sizeof(fname))
+    ccall(:jl_load_file_string, Any, (Ptr{UInt8},Csize_t,Cstring),
+          txt, sizeof(txt), fname)
 
 include_string(txt::AbstractString, fname::AbstractString="string") =
     include_string(String(txt), String(fname))

base/precompile.jl

@@ -154,10 +154,7 @@ precompile(Libc.TmStruct, (Float64,))
 precompile(Base.VersionNumber, (Int, Int, Int, Tuple{}, Tuple{String}))
 precompile(Base._atexit, ())
 precompile(Base._deleteat!, (Array{UInt8, 1}, Int, Int))
-precompile(Base._deleteat_beg!, (Array{UInt8, 1}, Int, Int))
-precompile(Base._deleteat_end!, (Array{UInt8, 1}, Int, Int))
-precompile(Base._growat_beg!, (Array{UInt8, 1}, Int, Int))
-precompile(Base._growat_end!, (Array{UInt8, 1}, Int, Int))
+precompile(Base._growat!, (Array{UInt8, 1}, Int, Int))
 precompile(Base._setindex!, (Base.Dict{Symbol, Any}, Base.LineEdit.Prompt, Symbol, Int))
 precompile(Base._setindex!, (Dict{Any, Any}, Base.LineEdit.PromptState, Base.LineEdit.Prompt, Int))
 precompile(Base._setindex!, (Dict{Any, Any}, Bool, WeakRef, Int))

base/process.jl

@@ -310,8 +310,8 @@ function _jl_spawn(cmd, argv, loop::Ptr{Void}, pp::Process,
     proc = Libc.malloc(_sizeof_uv_process)
     disassociate_julia_struct(proc)
     error = ccall(:jl_spawn, Int32,
-        (Ptr{UInt8}, Ptr{Ptr{UInt8}}, Ptr{Void}, Ptr{Void}, Any, Int32,
-         Ptr{Void}, Int32, Ptr{Void}, Int32, Ptr{Void}, Int32, Ptr{Ptr{UInt8}}, Ptr{UInt8}, Ptr{Void}),
+        (Cstring, Ptr{Cstring}, Ptr{Void}, Ptr{Void}, Any, Int32,
+         Ptr{Void}, Int32, Ptr{Void}, Int32, Ptr{Void}, Int32, Ptr{Cstring}, Cstring, Ptr{Void}),
         cmd, argv, loop, proc, pp, uvtype(in),
         uvhandle(in), uvtype(out), uvhandle(out), uvtype(err), uvhandle(err),
         pp.cmd.flags, pp.cmd.env === nothing ? C_NULL : pp.cmd.env, isempty(pp.cmd.dir) ? C_NULL : pp.cmd.dir,

base/refpointer.jl

@@ -1,5 +1,16 @@
 # This file is a part of Julia. License is MIT: http://julialang.org/license
 
+# C NUL-terminated string pointers; these can be used in ccall
+# instead of Ptr{Cchar} and Ptr{Cwchar_t}, respectively, to enforce
+# a check for embedded NUL chars in the string (to avoid silent truncation).
+if Int === Int64
+    bitstype 64 Cstring
+    bitstype 64 Cwstring
+else
+    bitstype 32 Cstring
+    bitstype 32 Cwstring
+end
+
 ### General Methods for Ref{T} type
 
 eltype{T}(x::Type{Ref{T}}) = T
@@ -62,10 +73,10 @@ end
 unsafe_convert{T}(::Type{Ptr{Void}}, b::RefArray{T}) = convert(Ptr{Void}, unsafe_convert(Ptr{T}, b))
 
 # convert Arrays to pointer arrays for ccall
-function (::Type{Ref{P}}){P<:Ptr,T<:Ptr}(a::Array{T}) # Ref{P<:Ptr}(a::Array{T<:Ptr})
+function (::Type{Ref{P}}){P<:Union{Ptr,Cwstring,Cstring},T<:Union{Ptr,Cwstring,Cstring}}(a::Array{T}) # Ref{P<:Ptr}(a::Array{T<:Ptr})
     return RefArray(a) # effectively a no-op
 end
-function (::Type{Ref{P}}){P<:Ptr,T}(a::Array{T}) # Ref{P<:Ptr}(a::Array)
+function (::Type{Ref{P}}){P<:Union{Ptr,Cwstring,Cstring},T}(a::Array{T}) # Ref{P<:Ptr}(a::Array)
     if (!isbits(T) && T <: eltype(P))
         # this Array already has the right memory layout for the requested Ref
         return RefArray(a,1,false) # root something, so that this function is type-stable
@@ -82,7 +93,7 @@ function (::Type{Ref{P}}){P<:Ptr,T}(a::Array{T}) # Ref{P<:Ptr}(a::Array)
     end
 end
 cconvert{P<:Ptr,T<:Ptr}(::Union{Type{Ptr{P}},Type{Ref{P}}}, a::Array{T}) = a
-cconvert{P<:Ptr}(::Union{Type{Ptr{P}},Type{Ref{P}}}, a::Array) = Ref{P}(a)
+cconvert{P<:Union{Ptr,Cwstring,Cstring}}(::Union{Type{Ptr{P}},Type{Ref{P}}}, a::Array) = Ref{P}(a)
 
 ###