From e811f1e9d3c5a174e9ccc0d252ac8b05aad9401d Mon Sep 17 00:00:00 2001 From: Rene Donner Date: Fri, 17 Apr 2015 13:35:08 +0200 Subject: [PATCH 01/12] adapt method definitions for tuple types --- src/JLD.jl | 18 +++++++++--------- src/JLD00.jl | 8 ++++---- src/jld_types.jl | 22 +++++++++++----------- src/plain.jl | 28 +++++++++++++--------------- test/extend_test.jl | 2 +- test/jld.jl | 2 +- test/plain.jl | 4 ++-- 7 files changed, 41 insertions(+), 43 deletions(-) diff --git a/src/JLD.jl b/src/JLD.jl index 9a629348c..c26a08d87 100644 --- a/src/JLD.jl +++ b/src/JLD.jl @@ -294,7 +294,7 @@ function delete!(parent::Union(JldFile, JldGroup), path::ByteString) exists(parent, path) || error("$path does not exist in $parent") delete!(parent[path]) end -delete!(parent::Union(JldFile, JldGroup), args::(ByteString...)) = for a in args delete!(parent,a) end +delete!(parent::Union(JldFile, JldGroup), args::@compat Tuple{Vararg{ByteString}}) = for a in args delete!(parent,a) end ismmappable(obj::JldDataset) = ismmappable(obj.plain) readmmap(obj::JldDataset, args...) = readmmap(obj.plain, args...) setindex!(parent::Union(JldFile, JldGroup), val, path::ASCIIString) = write(parent, path, val) @@ -380,7 +380,7 @@ end # Read an array function read_array(obj::JldDataset, dtype::HDF5Datatype, dspace_id::HDF5.Hid, dsel_id::HDF5.Hid, - dims::(Int...)=convert((Int...), HDF5.h5s_get_simple_extent_dims(dspace_id)[1])) + dims::(@compat Tuple{Vararg{Int}})=convert((@compat Tuple{Vararg{Int}}), HDF5.h5s_get_simple_extent_dims(dspace_id)[1])) if HDF5.h5t_get_class(dtype) == HDF5.H5T_REFERENCE return read_refs(obj, refarray_eltype(obj), dspace_id, dsel_id, dims) else @@ -390,7 +390,7 @@ end # Arrays of basic HDF5 kinds function read_vals{S<:HDF5BitsKind}(obj::JldDataset, dtype::HDF5Datatype, T::Union(Type{S}, Type{Complex{S}}), - dspace_id::HDF5.Hid, dsel_id::HDF5.Hid, dims::(Int...)) + dspace_id::HDF5.Hid, dsel_id::HDF5.Hid, dims::@compat Tuple{Vararg{Int}}) if obj.file.mmaparrays && HDF5.iscontiguous(obj.plain) && dsel_id == HDF5.H5S_ALL readmmap(obj.plain, Array{T}) else @@ -402,7 +402,7 @@ end # Arrays of immutables/bitstypes function read_vals(obj::JldDataset, dtype::HDF5Datatype, T::Type, dspace_id::HDF5.Hid, - dsel_id::HDF5.Hid, dims::(Int...)) + dsel_id::HDF5.Hid, dims::@compat Tuple{Vararg{Int}}) out = Array(T, dims) # Empty objects don't need to be read at all T.size == 0 && !T.mutable && return out @@ -437,7 +437,7 @@ end # Arrays of references function read_refs{T}(obj::JldDataset, ::Type{T}, dspace_id::HDF5.Hid, dsel_id::HDF5.Hid, - dims::(Int...)) + dims::@compat Tuple{Vararg{Int}}) refs = Array(HDF5ReferenceObj, dims) HDF5.h5d_read(obj.plain.id, HDF5.H5T_STD_REF_OBJ, dspace_id, dsel_id, HDF5.H5P_DEFAULT, refs) @@ -625,7 +625,7 @@ function write_ref(parent::JldFile, data, wsession::JldWriteSession) # Add reference to reference list ref = HDF5ReferenceObj(HDF5.objinfo(dset).addr) close(dset) - if !isa(data, Tuple) && typeof(data).mutable + if !isa(data, @compat Tuple) && typeof(data).mutable wsession.h5ref[data] = ref end ref @@ -811,7 +811,7 @@ function full_typename(io::IO, file::JldFile, tv::TypeVar) print(io, ')') end end -function full_typename(io::IO, file::JldFile, jltype::(Type...)) +function full_typename(io::IO, file::JldFile, jltype::@compat Tuple{Vararg{Type}}) print(io, '(') for t in jltype full_typename(io, file, t) @@ -976,7 +976,7 @@ function save(filename::AbstractString, dict::Associative; compress::Bool=false) end # Or the names and values may be specified as alternating pairs function save(filename::AbstractString, name::AbstractString, value, pairs...; compress::Bool=false) - if isodd(length(pairs)) || !isa(pairs[1:2:end], (AbstractString...)) + if isodd(length(pairs)) || !isa(pairs[1:2:end], @compat Tuple{Vararg{AbstractString}}) throw(ArgumentError("arguments must be in name-value pairs")) end jldopen(filename, "w"; compress=compress) do file @@ -1001,7 +1001,7 @@ function load(filename::AbstractString, varname::AbstractString) end end load(filename::AbstractString, varnames::AbstractString...) = load(filename, varnames) -function load(filename::AbstractString, varnames::(AbstractString...)) +function load(filename::AbstractString, varnames::@compat Tuple{Vararg{AbstractString}}) jldopen(filename, "r") do file map((var)->read(file, var), varnames) end diff --git a/src/JLD00.jl b/src/JLD00.jl index fe3a101cb..79774a74e 100644 --- a/src/JLD00.jl +++ b/src/JLD00.jl @@ -226,7 +226,7 @@ function delete!(parent::Union(JldFile, JldGroup), path::ByteString) exists(parent, path) || error("$path does not exist in $parent") delete!(parent[path]) end -delete!(parent::Union(JldFile, JldGroup), args::(ByteString...)) = for a in args delete!(parent,a) end +delete!(parent::Union(JldFile, JldGroup), args::Tuple{Vararg{ByteString}}) = for a in args delete!(parent,a) end ismmappable(obj::JldDataset) = ismmappable(obj.plain) readmmap(obj::JldDataset, args...) = readmmap(obj.plain, args...) setindex!(parent::Union(JldFile, JldGroup), val, path::ASCIIString) = write(parent, path, val) @@ -583,9 +583,9 @@ write(parent::Union(JldFile, JldGroup), name::ByteString, n::Nothing) = write(pa # Types # the first is needed to avoid an ambiguity warning if isdefined(Core, :Top) - write{T<:Top}(parent::Union(JldFile, JldGroup), name::ByteString, t::(Type{T}...)) = write(parent, name, Any[t...], "Tuple") + write{T<:Top}(parent::Union(JldFile, JldGroup), name::ByteString, t::Tuple{Vararg{Type{T}}}) = write(parent, name, Any[t...], "Tuple") else - write{T}(parent::Union(JldFile, JldGroup), name::ByteString, t::(Type{T}...)) = write(parent, name, Any[t...], "Tuple") + write{T}(parent::Union(JldFile, JldGroup), name::ByteString, t::Tuple{Vararg{Type{T}}}) = write(parent, name, Any[t...], "Tuple") end write{T}(parent::Union(JldFile, JldGroup), name::ByteString, t::Type{T}) = write(parent, name, nothing, string("Type{", full_typename(t), "}")) @@ -1117,7 +1117,7 @@ function load(filename::AbstractString, varname::AbstractString) end end load(filename::AbstractString, varnames::AbstractString...) = load(filename, varnames) -function load(filename::AbstractString, varnames::(AbstractString...)) +function load(filename::AbstractString, varnames::Tuple{Vararg{AbstractString}}) jldopen(filename, "r") do file map((var)->read(file, var), varnames) end diff --git a/src/jld_types.jl b/src/jld_types.jl index f203a86b0..bcca155e7 100644 --- a/src/jld_types.jl +++ b/src/jld_types.jl @@ -20,7 +20,7 @@ immutable JldTypeInfo end # Get information about the HDF5 types corresponding to Julia types -function JldTypeInfo(parent::JldFile, types::(Type...), commit::Bool) +function JldTypeInfo(parent::JldFile, types::(@compat Tuple{Vararg{Type}}), commit::Bool) dtypes = Array(JldDatatype, length(types)) offsets = Array(Int, length(types)) offset = 0 @@ -292,15 +292,15 @@ h5fieldtype{T,N}(parent::JldFile, ::Type{Array{T,N}}, ::Bool) = JLD_REF_TYPE ## Tuples if INLINE_TUPLE - h5fieldtype(parent::JldFile, T::(Type...), commit::Bool) = + h5fieldtype(parent::JldFile, T::(@compat Tuple{Vararg{Type}}), commit::Bool) = isleaftype(T) ? h5type(parent, T, commit) : JLD_REF_TYPE else - h5fieldtype(parent::JldFile, T::(Type...), ::Bool) = JLD_REF_TYPE + h5fieldtype(parent::JldFile, T::(@compat Tuple{Vararg{Type}}), ::Bool) = JLD_REF_TYPE end -function h5type(parent::JldFile, T::(Type...), commit::Bool) - !isa(T, (Union(Tuple, DataType)...)) && unknown_type_err(T) - T = T::(Union(Tuple, DataType)...) +function h5type(parent::JldFile, T::(@compat Tuple{Vararg{Type}}), commit::Bool) + !isa(T, (@compat Tuple{Vararg{Union((@compat Tuple), DataType)}})) && unknown_type_err(T) + T = T::(@compat Tuple{Vararg{Union((@compat Tuple), DataType)}}) haskey(parent.jlh5type, T) && return parent.jlh5type[T] # Tuples should always be concretely typed, unless we're @@ -331,7 +331,7 @@ function h5type(parent::JldFile, T::(Type...), commit::Bool) end end -function gen_jlconvert(typeinfo::JldTypeInfo, T::(Type...)) +function gen_jlconvert(typeinfo::JldTypeInfo, T::(@compat Tuple{Vararg{Type}})) haskey(JLCONVERT_DEFINED, T) && return ex = Expr(:block) @@ -523,11 +523,11 @@ end ## Common functions for all non-special types (including gen_h5convert) # Whether this datatype should be stored as opaque -isopaque(t::(Type...)) = isa(t, ()) +isopaque(t::@compat Tuple{Vararg{Type}}) = isa(t, ()) isopaque(t::DataType) = isempty(t.names) # The size of this datatype in the HDF5 file (if opaque) -opaquesize(t::(DataType...)) = 1 +opaquesize(t::@compat Tuple{Vararg{DataType}}) = 1 opaquesize(t::DataType) = max(1, t.size) # Whether a type that is stored inline in HDF5 should be stored as a @@ -535,7 +535,7 @@ opaquesize(t::DataType) = max(1, t.size) # true for some unions of special types defined above, unless either # INLINE_TUPLE or INLINE_POINTER_IMMUTABLE is true. uses_reference(T::DataType) = !T.pointerfree -uses_reference(::Tuple) = true +uses_reference(::@compat Tuple) = true uses_reference(::UnionType) = true unknown_type_err(T) = @@ -548,7 +548,7 @@ gen_h5convert(parent::JldFile, T) = # There is no point in specializing this function _gen_h5convert(parent::JldFile, T::ANY) dtype = parent.jlh5type[T].dtype - istuple = isa(T, Tuple) + istuple = isa(T, @compat Tuple) if istuple types = T else diff --git a/src/plain.jl b/src/plain.jl index 8b4e605f4..e9d7fba35 100644 --- a/src/plain.jl +++ b/src/plain.jl @@ -432,9 +432,9 @@ type EmptyArray{T}; end # Stub types to encode fixed-size arrays for H5T_ARRAY immutable DimSize{N}; end # Int-wrapper (can't use tuple of Int as param) -immutable FixedArray{T,D<:(DimSize...)}; end +immutable FixedArray{T,D<:(@compat Tuple{Vararg{DimSize}})}; end dimsize{N}(::Type{DimSize{N}}) = N -size{T,D}(::Type{FixedArray{T,D}}) = map(dimsize, D)::(Int...) +size{T,D}(::Type{FixedArray{T,D}}) = map(dimsize, D)::(@compat Tuple{Vararg{Int}}) eltype{T,D}(::Type{FixedArray{T,D}}) = T # VLEN objects @@ -593,7 +593,7 @@ function h5read(filename, name::ByteString) dat end -function h5read(filename, name::ByteString, indices::(Union(Range{Int},Int,Colon)...)) +function h5read(filename, name::ByteString, indices::@compat Tuple{Vararg{Union(Range{Int},Int,Colon)}}) local dat fid = h5open(filename, "r") try @@ -767,7 +767,7 @@ function d_create(parent::Union(HDF5File, HDF5Group), path::ByteString, dtype::H HDF5Dataset(h5d_create(parent, path, dtype.id, dspace.id, _link_properties(path), p.id, H5P_DEFAULT), file(parent)) end d_create(parent::Union(HDF5File, HDF5Group), path::ByteString, dtype::HDF5Datatype, dspace_dims::Dims, prop1::ASCIIString, val1, pv...) = d_create(checkvalid(parent), path, dtype, dataspace(dspace_dims), prop1, val1, pv...) -d_create(parent::Union(HDF5File, HDF5Group), path::ByteString, dtype::HDF5Datatype, dspace_dims::(Dims,Dims), prop1::ASCIIString, val1, pv...) = d_create(checkvalid(parent), path, dtype, dataspace(dspace_dims[1], max_dims=dspace_dims[2]), prop1, val1, pv...) +d_create(parent::Union(HDF5File, HDF5Group), path::ByteString, dtype::HDF5Datatype, dspace_dims::(@compat Tuple{Dims,Dims}), prop1::ASCIIString, val1, pv...) = d_create(checkvalid(parent), path, dtype, dataspace(dspace_dims[1], max_dims=dspace_dims[2]), prop1, val1, pv...) d_create(parent::Union(HDF5File, HDF5Group), path::ByteString, dtype::Type, dspace_dims, prop1::ASCIIString, val1, pv...) = d_create(checkvalid(parent), path, datatype(dtype), dataspace(dspace_dims[1], max_dims=dspace_dims[2]), prop1, val1, pv...) # Note that H5Tcreate is very different; H5Tcommit is the analog of these others @@ -889,7 +889,7 @@ function size(obj::Union(HDF5Dataset, HDF5Attribute)) dspace = dataspace(obj) dims, maxdims = get_dims(dspace) close(dspace) - convert((Int...), dims) + convert((@compat Tuple{Vararg{Int}}), dims) end size(dset::Union(HDF5Dataset, HDF5Attribute), d) = d > ndims(dset) ? 1 : size(dset)[d] length(dset::Union(HDF5Dataset, HDF5Attribute)) = prod(size(dset)) @@ -1054,7 +1054,7 @@ dataspace(attr::HDF5Attribute) = HDF5Dataspace(h5a_get_space(checkvalid(attr).id # Create a dataspace from in-memory types dataspace{T<:HDF5BitsKind}(x::T) = HDF5Dataspace(h5s_create(H5S_SCALAR)) -function _dataspace(sz::(Int...), max_dims::Union(Dims, ())=()) +function _dataspace(sz::(@compat Tuple{Vararg{Int}}), max_dims::Union(Dims, @compat Tuple{})=()) dims = Array(Hsize, length(sz)) any_zero = false for i = 1:length(sz) @@ -1078,13 +1078,13 @@ function _dataspace(sz::(Int...), max_dims::Union(Dims, ())=()) end HDF5Dataspace(space_id) end -dataspace(A::Array; max_dims::Union(Dims, ()) = ()) = _dataspace(size(A), max_dims) +dataspace(A::Array; max_dims::Union(Dims, @compat Tuple{}) = ()) = _dataspace(size(A), max_dims) dataspace(str::ByteString) = HDF5Dataspace(h5s_create(H5S_SCALAR)) -dataspace(R::Array{HDF5ReferenceObj}; max_dims::Union(Dims, ())=()) = _dataspace(size(R), max_dims) -dataspace(v::HDF5Vlen; max_dims::Union(Dims, ())=()) = _dataspace(size(v.data), max_dims) +dataspace(R::Array{HDF5ReferenceObj}; max_dims::Union(Dims, @compat Tuple{})=()) = _dataspace(size(R), max_dims) +dataspace(v::HDF5Vlen; max_dims::Union(Dims, @compat Tuple{})=()) = _dataspace(size(v.data), max_dims) dataspace(n::Nothing) = HDF5Dataspace(h5s_create(H5S_NULL)) -dataspace(sz::Dims; max_dims::Union(Dims, ())=()) = _dataspace(sz, max_dims) -dataspace(sz1::Int, sz2::Int, sz3::Int...; max_dims::Union(Dims, ())=()) = _dataspace(tuple(sz1, sz2, sz3...), max_dims) +dataspace(sz::Dims; max_dims::Union(Dims, @compat Tuple{})=()) = _dataspace(sz, max_dims) +dataspace(sz1::Int, sz2::Int, sz3::Int...; max_dims::Union(Dims, @compat Tuple{})=()) = _dataspace(tuple(sz1, sz2, sz3...), max_dims) # Get the array dimensions from a dataspace or a dataset # Returns both dims and maxdims @@ -1578,9 +1578,7 @@ function _setindex!(dset::HDF5Dataset,T::Type, X::Array, indices::Union(Range{In if !(T.parameters[1]<:HDF5BitsKind) error("Dataset indexing (hyperslab) is available only for bits types") end - if length(X) != prod([length(idxs) for idxs in - filter(idx -> isa(idx, Ranges), - [indices[i] for i in 1:length(indices)])]) + if length(X) != prod(map(length, indices)) error("number of elements in range and length of array must be equal") end if eltype(X) != T.parameters[1] @@ -1835,7 +1833,7 @@ end ### Utilities for generating ccall wrapper functions programmatically ### -function ccallexpr(lib::AbstractString, ccallsym::Symbol, outtype, argtypes::Tuple, argsyms::Tuple) +function ccallexpr(lib::AbstractString, ccallsym::Symbol, outtype, argtypes::(@compat Tuple), argsyms::@compat Tuple) ccallargs = Any[Expr(:tuple, Expr(:quote, ccallsym), lib), outtype, Expr(:tuple, argtypes...)] ccallargs = ccallsyms(ccallargs, length(argtypes), argsyms) Expr(:ccall, ccallargs...) diff --git a/test/extend_test.jl b/test/extend_test.jl index 42ea39709..0197b2adf 100644 --- a/test/extend_test.jl +++ b/test/extend_test.jl @@ -1,4 +1,4 @@ -using HDF5 +using HDF5, Compat fn = joinpath(tempdir(),"test.h5") diff --git a/test/jld.jl b/test/jld.jl index e896362ff..7975aaa5d 100644 --- a/test/jld.jl +++ b/test/jld.jl @@ -77,7 +77,7 @@ sa_utf8 = [:α, :β] # SubArray (to test tuple type params) subarray = sub([1:5;], 1:5) # Array of empty tuples (to test tuple type params) -arr_empty_tuple = ()[] +arr_empty_tuple = (@compat Tuple{})[] immutable EmptyImmutable end emptyimmutable = EmptyImmutable() arr_emptyimmutable = [emptyimmutable] diff --git a/test/plain.jl b/test/plain.jl index 332913e53..fc9cc3a90 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -187,7 +187,7 @@ end # Test reading multiple vars at once z = read(fr, "Float64", "Int16") @assert z == (3.2, 4) -@assert typeof(z) == (Float64, Int16) +@assert typeof(z) == @compat Tuple{Float64, Int16} # Test function syntax read(fr, "Float64") do x @assert x == 3.2 @@ -232,6 +232,6 @@ d = h5read(joinpath(test_path, "compound.h5"), "/data") @assert typeof(d) == HDF5.HDF5Compound @assert typeof(d.data) == Array{UInt8,1} @assert length(d.data) == 128 -@assert d.membertype == Type[Float64, HDF5.FixedArray{Float64,(HDF5.DimSize{3},)}, HDF5.FixedArray{Float64,(HDF5.DimSize{3},)}, Float64] +@assert d.membertype == Type[Float64, HDF5.FixedArray{Float64,(@compat Tuple{HDF5.DimSize{3}})}, HDF5.FixedArray{Float64,(@compat Tuple{HDF5.DimSize{3}})}, Float64] @assert d.membername == ASCIIString["wgt", "xyz", "uvw", "E"] @assert d.memberoffset == UInt64[0x00, 0x08, 0x20, 0x38] From 5c115bdacae18517a389ce609e29829db9b39432 Mon Sep 17 00:00:00 2001 From: Rene Donner Date: Mon, 20 Apr 2015 08:58:01 +0100 Subject: [PATCH 02/12] solve .names issue --- src/JLD00.jl | 12 +- src/jld_types.jl | 10 +- src/plain.jl | 6 + test/jld.jl | 460 +++++++++++++++++++++++------------------------ test/plain.jl | 14 +- 5 files changed, 254 insertions(+), 248 deletions(-) diff --git a/src/JLD00.jl b/src/JLD00.jl index 79774a74e..bdfbbe377 100644 --- a/src/JLD00.jl +++ b/src/JLD00.jl @@ -436,7 +436,7 @@ end # CompositeKind function read(obj::JldDataset, T::DataType) - if isempty(T.names) && T.size > 0 + if isempty(getnames(T)) && T.size > 0 return read_bitstype(obj, T) end local x @@ -453,12 +453,12 @@ function read(obj::JldDataset, T::DataType) if length(v) == 0 x = ccall(:jl_new_struct, Any, (Any,Any...), T) else - n = T.names + n = getnames(T) if length(v) != length(n) error("Wrong number of fields") end if !T.mutable - x = ccall(:jl_new_structv, Any, (Any,Ptr{Void},UInt32), T, v, length(T.names)) + x = ccall(:jl_new_structv, Any, (Any,Ptr{Void},UInt32), T, v, length(getnames(T))) else x = ccall(:jl_new_struct_uninit, Any, (Any,), T) for i = 1:length(v) @@ -720,7 +720,7 @@ write(parent::Union(JldFile, JldGroup), name::ByteString, s; rootmodule="") = wr function write_composite(parent::Union(JldFile, JldGroup), name::ByteString, s; rootmodule="") T = typeof(s) - if isempty(T.names) + if isempty(getnames(T)) if T.size > 0 return write_bitstype(parent, name, s) end @@ -730,7 +730,7 @@ function write_composite(parent::Union(JldFile, JldGroup), name::ByteString, s; return end Tname = string(T.name.name) - n = T.names + n = getnames(T) local gtypes if !exists(file(parent), pathtypes) gtypes = g_create(file(parent), pathtypes) @@ -803,7 +803,7 @@ function has_pointer_field(obj::Tuple, name) end function has_pointer_field(obj, name) - names = typeof(obj).names + names = getnames(typeof(obj)) for fieldname in names if isdefined(obj, fieldname) x = getfield(obj, fieldname) diff --git a/src/jld_types.jl b/src/jld_types.jl index bcca155e7..bbaadd2c3 100644 --- a/src/jld_types.jl +++ b/src/jld_types.jl @@ -382,7 +382,7 @@ function h5type(parent::JldFile, T::ANY, commit::Bool) id = HDF5.h5t_create(HDF5.H5T_COMPOUND, typeinfo.size) for i = 1:length(typeinfo.offsets) fielddtype = typeinfo.dtypes[i] - HDF5.h5t_insert(id, mangle_name(fielddtype, T.names[i]), typeinfo.offsets[i], fielddtype) + HDF5.h5t_insert(id, mangle_name(fielddtype, HDF5.getnames(T)[i]), typeinfo.offsets[i], fielddtype) end end @@ -410,11 +410,11 @@ function _gen_jlconvert_type(typeinfo::JldTypeInfo, T::ANY) push!(args, quote ref = unsafe_load(convert(Ptr{HDF5ReferenceObj}, ptr)+$h5offset) if ref != HDF5.HDF5ReferenceObj_NULL - out.$(T.names[i]) = convert($(T.types[i]), read_ref(file, ref)) + out.$(HDF5.getnames(T)[i]) = convert($(T.types[i]), read_ref(file, ref)) end end) else - push!(args, :(out.$(T.names[i]) = jlconvert($(T.types[i]), file, ptr+$h5offset))) + push!(args, :(out.$(HDF5.getnames(T)[i]) = jlconvert($(T.types[i]), file, ptr+$h5offset))) end end @eval function jlconvert(::Type{$T}, file::JldFile, ptr::Ptr) @@ -491,7 +491,7 @@ const DONT_STORE_SINGLETON_IMMUTABLES = VERSION >= v"0.4.0-dev+385" function gen_jlconvert(typeinfo::JldTypeInfo, T::ANY) haskey(JLCONVERT_DEFINED, T) && return - if isempty(T.names) + if isempty(HDF5.getnames(T)) if T.size == 0 @eval begin jlconvert(::Type{$T}, ::JldFile, ::Ptr) = $T() @@ -524,7 +524,7 @@ end # Whether this datatype should be stored as opaque isopaque(t::@compat Tuple{Vararg{Type}}) = isa(t, ()) -isopaque(t::DataType) = isempty(t.names) +isopaque(t::DataType) = isempty(HDF5.getnames(t)) # The size of this datatype in the HDF5 file (if opaque) opaquesize(t::@compat Tuple{Vararg{DataType}}) = 1 diff --git a/src/plain.jl b/src/plain.jl index e9d7fba35..7e8ad9fb6 100644 --- a/src/plain.jl +++ b/src/plain.jl @@ -14,6 +14,12 @@ import Base: close, convert, done, dump, eltype, endof, flush, getindex, include("datafile.jl") +if VERSION < v"0.4.0-dev+4319" + getnames(T::DataType) = T.names +else + getnames(T::DataType) = T.name.names +end + ## C types typealias C_time_t Int diff --git a/test/jld.jl b/test/jld.jl index 7975aaa5d..03e879012 100644 --- a/test/jld.jl +++ b/test/jld.jl @@ -294,241 +294,241 @@ close(fid) rm(fn) -for compress in (true,false) - fnc = compress ? fn*".c" : fn # workaround #176 - fid = jldopen(fnc, "w", compress=compress) - @write fid x - @write fid A - @write fid Aarray - @write fid str - @write fid stringsA - @write fid stringsU - @write fid strings16 - @write fid strings16_2d - @write fid empty_string - @write fid empty_string_array - @write fid empty_array_of_strings - @write fid tf - @write fid TF - @write fid AB - @write fid t - @write fid c - @write fid cint - @write fid C - @write fid emptyA - @write fid emptyB - @write fid ms - @write fid msempty - @write fid sym - @write fid syms - @write fid d - if compress # work around #176 - @write fid ex - end - @write fid T - @write fid char - @write fid unicode_char - @write fid α - @write fid β - @write fid vv - @write fid cpus - @write fid rng - @write fid typevar - @write fid typevar_lb - @write fid typevar_ub - @write fid typevar_lb_ub - @write fid undef - @write fid undefs - @write fid ms_undef - @test_throws JLD.PointerException @write fid objwithpointer - @write fid bt - @write fid sa_asc - @write fid sa_utf8 - @write fid subarray - @write fid arr_empty_tuple - @write fid emptyimmutable - @write fid emptytype - @write fid arr_emptyimmutable - @write fid arr_emptytype - @write fid emptyii - @write fid emptyit - @write fid emptyti - @write fid emptytt - @write fid emptyiiotherfield - @write fid unicodestruct☺ - @write fid array_of_matrices - @write fid tup - @write fid empty_tup - @write fid nonpointerfree_immutable_1 - @write fid nonpointerfree_immutable_2 - @write fid nonpointerfree_immutable_3 - @write fid vague - @write fid bitsunion - @write fid typeunionfield - @write fid genericunionfield - @write fid arr_ref - @write fid obj_ref - @write fid padding_test - @write fid empty_arr_1 - @write fid empty_arr_2 - @write fid empty_arr_3 - @write fid empty_arr_4 - @write fid bigdata - @write fid bigfloats - @write fid bigints - @write fid none - @write fid nonearr - @write fid scalar_nothing - @write fid vector_nothing - @write fid Abig - @write fid Bbig - @write fid Sbig - @test_throws ErrorException @write fid bitsparamint16 - @write fid bitsparamfloat - @write fid bitsparambool - @write fid bitsparamsymbol - @write fid bitsparamint - @write fid bitsparamuint - @write fid tuple_of_tuples +# for compress in (true,false) +# fnc = compress ? fn*".c" : fn # workaround #176 +# fid = jldopen(fnc, "w", compress=compress) +# @write fid x +# @write fid A +# @write fid Aarray +# @write fid str +# @write fid stringsA +# @write fid stringsU +# @write fid strings16 +# @write fid strings16_2d +# @write fid empty_string +# @write fid empty_string_array +# @write fid empty_array_of_strings +# @write fid tf +# @write fid TF +# @write fid AB +# @write fid t +# @write fid c +# @write fid cint +# @write fid C +# @write fid emptyA +# @write fid emptyB +# @write fid ms +# @write fid msempty +# @write fid sym +# @write fid syms +# @write fid d +# if compress # work around #176 +# @write fid ex +# end +# @write fid T +# @write fid char +# @write fid unicode_char +# @write fid α +# @write fid β +# @write fid vv +# @write fid cpus +# @write fid rng +# @write fid typevar +# @write fid typevar_lb +# @write fid typevar_ub +# @write fid typevar_lb_ub +# @write fid undef +# @write fid undefs +# @write fid ms_undef +# @test_throws JLD.PointerException @write fid objwithpointer +# @write fid bt +# @write fid sa_asc +# @write fid sa_utf8 +# @write fid subarray +# @write fid arr_empty_tuple +# @write fid emptyimmutable +# @write fid emptytype +# @write fid arr_emptyimmutable +# @write fid arr_emptytype +# @write fid emptyii +# @write fid emptyit +# @write fid emptyti +# @write fid emptytt +# @write fid emptyiiotherfield +# @write fid unicodestruct☺ +# @write fid array_of_matrices +# @write fid tup +# @write fid empty_tup +# @write fid nonpointerfree_immutable_1 +# @write fid nonpointerfree_immutable_2 +# @write fid nonpointerfree_immutable_3 +# @write fid vague +# @write fid bitsunion +# @write fid typeunionfield +# @write fid genericunionfield +# @write fid arr_ref +# @write fid obj_ref +# @write fid padding_test +# @write fid empty_arr_1 +# @write fid empty_arr_2 +# @write fid empty_arr_3 +# @write fid empty_arr_4 +# @write fid bigdata +# @write fid bigfloats +# @write fid bigints +# @write fid none +# @write fid nonearr +# @write fid scalar_nothing +# @write fid vector_nothing +# @write fid Abig +# @write fid Bbig +# @write fid Sbig +# @test_throws ErrorException @write fid bitsparamint16 +# @write fid bitsparamfloat +# @write fid bitsparambool +# @write fid bitsparamsymbol +# @write fid bitsparamint +# @write fid bitsparamuint +# @write fid tuple_of_tuples - # Make sure we can create groups (i.e., use HDF5 features) - g = g_create(fid, "mygroup") - i = 7 - @write g i - write(fid, "group1/x", Any[1]) - write(fid, "group2/x", Any[2]) - close(fid) +# # Make sure we can create groups (i.e., use HDF5 features) +# g = g_create(fid, "mygroup") +# i = 7 +# @write g i +# write(fid, "group1/x", Any[1]) +# write(fid, "group2/x", Any[2]) +# close(fid) - # mmapping currently fails on Windows; re-enable once it can work - for mmap = (@windows ? false : (false, true)) - fidr = jldopen(fnc, "r", mmaparrays=mmap) - @check fidr x - @check fidr A - dsetA = fidr["A"] - @test ndims(dsetA) == ndims(A) - @test size(dsetA) == size(A) - @test size(dsetA, 1) == size(A, 1) - @test size(dsetA, 2) == size(A, 2) - @test size(dsetA, 3) == size(A, 3) - @check fidr Aarray - @check fidr str - @check fidr stringsA - @check fidr stringsU - @check fidr strings16 - @check fidr strings16_2d - @check fidr empty_string - @check fidr empty_string_array - @check fidr empty_array_of_strings - @check fidr tf - @check fidr TF - @check fidr AB - @check fidr t - @check fidr c - @check fidr cint - @check fidr C - @check fidr emptyA - @check fidr emptyB - @check fidr ms - @check fidr msempty - @check fidr sym - @check fidr syms - @check fidr d - if compress # work around #176 - exr = read(fidr, "ex") # line numbers are stripped, don't expect equality - checkexpr(ex, exr) - end - @check fidr T - @check fidr char - @check fidr unicode_char - @check fidr α - @check fidr β - @check fidr vv - @check fidr cpus - @check fidr rng - @check fidr typevar - @check fidr typevar_lb - @check fidr typevar_ub - @check fidr typevar_lb_ub +# # mmapping currently fails on Windows; re-enable once it can work +# for mmap = (@windows ? false : (false, true)) +# fidr = jldopen(fnc, "r", mmaparrays=mmap) +# @check fidr x +# @check fidr A +# dsetA = fidr["A"] +# @test ndims(dsetA) == ndims(A) +# @test size(dsetA) == size(A) +# @test size(dsetA, 1) == size(A, 1) +# @test size(dsetA, 2) == size(A, 2) +# @test size(dsetA, 3) == size(A, 3) +# @check fidr Aarray +# @check fidr str +# @check fidr stringsA +# @check fidr stringsU +# @check fidr strings16 +# @check fidr strings16_2d +# @check fidr empty_string +# @check fidr empty_string_array +# @check fidr empty_array_of_strings +# @check fidr tf +# @check fidr TF +# @check fidr AB +# @check fidr t +# @check fidr c +# @check fidr cint +# @check fidr C +# @check fidr emptyA +# @check fidr emptyB +# @check fidr ms +# @check fidr msempty +# @check fidr sym +# @check fidr syms +# @check fidr d +# if compress # work around #176 +# exr = read(fidr, "ex") # line numbers are stripped, don't expect equality +# checkexpr(ex, exr) +# end +# @check fidr T +# @check fidr char +# @check fidr unicode_char +# @check fidr α +# @check fidr β +# @check fidr vv +# @check fidr cpus +# @check fidr rng +# @check fidr typevar +# @check fidr typevar_lb +# @check fidr typevar_ub +# @check fidr typevar_lb_ub - # Special cases for reading undefs - undef = read(fidr, "undef") - if !isa(undef, Array{Any, 1}) || length(undef) != 1 || isdefined(undef, 1) - error("For undef, read value does not agree with written value") - end - undefs = read(fidr, "undefs") - if !isa(undefs, Array{Any, 2}) || length(undefs) != 4 || any(map(i->isdefined(undefs, i), 1:4)) - error("For undefs, read value does not agree with written value") - end - ms_undef = read(fidr, "ms_undef") - if !isa(ms_undef, MyStruct) || ms_undef.len != 0 || isdefined(ms_undef, :data) - error("For ms_undef, read value does not agree with written value") - end +# # Special cases for reading undefs +# undef = read(fidr, "undef") +# if !isa(undef, Array{Any, 1}) || length(undef) != 1 || isdefined(undef, 1) +# error("For undef, read value does not agree with written value") +# end +# undefs = read(fidr, "undefs") +# if !isa(undefs, Array{Any, 2}) || length(undefs) != 4 || any(map(i->isdefined(undefs, i), 1:4)) +# error("For undefs, read value does not agree with written value") +# end +# ms_undef = read(fidr, "ms_undef") +# if !isa(ms_undef, MyStruct) || ms_undef.len != 0 || isdefined(ms_undef, :data) +# error("For ms_undef, read value does not agree with written value") +# end - @check fidr bt - @check fidr sa_asc - @check fidr sa_utf8 - @check fidr subarray - @check fidr arr_empty_tuple - @check fidr emptyimmutable - @check fidr emptytype - @check fidr arr_emptyimmutable - @check fidr arr_emptytype - @check fidr emptyii - @check fidr emptyit - @check fidr emptyti - @check fidr emptytt - @check fidr emptyiiotherfield - @check fidr unicodestruct☺ - @check fidr array_of_matrices - @check fidr tup - @check fidr empty_tup - @check fidr nonpointerfree_immutable_1 - @check fidr nonpointerfree_immutable_2 - @check fidr nonpointerfree_immutable_3 - vaguer = read(fidr, "vague") - @test typeof(vaguer) == typeof(vague) && vaguer.name == vague.name - @check fidr bitsunion - @check fidr typeunionfield - @check fidr genericunionfield +# @check fidr bt +# @check fidr sa_asc +# @check fidr sa_utf8 +# @check fidr subarray +# @check fidr arr_empty_tuple +# @check fidr emptyimmutable +# @check fidr emptytype +# @check fidr arr_emptyimmutable +# @check fidr arr_emptytype +# @check fidr emptyii +# @check fidr emptyit +# @check fidr emptyti +# @check fidr emptytt +# @check fidr emptyiiotherfield +# @check fidr unicodestruct☺ +# @check fidr array_of_matrices +# @check fidr tup +# @check fidr empty_tup +# @check fidr nonpointerfree_immutable_1 +# @check fidr nonpointerfree_immutable_2 +# @check fidr nonpointerfree_immutable_3 +# vaguer = read(fidr, "vague") +# @test typeof(vaguer) == typeof(vague) && vaguer.name == vague.name +# @check fidr bitsunion +# @check fidr typeunionfield +# @check fidr genericunionfield - arr = read(fidr, "arr_ref") - @test arr == arr_ref - @test arr[1] === arr[2] +# arr = read(fidr, "arr_ref") +# @test arr == arr_ref +# @test arr[1] === arr[2] - obj = read(fidr, "obj_ref") - @test obj.x.x === obj.x.y == obj.y.x === obj.y.y - @test obj.x !== obj.y +# obj = read(fidr, "obj_ref") +# @test obj.x.x === obj.x.y == obj.y.x === obj.y.y +# @test obj.x !== obj.y - @check fidr padding_test - @check fidr empty_arr_1 - @check fidr empty_arr_2 - @check fidr empty_arr_3 - @check fidr empty_arr_4 - @check fidr bigdata - @check fidr bigfloats - @check fidr bigints - @check fidr none - @check fidr nonearr - @check fidr scalar_nothing - @check fidr vector_nothing - @check fidr Abig - @check fidr Bbig - @check fidr Sbig - @check fidr bitsparamfloat - @check fidr bitsparambool - @check fidr bitsparamsymbol - @check fidr bitsparamint - @check fidr bitsparamuint - @check fidr tuple_of_tuples +# @check fidr padding_test +# @check fidr empty_arr_1 +# @check fidr empty_arr_2 +# @check fidr empty_arr_3 +# @check fidr empty_arr_4 +# @check fidr bigdata +# @check fidr bigfloats +# @check fidr bigints +# @check fidr none +# @check fidr nonearr +# @check fidr scalar_nothing +# @check fidr vector_nothing +# @check fidr Abig +# @check fidr Bbig +# @check fidr Sbig +# @check fidr bitsparamfloat +# @check fidr bitsparambool +# @check fidr bitsparamsymbol +# @check fidr bitsparamint +# @check fidr bitsparamuint +# @check fidr tuple_of_tuples - x1 = read(fidr, "group1/x") - @assert x1 == Any[1] - x2 = read(fidr, "group2/x") - @assert x2 == Any[2] +# x1 = read(fidr, "group1/x") +# @assert x1 == Any[1] +# x2 = read(fidr, "group2/x") +# @assert x2 == Any[2] - close(fidr) - end -end # compress in (true,false) +# close(fidr) +# end +# end # compress in (true,false) for compress in (false,true) # object references in a write session @@ -734,17 +734,17 @@ jldopen(fn, "r") do file for i = 1:5 @test x[i].x.x == i end - @test typeof(read(file, "x6")).names == () + @test HDF5.getnames(typeof(read(file, "x6"))) == (@compat Tuple{}) @test reinterpret(UInt8, read(file, "x7")) == 0x77 x = read(file, "x8") @test x.a.x == 2 @test x.b.x.x == 3 - @test typeof(x.c).names == () + @test HDF5.getnames(typeof(x.c)) == (@compat Tuple{}) @test reinterpret(UInt8, x.d) == 0x12 x = read(file, "x9") - @test isa(x, Tuple) + @test isa(x, (@compat Tuple)) @test length(x) == 3 @test x[1].x == 1 @test isa(x[2], Tuple) @@ -753,7 +753,7 @@ jldopen(fn, "r") do file for i = 1:5 @test x[2][2][i].x.x == i end - @test typeof(x[3]).names == () + @test HDF5.getnames(typeof(x[3])) == (@compat Tuple{}) end # Issue #176 diff --git a/test/plain.jl b/test/plain.jl index fc9cc3a90..f64d6749c 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -228,10 +228,10 @@ g = fid["mygroup"] close(g) close(fid) -d = h5read(joinpath(test_path, "compound.h5"), "/data") -@assert typeof(d) == HDF5.HDF5Compound -@assert typeof(d.data) == Array{UInt8,1} -@assert length(d.data) == 128 -@assert d.membertype == Type[Float64, HDF5.FixedArray{Float64,(@compat Tuple{HDF5.DimSize{3}})}, HDF5.FixedArray{Float64,(@compat Tuple{HDF5.DimSize{3}})}, Float64] -@assert d.membername == ASCIIString["wgt", "xyz", "uvw", "E"] -@assert d.memberoffset == UInt64[0x00, 0x08, 0x20, 0x38] +# d = h5read(joinpath(test_path, "compound.h5"), "/data") +# @assert typeof(d) == HDF5.HDF5Compound +# @assert typeof(d.data) == Array{UInt8,1} +# @assert length(d.data) == 128 +# @assert d.membertype == Type[Float64, HDF5.FixedArray{Float64,(@compat Tuple{HDF5.DimSize{3}})}, HDF5.FixedArray{Float64,(@compat Tuple{HDF5.DimSize{3}})}, Float64] +# @assert d.membername == ASCIIString["wgt", "xyz", "uvw", "E"] +# @assert d.memberoffset == UInt64[0x00, 0x08, 0x20, 0x38] From fbdc6316f6835c27b2ed22717b95b041d9feb71a Mon Sep 17 00:00:00 2001 From: Rene Donner Date: Thu, 23 Apr 2015 17:34:07 +0200 Subject: [PATCH 03/12] remove unnecessary @compat --- src/JLD.jl | 2 +- test/jld.jl | 452 +++++++++++++++++++++++++------------------------- test/plain.jl | 14 +- 3 files changed, 234 insertions(+), 234 deletions(-) diff --git a/src/JLD.jl b/src/JLD.jl index c26a08d87..f104fa50c 100644 --- a/src/JLD.jl +++ b/src/JLD.jl @@ -625,7 +625,7 @@ function write_ref(parent::JldFile, data, wsession::JldWriteSession) # Add reference to reference list ref = HDF5ReferenceObj(HDF5.objinfo(dset).addr) close(dset) - if !isa(data, @compat Tuple) && typeof(data).mutable + if !isa(data, Tuple) && typeof(data).mutable wsession.h5ref[data] = ref end ref diff --git a/test/jld.jl b/test/jld.jl index 03e879012..3f7758934 100644 --- a/test/jld.jl +++ b/test/jld.jl @@ -294,241 +294,241 @@ close(fid) rm(fn) -# for compress in (true,false) -# fnc = compress ? fn*".c" : fn # workaround #176 -# fid = jldopen(fnc, "w", compress=compress) -# @write fid x -# @write fid A -# @write fid Aarray -# @write fid str -# @write fid stringsA -# @write fid stringsU -# @write fid strings16 -# @write fid strings16_2d -# @write fid empty_string -# @write fid empty_string_array -# @write fid empty_array_of_strings -# @write fid tf -# @write fid TF -# @write fid AB -# @write fid t -# @write fid c -# @write fid cint -# @write fid C -# @write fid emptyA -# @write fid emptyB -# @write fid ms -# @write fid msempty -# @write fid sym -# @write fid syms -# @write fid d -# if compress # work around #176 -# @write fid ex -# end -# @write fid T -# @write fid char -# @write fid unicode_char -# @write fid α -# @write fid β -# @write fid vv -# @write fid cpus -# @write fid rng -# @write fid typevar -# @write fid typevar_lb -# @write fid typevar_ub -# @write fid typevar_lb_ub -# @write fid undef -# @write fid undefs -# @write fid ms_undef -# @test_throws JLD.PointerException @write fid objwithpointer -# @write fid bt -# @write fid sa_asc -# @write fid sa_utf8 -# @write fid subarray -# @write fid arr_empty_tuple -# @write fid emptyimmutable -# @write fid emptytype -# @write fid arr_emptyimmutable -# @write fid arr_emptytype -# @write fid emptyii -# @write fid emptyit -# @write fid emptyti -# @write fid emptytt -# @write fid emptyiiotherfield -# @write fid unicodestruct☺ -# @write fid array_of_matrices -# @write fid tup -# @write fid empty_tup -# @write fid nonpointerfree_immutable_1 -# @write fid nonpointerfree_immutable_2 -# @write fid nonpointerfree_immutable_3 -# @write fid vague -# @write fid bitsunion -# @write fid typeunionfield -# @write fid genericunionfield -# @write fid arr_ref -# @write fid obj_ref -# @write fid padding_test -# @write fid empty_arr_1 -# @write fid empty_arr_2 -# @write fid empty_arr_3 -# @write fid empty_arr_4 -# @write fid bigdata -# @write fid bigfloats -# @write fid bigints -# @write fid none -# @write fid nonearr -# @write fid scalar_nothing -# @write fid vector_nothing -# @write fid Abig -# @write fid Bbig -# @write fid Sbig -# @test_throws ErrorException @write fid bitsparamint16 -# @write fid bitsparamfloat -# @write fid bitsparambool -# @write fid bitsparamsymbol -# @write fid bitsparamint -# @write fid bitsparamuint -# @write fid tuple_of_tuples +for compress in (true,false) + fnc = compress ? fn*".c" : fn # workaround #176 + fid = jldopen(fnc, "w", compress=compress) + @write fid x + @write fid A + @write fid Aarray + @write fid str + @write fid stringsA + @write fid stringsU + @write fid strings16 + @write fid strings16_2d + @write fid empty_string + @write fid empty_string_array + @write fid empty_array_of_strings + @write fid tf + @write fid TF + @write fid AB + @write fid t + @write fid c + @write fid cint + @write fid C + @write fid emptyA + @write fid emptyB + @write fid ms + @write fid msempty + @write fid sym + @write fid syms + @write fid d + if compress # work around #176 + @write fid ex + end + @write fid T + @write fid char + @write fid unicode_char + @write fid α + @write fid β + @write fid vv + @write fid cpus + @write fid rng + @write fid typevar + @write fid typevar_lb + @write fid typevar_ub + @write fid typevar_lb_ub + @write fid undef + @write fid undefs + @write fid ms_undef + @test_throws JLD.PointerException @write fid objwithpointer + @write fid bt + @write fid sa_asc + @write fid sa_utf8 + @write fid subarray + @write fid arr_empty_tuple + @write fid emptyimmutable + @write fid emptytype + @write fid arr_emptyimmutable + @write fid arr_emptytype + @write fid emptyii + @write fid emptyit + @write fid emptyti + @write fid emptytt + @write fid emptyiiotherfield + @write fid unicodestruct☺ + @write fid array_of_matrices + @write fid tup + @write fid empty_tup + @write fid nonpointerfree_immutable_1 + @write fid nonpointerfree_immutable_2 + @write fid nonpointerfree_immutable_3 + @write fid vague + @write fid bitsunion + @write fid typeunionfield + @write fid genericunionfield + @write fid arr_ref + @write fid obj_ref + @write fid padding_test + @write fid empty_arr_1 + @write fid empty_arr_2 + @write fid empty_arr_3 + @write fid empty_arr_4 + @write fid bigdata + @write fid bigfloats + @write fid bigints + @write fid none + @write fid nonearr + @write fid scalar_nothing + @write fid vector_nothing + @write fid Abig + @write fid Bbig + @write fid Sbig + @test_throws ErrorException @write fid bitsparamint16 + @write fid bitsparamfloat + @write fid bitsparambool + @write fid bitsparamsymbol + @write fid bitsparamint + @write fid bitsparamuint + @write fid tuple_of_tuples -# # Make sure we can create groups (i.e., use HDF5 features) -# g = g_create(fid, "mygroup") -# i = 7 -# @write g i -# write(fid, "group1/x", Any[1]) -# write(fid, "group2/x", Any[2]) -# close(fid) + # Make sure we can create groups (i.e., use HDF5 features) + g = g_create(fid, "mygroup") + i = 7 + @write g i + write(fid, "group1/x", Any[1]) + write(fid, "group2/x", Any[2]) + close(fid) -# # mmapping currently fails on Windows; re-enable once it can work -# for mmap = (@windows ? false : (false, true)) -# fidr = jldopen(fnc, "r", mmaparrays=mmap) -# @check fidr x -# @check fidr A -# dsetA = fidr["A"] -# @test ndims(dsetA) == ndims(A) -# @test size(dsetA) == size(A) -# @test size(dsetA, 1) == size(A, 1) -# @test size(dsetA, 2) == size(A, 2) -# @test size(dsetA, 3) == size(A, 3) -# @check fidr Aarray -# @check fidr str -# @check fidr stringsA -# @check fidr stringsU -# @check fidr strings16 -# @check fidr strings16_2d -# @check fidr empty_string -# @check fidr empty_string_array -# @check fidr empty_array_of_strings -# @check fidr tf -# @check fidr TF -# @check fidr AB -# @check fidr t -# @check fidr c -# @check fidr cint -# @check fidr C -# @check fidr emptyA -# @check fidr emptyB -# @check fidr ms -# @check fidr msempty -# @check fidr sym -# @check fidr syms -# @check fidr d -# if compress # work around #176 -# exr = read(fidr, "ex") # line numbers are stripped, don't expect equality -# checkexpr(ex, exr) -# end -# @check fidr T -# @check fidr char -# @check fidr unicode_char -# @check fidr α -# @check fidr β -# @check fidr vv -# @check fidr cpus -# @check fidr rng -# @check fidr typevar -# @check fidr typevar_lb -# @check fidr typevar_ub -# @check fidr typevar_lb_ub + # mmapping currently fails on Windows; re-enable once it can work + for mmap = (@windows ? false : (false, true)) + fidr = jldopen(fnc, "r", mmaparrays=mmap) + @check fidr x + @check fidr A + dsetA = fidr["A"] + @test ndims(dsetA) == ndims(A) + @test size(dsetA) == size(A) + @test size(dsetA, 1) == size(A, 1) + @test size(dsetA, 2) == size(A, 2) + @test size(dsetA, 3) == size(A, 3) + @check fidr Aarray + @check fidr str + @check fidr stringsA + @check fidr stringsU + @check fidr strings16 + @check fidr strings16_2d + @check fidr empty_string + @check fidr empty_string_array + @check fidr empty_array_of_strings + @check fidr tf + @check fidr TF + @check fidr AB + @check fidr t + @check fidr c + @check fidr cint + @check fidr C + @check fidr emptyA + @check fidr emptyB + @check fidr ms + @check fidr msempty + @check fidr sym + @check fidr syms + @check fidr d + if compress # work around #176 + exr = read(fidr, "ex") # line numbers are stripped, don't expect equality + checkexpr(ex, exr) + end + @check fidr T + @check fidr char + @check fidr unicode_char + @check fidr α + @check fidr β + @check fidr vv + @check fidr cpus + @check fidr rng + @check fidr typevar + @check fidr typevar_lb + @check fidr typevar_ub + @check fidr typevar_lb_ub -# # Special cases for reading undefs -# undef = read(fidr, "undef") -# if !isa(undef, Array{Any, 1}) || length(undef) != 1 || isdefined(undef, 1) -# error("For undef, read value does not agree with written value") -# end -# undefs = read(fidr, "undefs") -# if !isa(undefs, Array{Any, 2}) || length(undefs) != 4 || any(map(i->isdefined(undefs, i), 1:4)) -# error("For undefs, read value does not agree with written value") -# end -# ms_undef = read(fidr, "ms_undef") -# if !isa(ms_undef, MyStruct) || ms_undef.len != 0 || isdefined(ms_undef, :data) -# error("For ms_undef, read value does not agree with written value") -# end + # Special cases for reading undefs + undef = read(fidr, "undef") + if !isa(undef, Array{Any, 1}) || length(undef) != 1 || isdefined(undef, 1) + error("For undef, read value does not agree with written value") + end + undefs = read(fidr, "undefs") + if !isa(undefs, Array{Any, 2}) || length(undefs) != 4 || any(map(i->isdefined(undefs, i), 1:4)) + error("For undefs, read value does not agree with written value") + end + ms_undef = read(fidr, "ms_undef") + if !isa(ms_undef, MyStruct) || ms_undef.len != 0 || isdefined(ms_undef, :data) + error("For ms_undef, read value does not agree with written value") + end -# @check fidr bt -# @check fidr sa_asc -# @check fidr sa_utf8 -# @check fidr subarray -# @check fidr arr_empty_tuple -# @check fidr emptyimmutable -# @check fidr emptytype -# @check fidr arr_emptyimmutable -# @check fidr arr_emptytype -# @check fidr emptyii -# @check fidr emptyit -# @check fidr emptyti -# @check fidr emptytt -# @check fidr emptyiiotherfield -# @check fidr unicodestruct☺ -# @check fidr array_of_matrices -# @check fidr tup -# @check fidr empty_tup -# @check fidr nonpointerfree_immutable_1 -# @check fidr nonpointerfree_immutable_2 -# @check fidr nonpointerfree_immutable_3 -# vaguer = read(fidr, "vague") -# @test typeof(vaguer) == typeof(vague) && vaguer.name == vague.name -# @check fidr bitsunion -# @check fidr typeunionfield -# @check fidr genericunionfield + @check fidr bt + @check fidr sa_asc + @check fidr sa_utf8 + @check fidr subarray + @check fidr arr_empty_tuple + @check fidr emptyimmutable + @check fidr emptytype + @check fidr arr_emptyimmutable + @check fidr arr_emptytype + @check fidr emptyii + @check fidr emptyit + @check fidr emptyti + @check fidr emptytt + @check fidr emptyiiotherfield + @check fidr unicodestruct☺ + @check fidr array_of_matrices + @check fidr tup + @check fidr empty_tup + @check fidr nonpointerfree_immutable_1 + @check fidr nonpointerfree_immutable_2 + @check fidr nonpointerfree_immutable_3 + vaguer = read(fidr, "vague") + @test typeof(vaguer) == typeof(vague) && vaguer.name == vague.name + @check fidr bitsunion + @check fidr typeunionfield + @check fidr genericunionfield -# arr = read(fidr, "arr_ref") -# @test arr == arr_ref -# @test arr[1] === arr[2] + arr = read(fidr, "arr_ref") + @test arr == arr_ref + @test arr[1] === arr[2] -# obj = read(fidr, "obj_ref") -# @test obj.x.x === obj.x.y == obj.y.x === obj.y.y -# @test obj.x !== obj.y + obj = read(fidr, "obj_ref") + @test obj.x.x === obj.x.y == obj.y.x === obj.y.y + @test obj.x !== obj.y -# @check fidr padding_test -# @check fidr empty_arr_1 -# @check fidr empty_arr_2 -# @check fidr empty_arr_3 -# @check fidr empty_arr_4 -# @check fidr bigdata -# @check fidr bigfloats -# @check fidr bigints -# @check fidr none -# @check fidr nonearr -# @check fidr scalar_nothing -# @check fidr vector_nothing -# @check fidr Abig -# @check fidr Bbig -# @check fidr Sbig -# @check fidr bitsparamfloat -# @check fidr bitsparambool -# @check fidr bitsparamsymbol -# @check fidr bitsparamint -# @check fidr bitsparamuint -# @check fidr tuple_of_tuples + @check fidr padding_test + @check fidr empty_arr_1 + @check fidr empty_arr_2 + @check fidr empty_arr_3 + @check fidr empty_arr_4 + @check fidr bigdata + @check fidr bigfloats + @check fidr bigints + @check fidr none + @check fidr nonearr + @check fidr scalar_nothing + @check fidr vector_nothing + @check fidr Abig + @check fidr Bbig + @check fidr Sbig + @check fidr bitsparamfloat + @check fidr bitsparambool + @check fidr bitsparamsymbol + @check fidr bitsparamint + @check fidr bitsparamuint + @check fidr tuple_of_tuples -# x1 = read(fidr, "group1/x") -# @assert x1 == Any[1] -# x2 = read(fidr, "group2/x") -# @assert x2 == Any[2] + x1 = read(fidr, "group1/x") + @assert x1 == Any[1] + x2 = read(fidr, "group2/x") + @assert x2 == Any[2] -# close(fidr) -# end -# end # compress in (true,false) + close(fidr) + end +end # compress in (true,false) for compress in (false,true) # object references in a write session diff --git a/test/plain.jl b/test/plain.jl index f64d6749c..fc9cc3a90 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -228,10 +228,10 @@ g = fid["mygroup"] close(g) close(fid) -# d = h5read(joinpath(test_path, "compound.h5"), "/data") -# @assert typeof(d) == HDF5.HDF5Compound -# @assert typeof(d.data) == Array{UInt8,1} -# @assert length(d.data) == 128 -# @assert d.membertype == Type[Float64, HDF5.FixedArray{Float64,(@compat Tuple{HDF5.DimSize{3}})}, HDF5.FixedArray{Float64,(@compat Tuple{HDF5.DimSize{3}})}, Float64] -# @assert d.membername == ASCIIString["wgt", "xyz", "uvw", "E"] -# @assert d.memberoffset == UInt64[0x00, 0x08, 0x20, 0x38] +d = h5read(joinpath(test_path, "compound.h5"), "/data") +@assert typeof(d) == HDF5.HDF5Compound +@assert typeof(d.data) == Array{UInt8,1} +@assert length(d.data) == 128 +@assert d.membertype == Type[Float64, HDF5.FixedArray{Float64,(@compat Tuple{HDF5.DimSize{3}})}, HDF5.FixedArray{Float64,(@compat Tuple{HDF5.DimSize{3}})}, Float64] +@assert d.membername == ASCIIString["wgt", "xyz", "uvw", "E"] +@assert d.memberoffset == UInt64[0x00, 0x08, 0x20, 0x38] From 76a0ce018a75a1a1bac39367534e3e8196b654c0 Mon Sep 17 00:00:00 2001 From: Rene Donner Date: Thu, 23 Apr 2015 17:54:26 +0200 Subject: [PATCH 04/12] use fieldnames instead of .names --- src/JLD00.jl | 12 ++++++------ src/jld_types.jl | 10 +++++----- src/plain.jl | 6 ------ test/jld.jl | 6 +++--- 4 files changed, 14 insertions(+), 20 deletions(-) diff --git a/src/JLD00.jl b/src/JLD00.jl index bdfbbe377..8bf0db61d 100644 --- a/src/JLD00.jl +++ b/src/JLD00.jl @@ -436,7 +436,7 @@ end # CompositeKind function read(obj::JldDataset, T::DataType) - if isempty(getnames(T)) && T.size > 0 + if isempty(fieldnames(T)) && T.size > 0 return read_bitstype(obj, T) end local x @@ -453,12 +453,12 @@ function read(obj::JldDataset, T::DataType) if length(v) == 0 x = ccall(:jl_new_struct, Any, (Any,Any...), T) else - n = getnames(T) + n = fieldnames(T) if length(v) != length(n) error("Wrong number of fields") end if !T.mutable - x = ccall(:jl_new_structv, Any, (Any,Ptr{Void},UInt32), T, v, length(getnames(T))) + x = ccall(:jl_new_structv, Any, (Any,Ptr{Void},UInt32), T, v, length(fieldnames(T))) else x = ccall(:jl_new_struct_uninit, Any, (Any,), T) for i = 1:length(v) @@ -720,7 +720,7 @@ write(parent::Union(JldFile, JldGroup), name::ByteString, s; rootmodule="") = wr function write_composite(parent::Union(JldFile, JldGroup), name::ByteString, s; rootmodule="") T = typeof(s) - if isempty(getnames(T)) + if isempty(fieldnames(T)) if T.size > 0 return write_bitstype(parent, name, s) end @@ -730,7 +730,7 @@ function write_composite(parent::Union(JldFile, JldGroup), name::ByteString, s; return end Tname = string(T.name.name) - n = getnames(T) + n = fieldnames(T) local gtypes if !exists(file(parent), pathtypes) gtypes = g_create(file(parent), pathtypes) @@ -803,7 +803,7 @@ function has_pointer_field(obj::Tuple, name) end function has_pointer_field(obj, name) - names = getnames(typeof(obj)) + names = fieldnames(typeof(obj)) for fieldname in names if isdefined(obj, fieldname) x = getfield(obj, fieldname) diff --git a/src/jld_types.jl b/src/jld_types.jl index bbaadd2c3..cec6c5f4d 100644 --- a/src/jld_types.jl +++ b/src/jld_types.jl @@ -382,7 +382,7 @@ function h5type(parent::JldFile, T::ANY, commit::Bool) id = HDF5.h5t_create(HDF5.H5T_COMPOUND, typeinfo.size) for i = 1:length(typeinfo.offsets) fielddtype = typeinfo.dtypes[i] - HDF5.h5t_insert(id, mangle_name(fielddtype, HDF5.getnames(T)[i]), typeinfo.offsets[i], fielddtype) + HDF5.h5t_insert(id, mangle_name(fielddtype, fieldnames(T)[i]), typeinfo.offsets[i], fielddtype) end end @@ -410,11 +410,11 @@ function _gen_jlconvert_type(typeinfo::JldTypeInfo, T::ANY) push!(args, quote ref = unsafe_load(convert(Ptr{HDF5ReferenceObj}, ptr)+$h5offset) if ref != HDF5.HDF5ReferenceObj_NULL - out.$(HDF5.getnames(T)[i]) = convert($(T.types[i]), read_ref(file, ref)) + out.$(fieldnames(T)[i]) = convert($(T.types[i]), read_ref(file, ref)) end end) else - push!(args, :(out.$(HDF5.getnames(T)[i]) = jlconvert($(T.types[i]), file, ptr+$h5offset))) + push!(args, :(out.$(fieldnames(T)[i]) = jlconvert($(T.types[i]), file, ptr+$h5offset))) end end @eval function jlconvert(::Type{$T}, file::JldFile, ptr::Ptr) @@ -491,7 +491,7 @@ const DONT_STORE_SINGLETON_IMMUTABLES = VERSION >= v"0.4.0-dev+385" function gen_jlconvert(typeinfo::JldTypeInfo, T::ANY) haskey(JLCONVERT_DEFINED, T) && return - if isempty(HDF5.getnames(T)) + if isempty(fieldnames(T)) if T.size == 0 @eval begin jlconvert(::Type{$T}, ::JldFile, ::Ptr) = $T() @@ -524,7 +524,7 @@ end # Whether this datatype should be stored as opaque isopaque(t::@compat Tuple{Vararg{Type}}) = isa(t, ()) -isopaque(t::DataType) = isempty(HDF5.getnames(t)) +isopaque(t::DataType) = isempty(fieldnames(t)) # The size of this datatype in the HDF5 file (if opaque) opaquesize(t::@compat Tuple{Vararg{DataType}}) = 1 diff --git a/src/plain.jl b/src/plain.jl index 7e8ad9fb6..e9d7fba35 100644 --- a/src/plain.jl +++ b/src/plain.jl @@ -14,12 +14,6 @@ import Base: close, convert, done, dump, eltype, endof, flush, getindex, include("datafile.jl") -if VERSION < v"0.4.0-dev+4319" - getnames(T::DataType) = T.names -else - getnames(T::DataType) = T.name.names -end - ## C types typealias C_time_t Int diff --git a/test/jld.jl b/test/jld.jl index 3f7758934..7e01712da 100644 --- a/test/jld.jl +++ b/test/jld.jl @@ -734,13 +734,13 @@ jldopen(fn, "r") do file for i = 1:5 @test x[i].x.x == i end - @test HDF5.getnames(typeof(read(file, "x6"))) == (@compat Tuple{}) + @test isempty(fieldnames(typeof(read(file, "x6")))) @test reinterpret(UInt8, read(file, "x7")) == 0x77 x = read(file, "x8") @test x.a.x == 2 @test x.b.x.x == 3 - @test HDF5.getnames(typeof(x.c)) == (@compat Tuple{}) + @test isempty(fieldnames(typeof(x.c))) @test reinterpret(UInt8, x.d) == 0x12 x = read(file, "x9") @@ -753,7 +753,7 @@ jldopen(fn, "r") do file for i = 1:5 @test x[2][2][i].x.x == i end - @test HDF5.getnames(typeof(x[3])) == (@compat Tuple{}) + @test isempty(fieldnames(typeof(x[3]))) end # Issue #176 From 91d2c419239a41cbd2a9447be2e43a0ce6fefbf3 Mon Sep 17 00:00:00 2001 From: Rene Donner Date: Thu, 23 Apr 2015 18:14:28 +0200 Subject: [PATCH 05/12] correct FixedArray params --- src/plain.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/plain.jl b/src/plain.jl index e9d7fba35..5811ceb55 100644 --- a/src/plain.jl +++ b/src/plain.jl @@ -2134,7 +2134,11 @@ function hdf5array(objtype) eltyp = HDF5Datatype(h5t_get_super(objtype.id)) T = hdf5_to_julia_eltype(eltyp) dimsizes = ntuple(nd, i->DimSize{@compat Int(dims[nd-i+1])}) # reverse order - FixedArray{T, dimsizes} + if VERSION < v"0.4.0-dev+4319" + FixedArray{T, dimsizes} + else + FixedArray{T, Tuple{dimsizes...}} + end end ### Property manipulation ### From b183249da125ea30b4101118df68add41eae9c07 Mon Sep 17 00:00:00 2001 From: Simon Kornblith Date: Fri, 24 Apr 2015 18:31:41 -0400 Subject: [PATCH 06/12] Remove DimSize wrapper It seems that tuples are now allowed as type parameters (and have been for some time; 0.3 works) so this can be simplified. This also fixes a test failure in readremote.jl on 0.4 --- src/plain.jl | 14 ++++---------- test/plain.jl | 4 ++-- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/plain.jl b/src/plain.jl index 5811ceb55..3c7e6179c 100644 --- a/src/plain.jl +++ b/src/plain.jl @@ -431,10 +431,8 @@ end type EmptyArray{T}; end # Stub types to encode fixed-size arrays for H5T_ARRAY -immutable DimSize{N}; end # Int-wrapper (can't use tuple of Int as param) -immutable FixedArray{T,D<:(@compat Tuple{Vararg{DimSize}})}; end -dimsize{N}(::Type{DimSize{N}}) = N -size{T,D}(::Type{FixedArray{T,D}}) = map(dimsize, D)::(@compat Tuple{Vararg{Int}}) +immutable FixedArray{T,D}; end +size{T,D}(::Type{FixedArray{T,D}}) = D eltype{T,D}(::Type{FixedArray{T,D}}) = T # VLEN objects @@ -2133,12 +2131,8 @@ function hdf5array(objtype) h5t_get_array_dims(objtype.id, dims) eltyp = HDF5Datatype(h5t_get_super(objtype.id)) T = hdf5_to_julia_eltype(eltyp) - dimsizes = ntuple(nd, i->DimSize{@compat Int(dims[nd-i+1])}) # reverse order - if VERSION < v"0.4.0-dev+4319" - FixedArray{T, dimsizes} - else - FixedArray{T, Tuple{dimsizes...}} - end + dimsizes = ntuple(nd, i->@compat Int(dims[nd-i+1])) # reverse order + FixedArray{T, dimsizes} end ### Property manipulation ### diff --git a/test/plain.jl b/test/plain.jl index fc9cc3a90..0cee20f04 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -1,4 +1,4 @@ -using HDF5, Compat +using HDF5, Compat, Base.Test const test_path = splitdir(@__FILE__)[1] # Create a new file @@ -232,6 +232,6 @@ d = h5read(joinpath(test_path, "compound.h5"), "/data") @assert typeof(d) == HDF5.HDF5Compound @assert typeof(d.data) == Array{UInt8,1} @assert length(d.data) == 128 -@assert d.membertype == Type[Float64, HDF5.FixedArray{Float64,(@compat Tuple{HDF5.DimSize{3}})}, HDF5.FixedArray{Float64,(@compat Tuple{HDF5.DimSize{3}})}, Float64] +@test d.membertype == Type[Float64, HDF5.FixedArray{Float64,(3,)}, HDF5.FixedArray{Float64,(3,)}, Float64] @assert d.membername == ASCIIString["wgt", "xyz", "uvw", "E"] @assert d.memberoffset == UInt64[0x00, 0x08, 0x20, 0x38] From b92cf62c231a26894ea5ac9eb304d1f4c6f39ffd Mon Sep 17 00:00:00 2001 From: Simon Kornblith Date: Fri, 24 Apr 2015 20:43:49 -0400 Subject: [PATCH 07/12] Remove workarounds for #176 (since it's fixed) --- test/jld.jl | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/test/jld.jl b/test/jld.jl index 7e01712da..d1fb0a566 100644 --- a/test/jld.jl +++ b/test/jld.jl @@ -295,8 +295,7 @@ rm(fn) for compress in (true,false) - fnc = compress ? fn*".c" : fn # workaround #176 - fid = jldopen(fnc, "w", compress=compress) + fid = jldopen(fn, "w", compress=compress) @write fid x @write fid A @write fid Aarray @@ -322,9 +321,7 @@ for compress in (true,false) @write fid sym @write fid syms @write fid d - if compress # work around #176 - @write fid ex - end + @write fid ex @write fid T @write fid char @write fid unicode_char @@ -401,7 +398,7 @@ for compress in (true,false) # mmapping currently fails on Windows; re-enable once it can work for mmap = (@windows ? false : (false, true)) - fidr = jldopen(fnc, "r", mmaparrays=mmap) + fidr = jldopen(fn, "r", mmaparrays=mmap) @check fidr x @check fidr A dsetA = fidr["A"] @@ -433,10 +430,8 @@ for compress in (true,false) @check fidr sym @check fidr syms @check fidr d - if compress # work around #176 - exr = read(fidr, "ex") # line numbers are stripped, don't expect equality - checkexpr(ex, exr) - end + exr = read(fidr, "ex") # line numbers are stripped, don't expect equality + checkexpr(ex, exr) @check fidr T @check fidr char @check fidr unicode_char From 5c2652ba413796bcecb9ec12deffc4387d5a77f9 Mon Sep 17 00:00:00 2001 From: Simon Kornblith Date: Fri, 24 Apr 2015 16:17:58 -0400 Subject: [PATCH 08/12] Add missing compat to JLD00 --- src/JLD00.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/JLD00.jl b/src/JLD00.jl index 8bf0db61d..1761af8af 100644 --- a/src/JLD00.jl +++ b/src/JLD00.jl @@ -226,7 +226,7 @@ function delete!(parent::Union(JldFile, JldGroup), path::ByteString) exists(parent, path) || error("$path does not exist in $parent") delete!(parent[path]) end -delete!(parent::Union(JldFile, JldGroup), args::Tuple{Vararg{ByteString}}) = for a in args delete!(parent,a) end +delete!(parent::Union(JldFile, JldGroup), args::@compat Tuple{Vararg{ByteString}}) = for a in args delete!(parent,a) end ismmappable(obj::JldDataset) = ismmappable(obj.plain) readmmap(obj::JldDataset, args...) = readmmap(obj.plain, args...) setindex!(parent::Union(JldFile, JldGroup), val, path::ASCIIString) = write(parent, path, val) @@ -583,9 +583,9 @@ write(parent::Union(JldFile, JldGroup), name::ByteString, n::Nothing) = write(pa # Types # the first is needed to avoid an ambiguity warning if isdefined(Core, :Top) - write{T<:Top}(parent::Union(JldFile, JldGroup), name::ByteString, t::Tuple{Vararg{Type{T}}}) = write(parent, name, Any[t...], "Tuple") + write{T<:Top}(parent::Union(JldFile, JldGroup), name::ByteString, t::@compat Tuple{Vararg{Type{T}}}) = write(parent, name, Any[t...], "Tuple") else - write{T}(parent::Union(JldFile, JldGroup), name::ByteString, t::Tuple{Vararg{Type{T}}}) = write(parent, name, Any[t...], "Tuple") + write{T}(parent::Union(JldFile, JldGroup), name::ByteString, t::@compat Tuple{Vararg{Type{T}}}) = write(parent, name, Any[t...], "Tuple") end write{T}(parent::Union(JldFile, JldGroup), name::ByteString, t::Type{T}) = write(parent, name, nothing, string("Type{", full_typename(t), "}")) @@ -1117,7 +1117,7 @@ function load(filename::AbstractString, varname::AbstractString) end end load(filename::AbstractString, varnames::AbstractString...) = load(filename, varnames) -function load(filename::AbstractString, varnames::Tuple{Vararg{AbstractString}}) +function load(filename::AbstractString, varnames::@compat Tuple{Vararg{AbstractString}}) jldopen(filename, "r") do file map((var)->read(file, var), varnames) end From ac464ac78b117dd56b5c070b57be48b8a86dc44f Mon Sep 17 00:00:00 2001 From: Simon Kornblith Date: Fri, 24 Apr 2015 14:52:02 -0400 Subject: [PATCH 09/12] Update JLD for JuliaLang/julia#10380 --- src/JLD.jl | 28 +++++++++++++--- src/jld_types.jl | 86 ++++++++++++++++++++++++++++++++---------------- test/jld.jl | 12 +++++-- 3 files changed, 90 insertions(+), 36 deletions(-) diff --git a/src/JLD.jl b/src/JLD.jl index f104fa50c..1ecf3919c 100644 --- a/src/JLD.jl +++ b/src/JLD.jl @@ -111,6 +111,11 @@ immutable AssociativeWrapper{K,V,T<:Associative} values::Vector{V} end +# Wrapper for SimpleVector +immutable SimpleVectorWrapper + elements::Vector +end + include("jld_types.jl") file(x::JldFile) = x @@ -376,6 +381,11 @@ function after_read{K,V,T}(x::AssociativeWrapper{K,V,T}) ret end +# Special case for SimpleVector +if VERSION >= v"0.4.0-dev+4319" + after_read(x::SimpleVectorWrapper) = Base.svec(x.elements...) +end + ## Arrays # Read an array @@ -488,7 +498,7 @@ write(parent::Union(JldFile, JldGroup), name::ByteString, close(_write(parent, name, data, wsession; kargs...)) # Pick whether to use compact or default storage based on data size -function dset_create_properties(parent, sz::Int, obj, chunk=Int[]; mmap = false) +function dset_create_properties(parent, sz::Int, obj, chunk=Int[]; mmap::Bool=false) if sz <= 8192 && !ismmapped(parent) && !mmap return compact_properties(), false end @@ -634,10 +644,9 @@ write_ref(parent::JldGroup, data, wsession::JldWriteSession) = write_ref(file(parent), data, wsession) # Special case for associative, to rehash keys -function _write(parent::Union(JldFile, JldGroup), name::ByteString, - d::Associative, wsession::JldWriteSession; kargs...) +function _write{K,V}(parent::Union(JldFile, JldGroup), name::ByteString, + d::Associative{K,V}, wsession::JldWriteSession; kargs...) n = length(d) - K, V = eltype(d) ks = Array(K, n) vs = Array(V, n) i = 0 @@ -645,7 +654,14 @@ function _write(parent::Union(JldFile, JldGroup), name::ByteString, ks[i+=1] = k vs[i] = v end - write_compound(parent, name, AssociativeWrapper{K,V,typeof(d)}(ks, vs), wsession) + write_compound(parent, name, AssociativeWrapper{K,V,typeof(d)}(ks, vs), wsession; kargs...) +end + +# Special case for SimpleVector +if VERSION >= v"0.4.0-dev+4319" + _write(parent::Union(JldFile, JldGroup), name::ByteString, + d::SimpleVector, wsession::JldWriteSession; kargs...) = + write_compound(parent, name, SimpleVectorWrapper([d...]), wsession; kargs...) end # Expressions, drop line numbers @@ -859,6 +875,8 @@ function full_typename(io::IO, file::JldFile, jltype::DataType) full_typename(io, file, jltype.parameters[i]) end print(io, '}') + elseif jltype <: Tuple + print(io, "{}") end end function full_typename(file::JldFile, x) diff --git a/src/jld_types.jl b/src/jld_types.jl index cec6c5f4d..1bf31a5aa 100644 --- a/src/jld_types.jl +++ b/src/jld_types.jl @@ -10,6 +10,20 @@ const BUILTIN_TYPES = Set([Symbol, Type, UTF16String, BigFloat, BigInt]) const H5CONVERT_DEFINED = ObjectIdDict() const JLCONVERT_DEFINED = ObjectIdDict() +if VERSION >= v"0.4.0-dev+4319" + const EMPTY_TUPLE_TYPE = Tuple{} + typealias TypesType SimpleVector + typealias TupleType{T<:Tuple} Type{T} + tupletypes(T::TupleType) = T.parameters + typetuple(types) = Tuple{types...} +else + const EMPTY_TUPLE_TYPE = () + typealias TypesType (Type...) + typealias TupleType (Type...) + tupletypes(T::TupleType) = T + typetuple(types) = tuple(types...) +end + ## Helper functions # Holds information about the mapping between a Julia and HDF5 type @@ -20,7 +34,7 @@ immutable JldTypeInfo end # Get information about the HDF5 types corresponding to Julia types -function JldTypeInfo(parent::JldFile, types::(@compat Tuple{Vararg{Type}}), commit::Bool) +function JldTypeInfo(parent::JldFile, types::TypesType, commit::Bool) dtypes = Array(JldDatatype, length(types)) offsets = Array(Int, length(types)) offset = 0 @@ -257,7 +271,7 @@ function h5type{T<:Type}(parent::JldFile, ::Type{T}, commit::Bool) id = HDF5.h5t_create(HDF5.H5T_COMPOUND, 8) HDF5.h5t_insert(id, "typename_", 0, h5fieldtype(parent, UTF8String, commit)) dtype = HDF5Datatype(id, parent.plain) - commit ? commit_datatype(parent, dtype, Type) : JldDatatype(dtype, -1) + out = commit ? commit_datatype(parent, dtype, Type) : JldDatatype(dtype, -1) end gen_h5convert{T<:Type}(::JldFile, ::Type{T}) = nothing @@ -292,16 +306,13 @@ h5fieldtype{T,N}(parent::JldFile, ::Type{Array{T,N}}, ::Bool) = JLD_REF_TYPE ## Tuples if INLINE_TUPLE - h5fieldtype(parent::JldFile, T::(@compat Tuple{Vararg{Type}}), commit::Bool) = + h5fieldtype(parent::JldFile, T::TupleType, commit::Bool) = isleaftype(T) ? h5type(parent, T, commit) : JLD_REF_TYPE else - h5fieldtype(parent::JldFile, T::(@compat Tuple{Vararg{Type}}), ::Bool) = JLD_REF_TYPE + h5fieldtype(parent::JldFile, T::TupleType, ::Bool) = JLD_REF_TYPE end -function h5type(parent::JldFile, T::(@compat Tuple{Vararg{Type}}), commit::Bool) - !isa(T, (@compat Tuple{Vararg{Union((@compat Tuple), DataType)}})) && unknown_type_err(T) - T = T::(@compat Tuple{Vararg{Union((@compat Tuple), DataType)}}) - +function h5type(parent::JldFile, T::TupleType, commit::Bool) haskey(parent.jlh5type, T) && return parent.jlh5type[T] # Tuples should always be concretely typed, unless we're # reconstructing a tuple, in which case commit will be false @@ -321,7 +332,7 @@ function h5type(parent::JldFile, T::(@compat Tuple{Vararg{Type}}), commit::Bool) dtype = HDF5Datatype(id, parent.plain) if commit jlddtype = commit_datatype(parent, dtype, T) - if isempty(T) + if T == EMPTY_TUPLE_TYPE # to allow recovery of empty tuples, which HDF5 does not allow a_write(dtype, "empty", @compat UInt8(1)) end @@ -331,13 +342,14 @@ function h5type(parent::JldFile, T::(@compat Tuple{Vararg{Type}}), commit::Bool) end end -function gen_jlconvert(typeinfo::JldTypeInfo, T::(@compat Tuple{Vararg{Type}})) +function gen_jlconvert(typeinfo::JldTypeInfo, T::TupleType) haskey(JLCONVERT_DEFINED, T) && return ex = Expr(:block) args = ex.args tup = Expr(:tuple) tupargs = tup.args + types = tupletypes(T) for i = 1:length(typeinfo.dtypes) h5offset = typeinfo.offsets[i] field = symbol(string("field", i)) @@ -345,7 +357,7 @@ function gen_jlconvert(typeinfo::JldTypeInfo, T::(@compat Tuple{Vararg{Type}})) if HDF5.h5t_get_class(typeinfo.dtypes[i]) == HDF5.H5T_REFERENCE push!(args, :($field = read_ref(file, unsafe_load(convert(Ptr{HDF5ReferenceObj}, ptr)+$h5offset)))) else - push!(args, :($field = jlconvert($(T[i]), file, ptr+$h5offset))) + push!(args, :($field = jlconvert($(types[i]), file, ptr+$h5offset))) end push!(tupargs, field) end @@ -434,7 +446,20 @@ function _gen_jlconvert_immutable(typeinfo::JldTypeInfo, T::ANY) h5offset = typeinfo.offsets[i] jloffset = jloffsets[i] - if HDF5.h5t_get_class(typeinfo.dtypes[i]) == HDF5.H5T_REFERENCE + if isa(T.types[i], TupleType) && VERSION >= v"0.4.0-dev+4319" && T.types[i].pointerfree + # We continue to store tuples as references for the sake of + # backwards compatibility, but on 0.4 they are now stored + # inline + push!(args, quote + ref = unsafe_load(convert(Ptr{HDF5ReferenceObj}, ptr)+$h5offset) + if ref == HDF5.HDF5ReferenceObj_NULL + warn("""A pointerfree tuple field was undefined. + This is not supported in Julia 0.4 and the corresponding tuple will be uninitialized.""") + else + unsafe_store!(convert(Ptr{$(T.types[i])}, out)+$jloffset, read_ref(file, ref)) + end + end) + elseif HDF5.h5t_get_class(typeinfo.dtypes[i]) == HDF5.H5T_REFERENCE obj = gensym("obj") push!(args, quote ref = unsafe_load(convert(Ptr{HDF5ReferenceObj}, ptr)+$h5offset) @@ -523,11 +548,12 @@ end ## Common functions for all non-special types (including gen_h5convert) # Whether this datatype should be stored as opaque -isopaque(t::@compat Tuple{Vararg{Type}}) = isa(t, ()) -isopaque(t::DataType) = isempty(fieldnames(t)) +isopaque(t::TupleType) = t == EMPTY_TUPLE_TYPE +# isopaque(t::DataType) = isempty(fieldnames(t)) +isopaque(t::DataType) = isa(t, TupleType) ? t == EMPTY_TUPLE_TYPE : isempty(fieldnames(t)) # The size of this datatype in the HDF5 file (if opaque) -opaquesize(t::@compat Tuple{Vararg{DataType}}) = 1 +opaquesize(t::TupleType) = 1 opaquesize(t::DataType) = max(1, t.size) # Whether a type that is stored inline in HDF5 should be stored as a @@ -535,7 +561,7 @@ opaquesize(t::DataType) = max(1, t.size) # true for some unions of special types defined above, unless either # INLINE_TUPLE or INLINE_POINTER_IMMUTABLE is true. uses_reference(T::DataType) = !T.pointerfree -uses_reference(::@compat Tuple) = true +uses_reference(::TupleType) = true uses_reference(::UnionType) = true unknown_type_err(T) = @@ -548,19 +574,21 @@ gen_h5convert(parent::JldFile, T) = # There is no point in specializing this function _gen_h5convert(parent::JldFile, T::ANY) dtype = parent.jlh5type[T].dtype - istuple = isa(T, @compat Tuple) + istuple = isa(T, TupleType) + + if isopaque(T) + if T.size == 0 + @eval h5convert!(out::Ptr, ::JldFile, x::$T, ::JldWriteSession) = nothing + else + @eval h5convert!(out::Ptr, ::JldFile, x::$T, ::JldWriteSession) = + unsafe_store!(convert(Ptr{$T}, out), x) + end + return + end + if istuple - types = T + types = tupletypes(T) else - if isopaque(T::DataType) - if (T::DataType).size == 0 - @eval h5convert!(out::Ptr, ::JldFile, x::$T, ::JldWriteSession) = nothing - else - @eval h5convert!(out::Ptr, ::JldFile, x::$T, ::JldWriteSession) = - unsafe_store!(convert(Ptr{$T}, out), x) - end - return - end types = (T::DataType).types end @@ -704,9 +732,9 @@ function reconstruct_type(parent::JldFile, dtype::HDF5Datatype, savedname::Abstr end end - if startswith(savedname, "(") + if startswith(savedname, "(") || startswith(savedname, "Core.Tuple{") # We're reconstructing a tuple - tuple(fieldtypes...) + typetuple(fieldtypes) else # We're reconstructing some other type @eval begin diff --git a/test/jld.jl b/test/jld.jl index d1fb0a566..c1611a002 100644 --- a/test/jld.jl +++ b/test/jld.jl @@ -205,13 +205,19 @@ bitsparamint16 = BitsParams{@compat Int16(1)}() # Tuple of tuples tuple_of_tuples = (1, 2, (3, 4, [5, 6]), [7, 8]) +# SimpleVector +if VERSION >= v"0.4.0-dev+4319" + simplevec = Base.svec(1, 2, Int64, "foo") + iseq(x::SimpleVector, y::SimpleVector) = collect(x) == collect(y) +end + iseq(x,y) = isequal(x,y) iseq(x::MyStruct, y::MyStruct) = (x.len == y.len && x.data == y.data) iseq(x::MyImmutable, y::MyImmutable) = (isequal(x.x, y.x) && isequal(x.y, y.y) && isequal(x.z, y.z)) iseq(x::Union(EmptyTI, EmptyTT), y::Union(EmptyTI, EmptyTT)) = isequal(x.x, y.x) iseq(c1::Array{Base.Sys.CPUinfo}, c2::Array{Base.Sys.CPUinfo}) = length(c1) == length(c2) && all([iseq(c1[i], c2[i]) for i = 1:length(c1)]) function iseq(c1::Base.Sys.CPUinfo, c2::Base.Sys.CPUinfo) - for n in Base.Sys.CPUinfo.names + for n in fieldnames(Base.Sys.CPUinfo) if getfield(c1, n) != getfield(c2, n) return false end @@ -387,6 +393,7 @@ for compress in (true,false) @write fid bitsparamint @write fid bitsparamuint @write fid tuple_of_tuples + VERSION >= v"0.4.0-dev+4319" && @write fid simplevec # Make sure we can create groups (i.e., use HDF5 features) g = g_create(fid, "mygroup") @@ -515,6 +522,7 @@ for compress in (true,false) @check fidr bitsparamint @check fidr bitsparamuint @check fidr tuple_of_tuples + VERSION >= v"0.4.0-dev+4319" && @check fidr simplevec x1 = read(fidr, "group1/x") @assert x1 == Any[1] @@ -739,7 +747,7 @@ jldopen(fn, "r") do file @test reinterpret(UInt8, x.d) == 0x12 x = read(file, "x9") - @test isa(x, (@compat Tuple)) + @test isa(x, Tuple) @test length(x) == 3 @test x[1].x == 1 @test isa(x[2], Tuple) From 6cf6f3bb55d9df35a38efbf2d85b1231254a677a Mon Sep 17 00:00:00 2001 From: Simon Kornblith Date: Fri, 24 Apr 2015 19:31:05 -0400 Subject: [PATCH 10/12] Improve type translation for cross-version compatibility Converts between Tuple{} and () types depending on Julia version. Also fixes #218 --- src/JLD.jl | 69 ++++++++++++++++++++++++++++++++++++++++++---------- src/JLD00.jl | 11 ++++++--- 2 files changed, 63 insertions(+), 17 deletions(-) diff --git a/src/JLD.jl b/src/JLD.jl index 1ecf3919c..3a40b91f9 100644 --- a/src/JLD.jl +++ b/src/JLD.jl @@ -9,15 +9,6 @@ import HDF5: close, dump, exists, file, getindex, setindex!, g_create, g_open, o HDF5ReferenceObj, HDF5BitsKind, ismmappable, readmmap import Base: length, endof, show, done, next, ndims, start, delete!, size, sizeof -# .jld files written before v"0.4.0-dev+1419" might have UInt32 instead of UInt32 as the typename string. -# See julia issue #8907 -if VERSION >= v"0.4.0-dev+1419" - julia_type(s::AbstractString) = _julia_type(replace(s, r"UInt(?=\d{1,3})", "UInt")) -else - julia_type(s::AbstractString) = _julia_type(s) -end - - const magic_base = "Julia data file (HDF5), version " const version_current = v"0.1" const pathrefs = "/_refs" @@ -773,14 +764,66 @@ is_valid_type_ex{T}(::T) = isbits(T) is_valid_type_ex(e::Expr) = ((e.head == :curly || e.head == :tuple || e.head == :.) && all(map(is_valid_type_ex, e.args))) || (e.head == :call && (e.args[1] == :Union || e.args[1] == :TypeVar)) -# Work around https://github.com/JuliaLang/julia/issues/8226 +if VERSION >= v"0.4.0-dev+1419" + const typemap_Core = @compat Dict( + :Uint8 => :UInt8, + :Uint16 => :Uint16, + :Uint32 => :UInt32, + :Uint64 => :UInt64, + :Nothing => :Void + ) +else + const typemap_Core = @compat Dict( + :UInt8 => :Uint8, + :UInt16 => :Uint16, + :UInt32 => :Uint32, + :UInt64 => :Uint64, + :Void => :Nothing + ) +end + const _typedict = Dict{UTF8String,Type}() -_typedict["Core.Type{TypeVar(:T,Union(Core.Any,Core.Undef))}"] = Type -function _julia_type(s::AbstractString) +fixtypes(typ) = typ +@eval begin + function fixtypes(typ::Expr) + if typ.head == :. + if length(typ.args) == 2 && typ.args[1] == :Core + arg = typ.args[2].value + return Expr(:., :Core, QuoteNode(get(typemap_Core, arg, arg))) + else + return typ + end + elseif typ == :(Core.Type{TypeVar(:T,Union(Core.Any,Core.Undef))}) || typ == :(Core.Type{TypeVar(:T)}) + # Work around https://github.com/JuliaLang/julia/issues/8226 and the removal of Top + return :(Core.Type) + end + + for i = 1:length(typ.args) + typ.args[i] = fixtypes(typ.args[i]) + end + + $(if VERSION >= v"0.4.0-dev+4319" + quote + if typ.head == :tuple + return Expr(:curly, :Tuple, typ.args...) + end + end + else + quote + if typ.head == :curly && !isempty(typ.args) && typ.args[1] == :(Core.Tuple) + return Expr(:tuple, typ.args[2:end]...) + end + end + end) + typ + end +end + +function julia_type(s::AbstractString) typ = get(_typedict, s, UnconvertedType) if typ == UnconvertedType - typ = julia_type(parse(s)) + typ = julia_type(fixtypes(parse(s))) if typ != UnsupportedType _typedict[s] = typ end diff --git a/src/JLD00.jl b/src/JLD00.jl index 1761af8af..77cbdfe44 100644 --- a/src/JLD00.jl +++ b/src/JLD00.jl @@ -8,6 +8,7 @@ using HDF5, Compat import HDF5: close, dump, exists, file, getindex, setindex!, g_create, g_open, o_delete, name, names, read, size, write, HDF5ReferenceObj, HDF5BitsKind, ismmappable, readmmap import Base: length, endof, show, done, next, start, delete! +import JLD if !isdefined(:setfield!) const setfield! = setfield @@ -367,7 +368,7 @@ end # Nothing read(obj::JldDataset, ::Type{Nothing}) = nothing -read(obj::JldDataset, ::Type{Bool}) = bool(read(obj, UInt8)) +read(obj::JldDataset, ::Type{Bool}) = read(obj, UInt8) != 0 # Types read{T}(obj::JldDataset, ::Type{Type{T}}) = T @@ -397,7 +398,7 @@ read(obj::JldDataset, ::Type{Symbol}) = symbol(read(obj.plain, ByteString)) read{N}(obj::JldDataset, ::Type{Array{Symbol,N}}) = map(symbol, read(obj.plain, Array{ByteString})) # Char -read(obj::JldDataset, ::Type{Char}) = char(read(obj.plain, UInt32)) +read(obj::JldDataset, ::Type{Char}) = @compat Char(read(obj.plain, UInt32)) # UTF16String (not defined in julia 0.2) if VERSION >= v"0.3-" @@ -937,6 +938,7 @@ function _julia_type(s::AbstractString) typ = get(_typedict, s, UnconvertedType) if typ == UnconvertedType e = parse(s) + e = JLD.fixtypes(e) typ = UnsupportedType if is_valid_type_ex(e) try # try needed to catch undefined symbols @@ -975,8 +977,9 @@ function full_typename(tv::TypeVar) "TypeVar(:$(tv.name),$(full_typename(tv.lb)),$(full_typename(tv.ub)))" end end -full_typename(jltype::(Type...)) = length(jltype) == 1 ? @sprintf("(%s,)", full_typename(jltype[1])) : - @sprintf("(%s)", join(map(full_typename, jltype), ",")) +full_typename(jltype::@compat Tuple{Vararg{Type}}) = + length(jltype) == 1 ? @sprintf("(%s,)", full_typename(jltype[1])) : + @sprintf("(%s)", join(map(full_typename, jltype), ",")) full_typename(x) = string(x) function full_typename(jltype::DataType) #tname = "$(jltype.name.module).$(jltype.name)" From 957605e9421814ed31b57be1a9f4aa8e4d636b77 Mon Sep 17 00:00:00 2001 From: Simon Kornblith Date: Fri, 24 Apr 2015 19:32:48 -0400 Subject: [PATCH 11/12] Temporarily disable type parameters that are themselves tuples On 0.4, we can't differentiate between these and tuples of types that were saved by earlier versions of Julia. In earlier versions of Julia, such tuples were not bits typed so they would not have been usable anyway. We can revisit this when we're ready to change the file format (perhaps as part of #204/#218) --- src/JLD.jl | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/JLD.jl b/src/JLD.jl index 3a40b91f9..6a95c1ade 100644 --- a/src/JLD.jl +++ b/src/JLD.jl @@ -870,13 +870,15 @@ function full_typename(io::IO, file::JldFile, tv::TypeVar) print(io, ')') end end -function full_typename(io::IO, file::JldFile, jltype::@compat Tuple{Vararg{Type}}) - print(io, '(') - for t in jltype - full_typename(io, file, t) - print(io, ',') +if VERSION < v"0.4.0-dev+4319" + function full_typename(io::IO, file::JldFile, jltype::@compat Tuple{Vararg{Type}}) + print(io, '(') + for t in jltype + full_typename(io, file, t) + print(io, ',') + end + print(io, ')') end - print(io, ')') end function full_typename(io::IO, ::JldFile, x) # Only allow bitstypes that show as AST literals and make sure that they @@ -885,7 +887,7 @@ function full_typename(io::IO, ::JldFile, x) # A different implementation will be required to support custom immutables # or things as simple as Int16(1). s = sprint(show, x) - if isbits(x) && parse(s) === x + if isbits(x) && parse(s) === x && !isa(x, Tuple) print(io, s) else error("type parameters with objects of type ", typeof(x), " are currently unsupported") From 12479c576c588fb2f8271e6af7f0129720159e93 Mon Sep 17 00:00:00 2001 From: Simon Kornblith Date: Sat, 25 Apr 2015 00:39:02 -0400 Subject: [PATCH 12/12] Run JLDArchives test after workspace() This is a bit of a hack, but necessary to avoid collision --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8e8b4a06a..f33839860 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,8 +9,8 @@ notifications: email: false script: - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi - - julia -e 'Pkg.clone(pwd()); Pkg.build("HDF5"); Pkg.add("JLDArchives")' - - if [ $TRAVIS_JULIA_VERSION = "nightly" ]; then julia --check-bounds=yes --inline=no --code-coverage=user -e 'include(Pkg.dir("HDF5", "test", "runtests.jl")); include(Pkg.dir("JLDArchives", "test", "runtests.jl"))'; fi + - julia -e 'Pkg.clone(pwd()); Pkg.build("HDF5"); Pkg.add("JLDArchives"); Pkg.checkout("JLDArchives")' + - if [ $TRAVIS_JULIA_VERSION = "nightly" ]; then julia --check-bounds=yes --inline=no --code-coverage=user -e 'reload("HDF5/test/runtests.jl"); workspace(); import LastMain.Compat, LastMain.HDF5; reload("HDF5/src/JLD.jl"); reload("JLDArchives/test/runtests.jl")'; fi - if [ $TRAVIS_JULIA_VERSION = "release" ]; then julia --check-bounds=yes -e 'Pkg.test("HDF5"); Pkg.test("JLDArchives")'; fi after_success: - if [ $TRAVIS_JULIA_VERSION = "nightly" ]; then julia -e 'cd(Pkg.dir("HDF5")); Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder())'; fi