From 3dde05ab45ee77f8e0de241d43406466b95a7ae0 Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Sat, 4 Jun 2022 14:53:17 -0700 Subject: [PATCH 1/5] Define AttibuteDict --- docs/src/api_bindings.md | 4 ++ gen/api_defs.jl | 2 + src/HDF5.jl | 2 +- src/api/functions.jl | 22 ++++++++ src/api/helpers.jl | 6 ++- src/attributes.jl | 105 +++++++++++++++++++++++++++++++++++---- src/show.jl | 14 +++++- test/attributes.jl | 37 ++++++++++++++ test/gc.jl | 2 +- test/plain.jl | 62 ++++++++++------------- test/properties.jl | 4 +- test/runtests.jl | 2 + test/swmr.jl | 2 +- 13 files changed, 211 insertions(+), 53 deletions(-) create mode 100644 test/attributes.jl diff --git a/docs/src/api_bindings.md b/docs/src/api_bindings.md index 600de918e..b4974bc07 100644 --- a/docs/src/api_bindings.md +++ b/docs/src/api_bindings.md @@ -60,7 +60,9 @@ h5_set_free_list_limits - [`h5a_get_type`](@ref h5a_get_type) - [`h5a_iterate`](@ref h5a_iterate) - [`h5a_open`](@ref h5a_open) +- [`h5a_open_by_idx`](@ref h5a_open_by_idx) - [`h5a_read`](@ref h5a_read) +- [`h5a_rename`](@ref h5a_rename) - [`h5a_write`](@ref h5a_write) ```@docs h5a_close @@ -78,7 +80,9 @@ h5a_get_space h5a_get_type h5a_iterate h5a_open +h5a_open_by_idx h5a_read +h5a_rename h5a_write ``` diff --git a/gen/api_defs.jl b/gen/api_defs.jl index f94c839f6..e2c3c9d07 100644 --- a/gen/api_defs.jl +++ b/gen/api_defs.jl @@ -49,7 +49,9 @@ @bind h5a_get_type(attr_id::hid_t)::hid_t "Error getting attribute type" @bind h5a_iterate2(obj_id::hid_t, idx_type::Cint, order::Cint, n::Ptr{hsize_t}, op::Ptr{Cvoid}, op_data::Any)::herr_t string("Error iterating attributes in object ", h5i_get_name(obj_id)) @bind h5a_open(obj_id::hid_t, pathname::Ptr{UInt8}, aapl_id::hid_t)::hid_t string("Error opening attribute ", h5i_get_name(obj_id), "/", pathname) +@bind h5a_open_by_idx(obj_id::hid_t, pathname::Ptr{UInt8}, idx_type::Cint, order::Cint, n::hsize_t, aapl_id::hid_t, lapl_id::hid_t)::hid_t string("Error opening attribute ", n, " of ", h5i_get_name(obj_id), "/", pathname) @bind h5a_read(attr_id::hid_t, mem_type_id::hid_t, buf::Ptr{Cvoid})::herr_t string("Error reading attribute ", h5a_get_name(attr_id)) +@bind h5a_rename(loc_id::hid_t, old_attr_name::Cstring, new_attr_name::Cstring)::herr_t string("Could not rename attribute") @bind h5a_write(attr_hid::hid_t, mem_type_id::hid_t, buf::Ptr{Cvoid})::herr_t "Error writing attribute data" ### diff --git a/src/HDF5.jl b/src/HDF5.jl index 566fd663b..04b00c205 100644 --- a/src/HDF5.jl +++ b/src/HDF5.jl @@ -12,7 +12,7 @@ import Mmap export @read, @write, h5open, h5read, h5write, h5rewrite, h5writeattr, h5readattr, -create_attribute, open_attribute, read_attribute, write_attribute, delete_attribute, attributes, +create_attribute, open_attribute, read_attribute, write_attribute, delete_attribute, attributes, attrs, create_dataset, open_dataset, read_dataset, write_dataset, create_group, open_group, copy_object, open_object, delete_object, move_link, diff --git a/src/api/functions.jl b/src/api/functions.jl index 9e2101489..6bfdd15fb 100644 --- a/src/api/functions.jl +++ b/src/api/functions.jl @@ -263,6 +263,17 @@ function h5a_open(obj_id, pathname, aapl_id) return var"#status#" end +""" + h5a_open_by_idx(obj_id::hid_t, pathname::Ptr{UInt8}, idx_type::Cint, order::Cint, n::hsize_t, aapl_id::hid_t, lapl_id::hid_t) -> hid_t + +See `libhdf5` documentation for [`H5Aopen_by_idx`](https://portal.hdfgroup.org/display/HDF5/H5A_OPEN_BY_IDX). +""" +function h5a_open_by_idx(obj_id, pathname, idx_type, order, n, aapl_id, lapl_id) + var"#status#" = ccall((:H5Aopen_by_idx, libhdf5), hid_t, (hid_t, Ptr{UInt8}, Cint, Cint, hsize_t, hid_t, hid_t), obj_id, pathname, idx_type, order, n, aapl_id, lapl_id) + var"#status#" < 0 && @h5error(string("Error opening attribute ", n, " of ", h5i_get_name(obj_id), "/", pathname)) + return var"#status#" +end + """ h5a_read(attr_id::hid_t, mem_type_id::hid_t, buf::Ptr{Cvoid}) @@ -274,6 +285,17 @@ function h5a_read(attr_id, mem_type_id, buf) return nothing end +""" + h5a_rename(loc_id::hid_t, old_attr_name::Cstring, new_attr_name::Cstring) + +See `libhdf5` documentation for [`H5Arename`](https://portal.hdfgroup.org/display/HDF5/H5A_RENAME). +""" +function h5a_rename(loc_id, old_attr_name, new_attr_name) + var"#status#" = ccall((:H5Arename, libhdf5), herr_t, (hid_t, Cstring, Cstring), loc_id, old_attr_name, new_attr_name) + var"#status#" < 0 && @h5error(string("Could not rename attribute")) + return nothing +end + """ h5a_write(attr_hid::hid_t, mem_type_id::hid_t, buf::Ptr{Cvoid}) diff --git a/src/api/helpers.jl b/src/api/helpers.jl index 61d2d4ee4..19ab230c6 100644 --- a/src/api/helpers.jl +++ b/src/api/helpers.jl @@ -53,7 +53,11 @@ end # `@cfunction` with `$f`. # This helper translates between the two preferred forms for each respective language. function h5a_iterate_helper(loc_id::hid_t, attr_name::Ptr{Cchar}, ainfo::Ptr{H5A_info_t}, @nospecialize(f::Any))::herr_t - return f(loc_id, attr_name, ainfo) + try + return herr_t(f(loc_id, attr_name, ainfo)) + catch e + return herr_t(-1) + end end """ diff --git a/src/attributes.jl b/src/attributes.jl index 20c16a0e0..2257ff3af 100644 --- a/src/attributes.jl +++ b/src/attributes.jl @@ -1,3 +1,5 @@ +# mid-level API + """ HDF5.Attribute @@ -77,7 +79,6 @@ Open the [`Attribute`](@ref) named `name` on the object `parent`. open_attribute(parent::Union{File,Object}, name::AbstractString, aapl::AttributeAccessProperties=AttributeAccessProperties()) = Attribute(API.h5a_open(checkvalid(parent), name, aapl), file(parent)) - """ create_attribute(parent::Union{File,Object}, name::AbstractString, dtype::Datatype, space::Dataspace) create_attribute(parent::Union{File,Object}, name::AbstractString, data) @@ -102,24 +103,25 @@ function create_attribute(parent::Union{File,Object}, name::AbstractString, dtyp return Attribute(attrid, file(parent)) end - +# generic method +write_attribute(attr::Attribute, memtype::Datatype, x) = API.h5a_write(attr, memtype, x) +# specific methods function write_attribute(attr::Attribute, memtype::Datatype, str::AbstractString) strbuf = Base.cconvert(Cstring, str) GC.@preserve strbuf begin buf = Base.unsafe_convert(Ptr{UInt8}, strbuf) - API.h5a_write(attr, memtype, buf) + write_attribute(attr, memtype, buf) end end function write_attribute(attr::Attribute, memtype::Datatype, x::T) where {T<:Union{ScalarType,Complex{<:ScalarType}}} tmp = Ref{T}(x) - API.h5a_write(attr, memtype, tmp) + write_attribute(attr, memtype, tmp) end function write_attribute(attr::Attribute, memtype::Datatype, strs::Array{<:AbstractString}) p = Ref{Cstring}(strs) - API.h5a_write(attr, memtype, p) + write_attribute(attr, memtype, p) end write_attribute(attr::Attribute, memtype::Datatype, ::EmptyArray) = nothing -write_attribute(attr::Attribute, memtype::Datatype, x) = API.h5a_write(attr, memtype, x) """ write_attribute(parent::Union{File,Object}, name::AbstractString, data) @@ -127,19 +129,26 @@ write_attribute(attr::Attribute, memtype::Datatype, x) = API.h5a_write(attr, mem Write `data` as an [`Attribute`](@ref) named `name` on the object `parent`. """ function write_attribute(parent::Union{File,Object}, name::AbstractString, data; pv...) - obj, dtype = create_attribute(parent, name, data; pv...) + attr, dtype = create_attribute(parent, name, data; pv...) try - write_attribute(obj, dtype, data) + write_attribute(attr, dtype, data) catch exc delete_attribute(parent, name) rethrow(exc) finally - close(obj) + close(attr) close(dtype) end nothing end +""" + rename_attribute(parent::Union{File,Object}, oldname::AbstractString, newname::AbstractString) + +Rename the [`Attribute`](@ref) of the object `parent` named `oldname` to `newname`. +""" +rename_attribute(parent::Union{File,Object}, oldname::AbstractString, newname::AbstractString) = + API.h5a_rename(checkvalid(parent), oldname, newname) """ delete_attribute(parent::Union{File,Object}, name::AbstractString) @@ -188,6 +197,84 @@ function h5readattr(filename, name::AbstractString) end +struct AttributeDict <: AbstractDict{String,Any} + parent::Object +end + +""" + attrs(object::Union{File,Group,Dataset,Datatype}) + +The attributes dictionary of `object`. Returns an `AttributeDict`, a `Dict`-like +object for accessing the attributes of `object`. + +```julia +attrs(object)["name"] = value # create/overwrite an attribute +attr = attrs(object)["name"] # read an attribute +delete!(attrs(object), "name") # delete an attribute +keys(attrs(object)) # list the attribute names +``` +""" +function attrs(parent::Object) + return AttributeDict(parent) +end +attrs(file::File) = attrs(open_group(file, ".")) + +Base.haskey(attrdict::AttributeDict, path::AbstractString) = API.h5a_exists(checkvalid(attrdict.parent), path) +Base.length(attrdict::AttributeDict) = Int(object_info(attrdict.parent).num_attrs) + +function Base.getindex(x::AttributeDict, name::AbstractString) + haskey(x, name) || throw(KeyError(name)) + read_attribute(x.parent, name) +end +function Base.get(x::AttributeDict, name::AbstractString, default) + haskey(x, name) || return default + read_attribute(x.parent, name) +end +function Base.setindex!(attrdict::AttributeDict, val, name::AbstractString) + if haskey(attrdict, name) + # in case of an error, we write first to a temporary, then rename + _name = tempname() + try + write_attribute(attrdict.parent, _name, val) + delete_attribute(attrdict.parent, name) + rename_attribute(attrdict.parent, _name, name) + finally + haskey(attrdict, _name) && delete_attribute(attrdict.parent, _name) + end + else + write_attribute(attrdict.parent, name, val) + end +end +Base.delete!(attrdict::AttributeDict, path::AbstractString) = delete_attribute(attrdict.parent, path) + +function Base.keys(attrdict::AttributeDict) + # faster than iteratively calling h5a_get_name_by_idx + checkvalid(attrdict.parent) + keyvec = sizehint!(String[], length(attrdict)) + API.h5a_iterate(attrdict.parent, IDX_TYPE[], ORDER[]) do _, attr_name, _ + push!(keyvec, unsafe_string(attr_name)) + return false + end + return keyvec +end + +function Base.iterate(attrdict::AttributeDict) + # constuct key vector, then iterate + # faster than calling h5a_open_by_idx + iterate(attrdict, (keys(attrdict), 1)) +end +function Base.iterate(attrdict::AttributeDict, (keyvec, n)) + iter = iterate(keyvec, n) + if isnothing(iter) + return iter + end + key, nn = iter + return (key => attrdict[key]), (keyvec, nn) +end + + + + struct Attributes parent::Union{File,Object} end diff --git a/src/show.jl b/src/show.jl index 204944cdb..42f4e813b 100644 --- a/src/show.jl +++ b/src/show.jl @@ -63,6 +63,16 @@ function Base.show(io::IO, attr::Attributes) print(io, "Attributes of ", attr.parent) end +Base.show(io::IO, attrdict::AttributeDict) = summary(io, attrdict) +function Base.summary(io::IO, attrdict::AttributeDict) + print(io, "AttributeDict of ", attrdict.parent) + if isvalid(attrdict.parent) + n = length(attrdict) + print(io, " with ", n, n == 1 ? " attribute" : " attributes") + end +end + + const ENDIAN_DICT = Dict( API.H5T_ORDER_LE => "little endian byte order", API.H5T_ORDER_BE => "big endian byte order", @@ -189,9 +199,9 @@ _tree_head(io::IO, obj) = print(io, _tree_icon(obj), " ", obj) _tree_head(io::IO, obj::Datatype) = print(io, _tree_icon(obj), " HDF5.Datatype: ", name(obj)) _tree_count(parent::Union{File,Group}, attributes::Bool) = - length(parent) + (attributes ? length(HDF5.attributes(parent)) : 0) + length(parent) + (attributes ? length(HDF5.attrs(parent)) : 0) _tree_count(parent::Dataset, attributes::Bool) = - attributes ? length(HDF5.attributes(parent)) : 0 + attributes ? length(HDF5.attrs(parent)) : 0 _tree_count(parent::Attributes, _::Bool) = length(parent) _tree_count(parent::Union{Attribute,Datatype}, _::Bool) = 0 diff --git a/test/attributes.jl b/test/attributes.jl new file mode 100644 index 000000000..9eb5b6d99 --- /dev/null +++ b/test/attributes.jl @@ -0,0 +1,37 @@ +using HDF5, Test + + +filename = tempname() +f = h5open(filename, "w") + +@test attrs(f) isa HDF5.AttributeDict + +attrs(f)["a"] = 1 +@test attrs(f)["a"] == 1 + +attrs(f)["b"] = [2,3] +@test attrs(f)["b"] == [2,3] +@test length(attrs(f)) == 2 +@test sort(keys(attrs(f))) == ["a", "b"] + +# overwrite: same type +attrs(f)["a"] = 4 +@test attrs(f)["a"] == 4 +@test length(attrs(f)) == 2 +@test sort(keys(attrs(f))) == ["a", "b"] + +# overwrite: different size +attrs(f)["b"] = [4,5,6] +@test attrs(f)["b"] == [4,5,6] +@test length(attrs(f)) == 2 +@test sort(keys(attrs(f))) == ["a", "b"] + +# overwrite: different type +attrs(f)["b"] = "a string" +@test attrs(f)["b"] == "a string" +@test length(attrs(f)) == 2 +@test sort(keys(attrs(f))) == ["a", "b"] + +delete!(attrs(f), "a") +@test length(attrs(f)) == 1 +@test sort(keys(attrs(f))) == ["b"] diff --git a/test/gc.jl b/test/gc.jl index c34eede9f..89e37bef7 100644 --- a/test/gc.jl +++ b/test/gc.jl @@ -58,7 +58,7 @@ for i = 1:10 d = file["d"] ds = dataspace(d) g = file["g"] - a = attributes(file)["a"] + a = open_attribute(file, "a") @gcvalid dt ds d g a close(file) end diff --git a/test/plain.jl b/test/plain.jl index 47e7d44ea..c63b0a134 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -100,22 +100,19 @@ empty_array_of_strings = [""] write(f, "empty_array_of_strings", empty_array_of_strings) # attributes species = [["N", "C"]; ["A", "B"]] -attributes(f)["species"] = species -@test read(attributes(f)["species"]) == species -@test attributes(f)["species"][] == species +attrs(f)["species"] = species +@test attrs(f)["species"] == species C∞ = 42 -attributes(f)["C∞"] = C∞ +attrs(f)["C∞"] = C∞ dset = f["salut"] @test !isempty(dset) label = "This is a string" -attributes(dset)["typeinfo"] = label -@test read(attributes(dset)["typeinfo"]) == label -@test attributes(dset)["typeinfo"][] == label -@test dset["typeinfo"][] == label +attrs(dset)["typeinfo"] = label +@test attrs(dset)["typeinfo"] == label close(dset) # Scalar reference values in attributes -attributes(f)["ref_test"] = HDF5.Reference(f, "empty_array_of_strings") -@test read(attributes(f)["ref_test"]) === HDF5.Reference(f, "empty_array_of_strings") +attrs(f)["ref_test"] = HDF5.Reference(f, "empty_array_of_strings") +@test attrs(f)["ref_test"] === HDF5.Reference(f, "empty_array_of_strings") # Group g = create_group(f, "mygroup") # Test dataset with compression @@ -320,10 +317,10 @@ A = rand(3, 3)' @test_throws ArgumentError write(hid, "A", A) @test !haskey(hid, "A") dset = create_dataset(hid, "attr", datatype(Int), dataspace(0)) -@test !haskey(attributes(dset), "attr") +@test !haskey(attrs(dset), "attr") # broken test - writing attributes does not check that the stride is correct @test_skip @test_throws ArgumentError write(dset, "attr", A) -@test !haskey(attributes(dset), "attr") +@test !haskey(attrs(dset), "attr") close(hid) # more do syntax @@ -507,7 +504,7 @@ end # testset plain f = h5open(fn, "w") f["ComplexF64"] = 1.0 + 2.0im - attributes(f["ComplexF64"])["ComplexInt64"] = 1im + attrs(f["ComplexF64"])["ComplexInt64"] = 1im Acmplx = rand(ComplexF64, 3, 5) write(f, "Acmplx64", convert(Matrix{ComplexF64}, Acmplx)) @@ -529,8 +526,8 @@ end # testset plain fr = h5open(fn) z = read(fr, "ComplexF64") @test z == 1.0 + 2.0im && isa(z, ComplexF64) - z_attrs = attributes(fr["ComplexF64"]) - @test read(z_attrs["ComplexInt64"]) == 1im + z_attrs = attrs(fr["ComplexF64"]) + @test z_attrs["ComplexInt64"] == 1im Acmplx32 = read(fr, "Acmplx32") @test convert(Matrix{ComplexF32}, Acmplx) == Acmplx32 @@ -782,8 +779,8 @@ dset = create_dataset(group, "dset", datatype(Int), dataspace((1,))) meta = create_attribute(dset, "meta", datatype(Bool), dataspace((1,))) @test sprint(show, meta) == "HDF5.Attribute: meta" -dsetattrs = attributes(dset) -@test sprint(show, dsetattrs) == "Attributes of HDF5.Dataset: /group/dset (file: $fn xfer_mode: 0)" +dsetattrs = attrs(dset) +@test sprint(show, dsetattrs) == "AttributeDict of HDF5.Dataset: /group/dset (file: $fn xfer_mode: 0) with 1 attribute" prop = HDF5.init!(HDF5.LinkCreateProperties()) @test sprint(show, prop) == """ @@ -803,8 +800,8 @@ commit_datatype(hfile, "type", dtype) dtypemeta = create_attribute(dtype, "dtypemeta", datatype(Bool), dataspace((1,))) @test sprint(show, dtypemeta) == "HDF5.Attribute: dtypemeta" -dtypeattrs = attributes(dtype) -@test sprint(show, dtypeattrs) == "Attributes of HDF5.Datatype: /type H5T_IEEE_F64LE" +dtypeattrs = attrs(dtype) +@test sprint(show, dtypeattrs) == "AttributeDict of HDF5.Datatype: /type H5T_IEEE_F64LE with 1 attribute" dspace_null = HDF5.Dataspace(HDF5.API.h5s_create(HDF5.API.H5S_NULL)) dspace_scal = HDF5.Dataspace(HDF5.API.h5s_create(HDF5.API.H5S_SCALAR)) @@ -842,7 +839,7 @@ close(dtypemeta) close(dset) @test sprint(show, dset) == "HDF5.Dataset: (invalid)" -@test sprint(show, dsetattrs) == "Attributes of HDF5.Dataset: (invalid)" +@test sprint(show, dsetattrs) == "AttributeDict of HDF5.Dataset: (invalid)" close(group) @test sprint(show, group) == "HDF5.Group: (invalid)" @@ -870,13 +867,13 @@ rm(fn) hfile = h5open(fn, "w") # file level hfile["version"] = 1.0 -attributes(hfile)["creator"] = "HDF5.jl" +attrs(hfile)["creator"] = "HDF5.jl" # group level create_group(hfile, "inner") -attributes(hfile["inner"])["dirty"] = true +attrs(hfile["inner"])["dirty"] = true # dataset level hfile["inner/data"] = collect(-5:5) -attributes(hfile["inner/data"])["mode"] = 1 +attrs(hfile["inner/data"])["mode"] = 1 # non-trivial committed datatype # TODO: print more datatype information tmeta = HDF5.Datatype(HDF5.API.h5t_create(HDF5.API.H5T_COMPOUND, sizeof(Int) + sizeof(Float64))) @@ -913,13 +910,6 @@ HDF5.show_tree(iobuf, hfile, attributes = false) │ └─ 🔢 data └─ 🔢 version"""m, String(take!(buf))) -HDF5.show_tree(iobuf, attributes(hfile)) -msg = String(take!(buf)) -@test occursin(r""" -🗂️ Attributes of HDF5.File: .*$ -└─ 🏷️ creator"""m, msg) -@test sprint(show3, attributes(hfile)) == msg - HDF5.show_tree(iobuf, hfile["inner"]) msg = String(take!(buf)) @test occursin(r""" @@ -1106,7 +1096,7 @@ meta1 = create_attribute(dset1, "meta1", datatype(Bool), dataspace((1,))) @test_throws KeyError dset1["nothing"] -attribs = attributes(hfile) +attribs = attrs(hfile) attribs["test1"] = true attribs["test2"] = "foo" @@ -1115,7 +1105,7 @@ attribs["test2"] = "foo" @test !haskey(attribs, "testna") @test_throws KeyError attribs["nothing"] -attribs = attributes(dset2) +attribs = attrs(dset2) attribs["attr"] = "foo" @test haskey(attribs, GenericString("attr")) @@ -1148,8 +1138,8 @@ dset1 = hfile["dset1"] array_of_strings = ["test",] write(hfile, "array_of_strings", array_of_strings) -@test_nowarn attributes(hfile)[GenericString("ref_test")] = HDF5.Reference(hfile, GenericString("array_of_strings")) -@test read(attributes(hfile)[GenericString("ref_test")]) === HDF5.Reference(hfile, "array_of_strings") +@test_nowarn attrs(hfile)[GenericString("ref_test")] = HDF5.Reference(hfile, GenericString("array_of_strings")) +@test attrs(hfile)[GenericString("ref_test")] === HDF5.Reference(hfile, "array_of_strings") hfile[GenericString("test")] = 17.2 @test_nowarn delete_object(hfile, GenericString("test")) @@ -1172,7 +1162,7 @@ for obj in (d, g) @test_nowarn write_attribute(obj, GenericString("a"), 1) @test_nowarn read_attribute(obj, GenericString("a")) @test_nowarn write(obj, GenericString("aa"), 1) - @test_nowarn attributes(obj)["attr1"] = GenericString("b") + @test_nowarn attrs(obj)["attr1"] = GenericString("b") end @test_nowarn write(d, "attr2", GenericString("c")) @test_nowarn write_dataset(g, GenericString("ag"), GenericString("gg")) @@ -1189,7 +1179,7 @@ for obj in (hfile,) @test_nowarn read(obj, GenericString("dd")) @test_nowarn read(obj, GenericString("dd")=>Int) end -read(attributes(hfile), GenericString("a")) +@test_nowarn attrs(hfile)[GenericString("a")] write(hfile, GenericString("ASD"), GenericString("Aa")) write(g, GenericString("ASD"), GenericString("Aa")) diff --git a/test/properties.jl b/test/properties.jl index bead49338..1939a1e0e 100644 --- a/test/properties.jl +++ b/test/properties.jl @@ -28,7 +28,7 @@ h5open(fn, "w"; fill_time = :never, obj_track_times = false, kwargs...) - attributes(d)["metadata"] = "test" + attrs(d)["metadata"] = "test" flush(hfile) @@ -36,7 +36,7 @@ h5open(fn, "w"; fapl = HDF5.get_access_properties(hfile) gcpl = HDF5.get_create_properties(hfile["group"]) dcpl = HDF5.get_create_properties(hfile["group/dataset"]) - acpl = HDF5.get_create_properties(attributes(hfile["group/dataset"])["metadata"]) + acpl = HDF5.get_create_properties(open_attribute(hfile["group/dataset"], "metadata")) # Retrievability of properties @test isvalid(fcpl) diff --git a/test/runtests.jl b/test/runtests.jl index 3dfb41e33..869fdb946 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -28,6 +28,8 @@ include("dataspace.jl") include("datatype.jl") @debug "hyperslab" include("hyperslab.jl") +@debug "attributes" +include("attributes.jl") @debug "readremote" include("readremote.jl") @debug "extend_test" diff --git a/test/swmr.jl b/test/swmr.jl index 0cc6e7e2a..88c127010 100644 --- a/test/swmr.jl +++ b/test/swmr.jl @@ -92,7 +92,7 @@ end # create datasets and attributes before staring swmr writing function prep_h5_file(h5) d = create_dataset(h5, "foo", datatype(Int), ((1,), (100,)), chunk=(1,)) - attributes(h5)["bar"] = "bar" + attrs(h5)["bar"] = "bar" g = create_group(h5, "group") end From fddc71ae038329187275646acfe029acc1b42098 Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Sat, 4 Jun 2022 23:16:26 -0700 Subject: [PATCH 2/5] update docs and deprecate --- docs/src/attributes.md | 6 +++-- src/attributes.jl | 53 ++++-------------------------------------- src/deprecated.jl | 33 +++++++++++++++++++++++++- test/attributes.jl | 3 +++ test/plain.jl | 37 ++++++++++++++--------------- test/reference.jl | 4 ++-- 6 files changed, 63 insertions(+), 73 deletions(-) diff --git a/docs/src/attributes.md b/docs/src/attributes.md index 06a188c5e..c29bdcb23 100644 --- a/docs/src/attributes.md +++ b/docs/src/attributes.md @@ -1,13 +1,15 @@ # Attributes +## Dictionary interface + ```@docs -HDF5.Attribute -attributes +attrs ``` ## Mid-level Interface ```@docs +HDF5.Attribute open_attribute create_attribute read_attribute diff --git a/src/attributes.jl b/src/attributes.jl index 2257ff3af..d8e53fd56 100644 --- a/src/attributes.jl +++ b/src/attributes.jl @@ -167,10 +167,7 @@ function h5writeattr(filename, name::AbstractString, data::Dict) file = h5open(filename, "r+") try obj = file[name] - attrs = attributes(obj) - for x in keys(data) - attrs[x] = data[x] - end + merge!(attrs(obj), data) close(obj) finally close(file) @@ -187,8 +184,7 @@ function h5readattr(filename, name::AbstractString) file = h5open(filename,"r") try obj = file[name] - a = attributes(obj) - dat = Dict(x => read(a[x]) for x in keys(a)) + dat = Dict(attrs(obj)) close(obj) finally close(file) @@ -272,48 +268,7 @@ function Base.iterate(attrdict::AttributeDict, (keyvec, n)) return (key => attrdict[key]), (keyvec, nn) end - - - +# deprecated, but retain definition as type is used in show.jl struct Attributes parent::Union{File,Object} -end - -""" - attributes(object::Union{File,Object}) - -The attributes of a file or object: this returns an `Attributes` object, which -is `Dict`-like object for accessing the attributes of `object`: `getindex` will -return an [`Attribute`](@ref) object, and `setindex!` will call [`write_attribute`](@ref). -""" -attributes(p::Union{File,Object}) = Attributes(p) - -Base.isvalid(obj::Attributes) = isvalid(obj.parent) - -function Base.getindex(x::Attributes, name::AbstractString) - haskey(x, name) || throw(KeyError(name)) - open_attribute(x.parent, name) -end -Base.setindex!(x::Attributes, val, name::AbstractString) = write_attribute(x.parent, name, val) -Base.haskey(attr::Attributes, path::AbstractString) = API.h5a_exists(checkvalid(attr.parent), path) -Base.length(x::Attributes) = Int(object_info(x.parent).num_attrs) - -function Base.keys(x::Attributes) - checkvalid(x.parent) - children = sizehint!(String[], length(x)) - API.h5a_iterate(x.parent, IDX_TYPE[], ORDER[]) do _, attr_name, _ - push!(children, unsafe_string(attr_name)) - return API.herr_t(0) - end - return children -end -Base.read(attr::Attributes, name::AbstractString) = read_attribute(attr.parent, name) - - -# Datasets act like attributes -Base.write(parent::Dataset, name::AbstractString, data; pv...) = write_attribute(parent, name, data; pv...) -function Base.getindex(dset::Dataset, name::AbstractString) - haskey(dset, name) || throw(KeyError(name)) - open_attribute(dset, name) -end -Base.setindex!(dset::Dataset, val, name::AbstractString) = write_attribute(dset, name, val) +end \ No newline at end of file diff --git a/src/deprecated.jl b/src/deprecated.jl index 2a57bf6f8..da882a9d5 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -15,7 +15,7 @@ for name in names(API; all=true) end ### Changed in PR #847 -import Base: getindex, setindex! +import Base: getindex, setindex!, write @deprecate getindex(p::Properties, name::Symbol) Base.getproperty(p, name) @deprecate setindex!(p::Properties, val, name::Symbol) Base.setproperty!(p, name, val) @@ -68,3 +68,34 @@ import .Filters: ExternalFilter @deprecate append!(filters::Filters.FilterPipeline, extra::NTuple{N, Integer}) where N append!(filters, [ExternalFilter(extra...)]) @deprecate push!(p::Filters.FilterPipeline, f::NTuple{N, Integer}) where N push!(p, ExternalFilter(f...)) @deprecate ExternalFilter(t::Tuple) ExternalFilter(t...) false + +### Changed in PR #948 +function attributes(p::Union{File,Object}) + Base.depwarn("`attributes(obj)` has been deprecated, use `attrs(obj)` instead.", :attributes) + Attributes(p) +end +Base.isvalid(obj::Attributes) = isvalid(obj.parent) + +function Base.getindex(x::Attributes, name::AbstractString) + haskey(x, name) || throw(KeyError(name)) + open_attribute(x.parent, name) +end +Base.setindex!(x::Attributes, val, name::AbstractString) = write_attribute(x.parent, name, val) +Base.haskey(attr::Attributes, path::AbstractString) = API.h5a_exists(checkvalid(attr.parent), path) +Base.length(x::Attributes) = Int(object_info(x.parent).num_attrs) + +function Base.keys(x::Attributes) + checkvalid(x.parent) + children = sizehint!(String[], length(x)) + API.h5a_iterate(x.parent, IDX_TYPE[], ORDER[]) do _, attr_name, _ + push!(children, unsafe_string(attr_name)) + return API.herr_t(0) + end + return children +end +Base.read(attr::Attributes, name::AbstractString) = read_attribute(attr.parent, name) + +# Dataset methods which act like attributes +@deprecate write(parent::Dataset, name::AbstractString, data; pv...) write_attribute(parent, name, data; pv...) +@deprecate getindex(dset::Dataset, name::AbstractString) open_attribute(dset, name) +@deprecate setindex!(dset::Dataset, val, name::AbstractString) attrs(dset)[name] = val diff --git a/test/attributes.jl b/test/attributes.jl index 9eb5b6d99..c938f493d 100644 --- a/test/attributes.jl +++ b/test/attributes.jl @@ -35,3 +35,6 @@ attrs(f)["b"] = "a string" delete!(attrs(f), "a") @test length(attrs(f)) == 1 @test sort(keys(attrs(f))) == ["b"] + +@test_throws KeyError attrs(f)["a"] +@test isnothing(get(attrs(f), "a", nothing)) \ No newline at end of file diff --git a/test/plain.jl b/test/plain.jl index c63b0a134..15e982689 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -215,7 +215,7 @@ salut_vlenr = read(fr, "salut_vlen") #@test salut_vlenr == salut_split vlen_intr = read(fr, "int_vlen") @test vlen_intr == vlen_int -vlen_attrr = read(fr["int_vlen"]["vlen_attr"]) +vlen_attrr = read_attribute(fr["int_vlen"], "vlen_attr") @test vlen_attrr == vlen_int Rr = read(fr, "mygroup/CompressedA") @test Rr == R @@ -653,16 +653,16 @@ dzeromat2 = hfile["zeromat2"] @test read(dzerodim) == 1.0π # Similar tests for writing to attributes -write(dempty, "attr", HDF5.EmptyArray{Float64}()) -write(dzerodim, "attr", fill(1.0ℯ)) -write(dzerovec, "attr", zeros(Int64, 0)) -write(dzeromat, "attr", zeros(Int64, 0, 0)) -write(dzeromat2, "attr", zeros(Int64, 0, 1)) -aempty = dempty["attr"] -azerodim = dzerodim["attr"] -azerovec = dzerovec["attr"] -azeromat = dzeromat["attr"] -azeromat2 = dzeromat2["attr"] +write_attribute(dempty, "attr", HDF5.EmptyArray{Float64}()) +write_attribute(dzerodim, "attr", fill(1.0ℯ)) +write_attribute(dzerovec, "attr", zeros(Int64, 0)) +write_attribute(dzeromat, "attr", zeros(Int64, 0, 0)) +write_attribute(dzeromat2, "attr", zeros(Int64, 0, 1)) +aempty = open_attribute(dempty, "attr") +azerodim = open_attribute(dzerodim, "attr") +azerovec = open_attribute(dzerovec, "attr") +azeromat = open_attribute(dzeromat, "attr") +azeromat2 = open_attribute(dzeromat2, "attr") # Test that eltype is preserved (especially for EmptyArray) @test eltype(aempty) == Float64 @test eltype(azerodim) == Float64 @@ -942,7 +942,7 @@ HDF5.show_tree(iobuf, hfile["dtype"]) @test occursin(r""" 📄 HDF5.Datatype: /dtype""", String(take!(buf))) -HDF5.show_tree(iobuf, hfile["inner/data"]["mode"], attributes = true) +HDF5.show_tree(iobuf, open_attribute(hfile["inner/data"],"mode"), attributes = true) @test occursin(r""" 🏷️ HDF5.Attribute: mode""", String(take!(buf))) @@ -1093,7 +1093,7 @@ dset2 = create_dataset(group1, "dset2", datatype(Int), dataspace((1,))) meta1 = create_attribute(dset1, "meta1", datatype(Bool), dataspace((1,))) @test haskey(dset1, "meta1") @test !haskey(dset1, "metana") -@test_throws KeyError dset1["nothing"] +@test_throws KeyError attrs(dset1)["nothing"] attribs = attrs(hfile) @@ -1133,8 +1133,8 @@ hfile = h5open(fn, "w") dset1 = hfile["dset1"] @test_nowarn create_attribute(dset1, GenericString("meta1"), datatype(Bool), dataspace((1,))) @test_nowarn create_attribute(dset1, GenericString("meta2"), 1) -@test_nowarn dset1[GenericString("meta1")] -@test_nowarn dset1[GenericString("x")] = 2 +@test_nowarn attrs(dset1)[GenericString("meta1")] +@test_nowarn attrs(dset1)[GenericString("x")] = 2 array_of_strings = ["test",] write(hfile, "array_of_strings", array_of_strings) @@ -1161,16 +1161,15 @@ a = create_attribute(hfile, GenericString("a"), dt, ds) for obj in (d, g) @test_nowarn write_attribute(obj, GenericString("a"), 1) @test_nowarn read_attribute(obj, GenericString("a")) - @test_nowarn write(obj, GenericString("aa"), 1) @test_nowarn attrs(obj)["attr1"] = GenericString("b") end -@test_nowarn write(d, "attr2", GenericString("c")) +@test_nowarn write_attribute(d, "attr2", GenericString("c")) @test_nowarn write_dataset(g, GenericString("ag"), GenericString("gg")) @test_nowarn write_dataset(g, GenericString("ag_array"), [GenericString("a1"), GenericString("a2")]) genstrs = GenericString["fee", "fi", "foo"] @test_nowarn write_attribute(d, GenericString("myattr"), genstrs) -@test genstrs == read(d["myattr"]) +@test genstrs == attrs(d)["myattr"] for obj in (hfile,) @test_nowarn open_dataset(obj, GenericString("d")) @@ -1190,7 +1189,7 @@ write(g, GenericString("ASD1"), [GenericString("Aa")]) # copy methods d1 = create_dataset(hfile, GenericString("d1"), dt, ds) -d1["x"] = 32 +attrs(d1)["x"] = 32 @test_nowarn copy_object(hfile, GenericString("d1"), hfile, GenericString("d1copy1")) @test_nowarn copy_object(d1, hfile, GenericString("d1copy2")) diff --git a/test/reference.jl b/test/reference.jl index 81dcc3de3..d890356f7 100644 --- a/test/reference.jl +++ b/test/reference.jl @@ -15,7 +15,7 @@ using Random, Test, HDF5 g = create_group(f, "sub") g["group_ref"] = HDF5.Reference(f, "group_data") # reference attached to dataset - f["data"]["attr_ref"] = HDF5.Reference(f, "attr_data") + attrs(f["data"])["attr_ref"] = HDF5.Reference(f, "attr_data") close(f) @@ -29,7 +29,7 @@ using Random, Test, HDF5 @test gref isa HDF5.Reference @test data == read(f["sub"][gref]) # read back dataset-attached reference - aref = read(f["data"]["attr_ref"]) + aref = attrs(f["data"])["attr_ref"] @test aref isa HDF5.Reference @test data == read(f["data"][aref]) From 4937d661e217eeaf8d4b6658d165003d4c085a6d Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Sun, 5 Jun 2022 23:04:34 -0700 Subject: [PATCH 3/5] revert deprecation --- docs/src/attributes.md | 2 + src/attributes.jl | 44 ++++++++++++++++++- src/deprecated.jl | 33 +------------- test/attributes.jl | 84 +++++++++++++++++++---------------- test/gc.jl | 2 +- test/plain.jl | 99 +++++++++++++++++++++++------------------- test/properties.jl | 4 +- test/reference.jl | 4 +- test/swmr.jl | 2 +- 9 files changed, 152 insertions(+), 122 deletions(-) diff --git a/docs/src/attributes.md b/docs/src/attributes.md index c29bdcb23..3277daead 100644 --- a/docs/src/attributes.md +++ b/docs/src/attributes.md @@ -4,6 +4,7 @@ ```@docs attrs +attributes ``` ## Mid-level Interface @@ -15,6 +16,7 @@ create_attribute read_attribute write_attribute delete_attribute +rename_attribute ``` ## Convenience interface diff --git a/src/attributes.jl b/src/attributes.jl index d8e53fd56..888d12e38 100644 --- a/src/attributes.jl +++ b/src/attributes.jl @@ -268,7 +268,47 @@ function Base.iterate(attrdict::AttributeDict, (keyvec, n)) return (key => attrdict[key]), (keyvec, nn) end -# deprecated, but retain definition as type is used in show.jl + + + struct Attributes parent::Union{File,Object} -end \ No newline at end of file +end + +""" + attributes(object::Union{File,Object}) + +The attributes of a file or object: this returns an `Attributes` object, which +is `Dict`-like object for accessing the attributes of `object`: `getindex` will +return an [`Attribute`](@ref) object, and `setindex!` will call [`write_attribute`](@ref). +""" +attributes(p::Union{File,Object}) = Attributes(p) + +Base.isvalid(obj::Attributes) = isvalid(obj.parent) + +function Base.getindex(x::Attributes, name::AbstractString) + haskey(x, name) || throw(KeyError(name)) + open_attribute(x.parent, name) +end +Base.setindex!(x::Attributes, val, name::AbstractString) = write_attribute(x.parent, name, val) +Base.haskey(attr::Attributes, path::AbstractString) = API.h5a_exists(checkvalid(attr.parent), path) +Base.length(x::Attributes) = Int(object_info(x.parent).num_attrs) + +function Base.keys(x::Attributes) + checkvalid(x.parent) + children = sizehint!(String[], length(x)) + API.h5a_iterate(x.parent, IDX_TYPE[], ORDER[]) do _, attr_name, _ + push!(children, unsafe_string(attr_name)) + return API.herr_t(0) + end + return children +end +Base.read(attr::Attributes, name::AbstractString) = read_attribute(attr.parent, name) + +# Dataset methods which act like attributes +Base.write(parent::Dataset, name::AbstractString, data; pv...) = write_attribute(parent, name, data; pv...) +function Base.getindex(dset::Dataset, name::AbstractString) + haskey(dset, name) || throw(KeyError(name)) + open_attribute(dset, name) +end +Base.setindex!(dset::Dataset, val, name::AbstractString) = write_attribute(dset, name, val) diff --git a/src/deprecated.jl b/src/deprecated.jl index da882a9d5..2a57bf6f8 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -15,7 +15,7 @@ for name in names(API; all=true) end ### Changed in PR #847 -import Base: getindex, setindex!, write +import Base: getindex, setindex! @deprecate getindex(p::Properties, name::Symbol) Base.getproperty(p, name) @deprecate setindex!(p::Properties, val, name::Symbol) Base.setproperty!(p, name, val) @@ -68,34 +68,3 @@ import .Filters: ExternalFilter @deprecate append!(filters::Filters.FilterPipeline, extra::NTuple{N, Integer}) where N append!(filters, [ExternalFilter(extra...)]) @deprecate push!(p::Filters.FilterPipeline, f::NTuple{N, Integer}) where N push!(p, ExternalFilter(f...)) @deprecate ExternalFilter(t::Tuple) ExternalFilter(t...) false - -### Changed in PR #948 -function attributes(p::Union{File,Object}) - Base.depwarn("`attributes(obj)` has been deprecated, use `attrs(obj)` instead.", :attributes) - Attributes(p) -end -Base.isvalid(obj::Attributes) = isvalid(obj.parent) - -function Base.getindex(x::Attributes, name::AbstractString) - haskey(x, name) || throw(KeyError(name)) - open_attribute(x.parent, name) -end -Base.setindex!(x::Attributes, val, name::AbstractString) = write_attribute(x.parent, name, val) -Base.haskey(attr::Attributes, path::AbstractString) = API.h5a_exists(checkvalid(attr.parent), path) -Base.length(x::Attributes) = Int(object_info(x.parent).num_attrs) - -function Base.keys(x::Attributes) - checkvalid(x.parent) - children = sizehint!(String[], length(x)) - API.h5a_iterate(x.parent, IDX_TYPE[], ORDER[]) do _, attr_name, _ - push!(children, unsafe_string(attr_name)) - return API.herr_t(0) - end - return children -end -Base.read(attr::Attributes, name::AbstractString) = read_attribute(attr.parent, name) - -# Dataset methods which act like attributes -@deprecate write(parent::Dataset, name::AbstractString, data; pv...) write_attribute(parent, name, data; pv...) -@deprecate getindex(dset::Dataset, name::AbstractString) open_attribute(dset, name) -@deprecate setindex!(dset::Dataset, val, name::AbstractString) attrs(dset)[name] = val diff --git a/test/attributes.jl b/test/attributes.jl index c938f493d..2dce603f6 100644 --- a/test/attributes.jl +++ b/test/attributes.jl @@ -1,40 +1,48 @@ using HDF5, Test - -filename = tempname() -f = h5open(filename, "w") - -@test attrs(f) isa HDF5.AttributeDict - -attrs(f)["a"] = 1 -@test attrs(f)["a"] == 1 - -attrs(f)["b"] = [2,3] -@test attrs(f)["b"] == [2,3] -@test length(attrs(f)) == 2 -@test sort(keys(attrs(f))) == ["a", "b"] - -# overwrite: same type -attrs(f)["a"] = 4 -@test attrs(f)["a"] == 4 -@test length(attrs(f)) == 2 -@test sort(keys(attrs(f))) == ["a", "b"] - -# overwrite: different size -attrs(f)["b"] = [4,5,6] -@test attrs(f)["b"] == [4,5,6] -@test length(attrs(f)) == 2 -@test sort(keys(attrs(f))) == ["a", "b"] - -# overwrite: different type -attrs(f)["b"] = "a string" -@test attrs(f)["b"] == "a string" -@test length(attrs(f)) == 2 -@test sort(keys(attrs(f))) == ["a", "b"] - -delete!(attrs(f), "a") -@test length(attrs(f)) == 1 -@test sort(keys(attrs(f))) == ["b"] - -@test_throws KeyError attrs(f)["a"] -@test isnothing(get(attrs(f), "a", nothing)) \ No newline at end of file +@testset "attrs interface" begin + filename = tempname() + f = h5open(filename, "w") + + @test attrs(f) isa HDF5.AttributeDict + + attrs(f)["a"] = 1 + @test haskey(attrs(f), "a") + @test attrs(f)["a"] == 1 + + attrs(f)["b"] = [2,3] + @test attrs(f)["b"] == [2,3] + @test haskey(attrs(f), "a") + @test length(attrs(f)) == 2 + @test sort(keys(attrs(f))) == ["a", "b"] + + @test !haskey(attrs(f), "c") + + # overwrite: same type + attrs(f)["a"] = 4 + @test attrs(f)["a"] == 4 + @test get(attrs(f), "a", nothing) == 4 + @test length(attrs(f)) == 2 + @test sort(keys(attrs(f))) == ["a", "b"] + + # overwrite: different size + attrs(f)["b"] = [4,5,6] + @test attrs(f)["b"] == [4,5,6] + @test length(attrs(f)) == 2 + @test sort(keys(attrs(f))) == ["a", "b"] + + # overwrite: different type + attrs(f)["b"] = "a string" + @test attrs(f)["b"] == "a string" + @test length(attrs(f)) == 2 + @test sort(keys(attrs(f))) == ["a", "b"] + + # delete a key + delete!(attrs(f), "a") + @test !haskey(attrs(f), "a") + @test length(attrs(f)) == 1 + @test sort(keys(attrs(f))) == ["b"] + + @test_throws KeyError attrs(f)["a"] + @test isnothing(get(attrs(f), "a", nothing)) +end \ No newline at end of file diff --git a/test/gc.jl b/test/gc.jl index 89e37bef7..c34eede9f 100644 --- a/test/gc.jl +++ b/test/gc.jl @@ -58,7 +58,7 @@ for i = 1:10 d = file["d"] ds = dataspace(d) g = file["g"] - a = open_attribute(file, "a") + a = attributes(file)["a"] @gcvalid dt ds d g a close(file) end diff --git a/test/plain.jl b/test/plain.jl index 15e982689..47e7d44ea 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -100,19 +100,22 @@ empty_array_of_strings = [""] write(f, "empty_array_of_strings", empty_array_of_strings) # attributes species = [["N", "C"]; ["A", "B"]] -attrs(f)["species"] = species -@test attrs(f)["species"] == species +attributes(f)["species"] = species +@test read(attributes(f)["species"]) == species +@test attributes(f)["species"][] == species C∞ = 42 -attrs(f)["C∞"] = C∞ +attributes(f)["C∞"] = C∞ dset = f["salut"] @test !isempty(dset) label = "This is a string" -attrs(dset)["typeinfo"] = label -@test attrs(dset)["typeinfo"] == label +attributes(dset)["typeinfo"] = label +@test read(attributes(dset)["typeinfo"]) == label +@test attributes(dset)["typeinfo"][] == label +@test dset["typeinfo"][] == label close(dset) # Scalar reference values in attributes -attrs(f)["ref_test"] = HDF5.Reference(f, "empty_array_of_strings") -@test attrs(f)["ref_test"] === HDF5.Reference(f, "empty_array_of_strings") +attributes(f)["ref_test"] = HDF5.Reference(f, "empty_array_of_strings") +@test read(attributes(f)["ref_test"]) === HDF5.Reference(f, "empty_array_of_strings") # Group g = create_group(f, "mygroup") # Test dataset with compression @@ -215,7 +218,7 @@ salut_vlenr = read(fr, "salut_vlen") #@test salut_vlenr == salut_split vlen_intr = read(fr, "int_vlen") @test vlen_intr == vlen_int -vlen_attrr = read_attribute(fr["int_vlen"], "vlen_attr") +vlen_attrr = read(fr["int_vlen"]["vlen_attr"]) @test vlen_attrr == vlen_int Rr = read(fr, "mygroup/CompressedA") @test Rr == R @@ -317,10 +320,10 @@ A = rand(3, 3)' @test_throws ArgumentError write(hid, "A", A) @test !haskey(hid, "A") dset = create_dataset(hid, "attr", datatype(Int), dataspace(0)) -@test !haskey(attrs(dset), "attr") +@test !haskey(attributes(dset), "attr") # broken test - writing attributes does not check that the stride is correct @test_skip @test_throws ArgumentError write(dset, "attr", A) -@test !haskey(attrs(dset), "attr") +@test !haskey(attributes(dset), "attr") close(hid) # more do syntax @@ -504,7 +507,7 @@ end # testset plain f = h5open(fn, "w") f["ComplexF64"] = 1.0 + 2.0im - attrs(f["ComplexF64"])["ComplexInt64"] = 1im + attributes(f["ComplexF64"])["ComplexInt64"] = 1im Acmplx = rand(ComplexF64, 3, 5) write(f, "Acmplx64", convert(Matrix{ComplexF64}, Acmplx)) @@ -526,8 +529,8 @@ end # testset plain fr = h5open(fn) z = read(fr, "ComplexF64") @test z == 1.0 + 2.0im && isa(z, ComplexF64) - z_attrs = attrs(fr["ComplexF64"]) - @test z_attrs["ComplexInt64"] == 1im + z_attrs = attributes(fr["ComplexF64"]) + @test read(z_attrs["ComplexInt64"]) == 1im Acmplx32 = read(fr, "Acmplx32") @test convert(Matrix{ComplexF32}, Acmplx) == Acmplx32 @@ -653,16 +656,16 @@ dzeromat2 = hfile["zeromat2"] @test read(dzerodim) == 1.0π # Similar tests for writing to attributes -write_attribute(dempty, "attr", HDF5.EmptyArray{Float64}()) -write_attribute(dzerodim, "attr", fill(1.0ℯ)) -write_attribute(dzerovec, "attr", zeros(Int64, 0)) -write_attribute(dzeromat, "attr", zeros(Int64, 0, 0)) -write_attribute(dzeromat2, "attr", zeros(Int64, 0, 1)) -aempty = open_attribute(dempty, "attr") -azerodim = open_attribute(dzerodim, "attr") -azerovec = open_attribute(dzerovec, "attr") -azeromat = open_attribute(dzeromat, "attr") -azeromat2 = open_attribute(dzeromat2, "attr") +write(dempty, "attr", HDF5.EmptyArray{Float64}()) +write(dzerodim, "attr", fill(1.0ℯ)) +write(dzerovec, "attr", zeros(Int64, 0)) +write(dzeromat, "attr", zeros(Int64, 0, 0)) +write(dzeromat2, "attr", zeros(Int64, 0, 1)) +aempty = dempty["attr"] +azerodim = dzerodim["attr"] +azerovec = dzerovec["attr"] +azeromat = dzeromat["attr"] +azeromat2 = dzeromat2["attr"] # Test that eltype is preserved (especially for EmptyArray) @test eltype(aempty) == Float64 @test eltype(azerodim) == Float64 @@ -779,8 +782,8 @@ dset = create_dataset(group, "dset", datatype(Int), dataspace((1,))) meta = create_attribute(dset, "meta", datatype(Bool), dataspace((1,))) @test sprint(show, meta) == "HDF5.Attribute: meta" -dsetattrs = attrs(dset) -@test sprint(show, dsetattrs) == "AttributeDict of HDF5.Dataset: /group/dset (file: $fn xfer_mode: 0) with 1 attribute" +dsetattrs = attributes(dset) +@test sprint(show, dsetattrs) == "Attributes of HDF5.Dataset: /group/dset (file: $fn xfer_mode: 0)" prop = HDF5.init!(HDF5.LinkCreateProperties()) @test sprint(show, prop) == """ @@ -800,8 +803,8 @@ commit_datatype(hfile, "type", dtype) dtypemeta = create_attribute(dtype, "dtypemeta", datatype(Bool), dataspace((1,))) @test sprint(show, dtypemeta) == "HDF5.Attribute: dtypemeta" -dtypeattrs = attrs(dtype) -@test sprint(show, dtypeattrs) == "AttributeDict of HDF5.Datatype: /type H5T_IEEE_F64LE with 1 attribute" +dtypeattrs = attributes(dtype) +@test sprint(show, dtypeattrs) == "Attributes of HDF5.Datatype: /type H5T_IEEE_F64LE" dspace_null = HDF5.Dataspace(HDF5.API.h5s_create(HDF5.API.H5S_NULL)) dspace_scal = HDF5.Dataspace(HDF5.API.h5s_create(HDF5.API.H5S_SCALAR)) @@ -839,7 +842,7 @@ close(dtypemeta) close(dset) @test sprint(show, dset) == "HDF5.Dataset: (invalid)" -@test sprint(show, dsetattrs) == "AttributeDict of HDF5.Dataset: (invalid)" +@test sprint(show, dsetattrs) == "Attributes of HDF5.Dataset: (invalid)" close(group) @test sprint(show, group) == "HDF5.Group: (invalid)" @@ -867,13 +870,13 @@ rm(fn) hfile = h5open(fn, "w") # file level hfile["version"] = 1.0 -attrs(hfile)["creator"] = "HDF5.jl" +attributes(hfile)["creator"] = "HDF5.jl" # group level create_group(hfile, "inner") -attrs(hfile["inner"])["dirty"] = true +attributes(hfile["inner"])["dirty"] = true # dataset level hfile["inner/data"] = collect(-5:5) -attrs(hfile["inner/data"])["mode"] = 1 +attributes(hfile["inner/data"])["mode"] = 1 # non-trivial committed datatype # TODO: print more datatype information tmeta = HDF5.Datatype(HDF5.API.h5t_create(HDF5.API.H5T_COMPOUND, sizeof(Int) + sizeof(Float64))) @@ -910,6 +913,13 @@ HDF5.show_tree(iobuf, hfile, attributes = false) │ └─ 🔢 data └─ 🔢 version"""m, String(take!(buf))) +HDF5.show_tree(iobuf, attributes(hfile)) +msg = String(take!(buf)) +@test occursin(r""" +🗂️ Attributes of HDF5.File: .*$ +└─ 🏷️ creator"""m, msg) +@test sprint(show3, attributes(hfile)) == msg + HDF5.show_tree(iobuf, hfile["inner"]) msg = String(take!(buf)) @test occursin(r""" @@ -942,7 +952,7 @@ HDF5.show_tree(iobuf, hfile["dtype"]) @test occursin(r""" 📄 HDF5.Datatype: /dtype""", String(take!(buf))) -HDF5.show_tree(iobuf, open_attribute(hfile["inner/data"],"mode"), attributes = true) +HDF5.show_tree(iobuf, hfile["inner/data"]["mode"], attributes = true) @test occursin(r""" 🏷️ HDF5.Attribute: mode""", String(take!(buf))) @@ -1093,10 +1103,10 @@ dset2 = create_dataset(group1, "dset2", datatype(Int), dataspace((1,))) meta1 = create_attribute(dset1, "meta1", datatype(Bool), dataspace((1,))) @test haskey(dset1, "meta1") @test !haskey(dset1, "metana") -@test_throws KeyError attrs(dset1)["nothing"] +@test_throws KeyError dset1["nothing"] -attribs = attrs(hfile) +attribs = attributes(hfile) attribs["test1"] = true attribs["test2"] = "foo" @@ -1105,7 +1115,7 @@ attribs["test2"] = "foo" @test !haskey(attribs, "testna") @test_throws KeyError attribs["nothing"] -attribs = attrs(dset2) +attribs = attributes(dset2) attribs["attr"] = "foo" @test haskey(attribs, GenericString("attr")) @@ -1133,13 +1143,13 @@ hfile = h5open(fn, "w") dset1 = hfile["dset1"] @test_nowarn create_attribute(dset1, GenericString("meta1"), datatype(Bool), dataspace((1,))) @test_nowarn create_attribute(dset1, GenericString("meta2"), 1) -@test_nowarn attrs(dset1)[GenericString("meta1")] -@test_nowarn attrs(dset1)[GenericString("x")] = 2 +@test_nowarn dset1[GenericString("meta1")] +@test_nowarn dset1[GenericString("x")] = 2 array_of_strings = ["test",] write(hfile, "array_of_strings", array_of_strings) -@test_nowarn attrs(hfile)[GenericString("ref_test")] = HDF5.Reference(hfile, GenericString("array_of_strings")) -@test attrs(hfile)[GenericString("ref_test")] === HDF5.Reference(hfile, "array_of_strings") +@test_nowarn attributes(hfile)[GenericString("ref_test")] = HDF5.Reference(hfile, GenericString("array_of_strings")) +@test read(attributes(hfile)[GenericString("ref_test")]) === HDF5.Reference(hfile, "array_of_strings") hfile[GenericString("test")] = 17.2 @test_nowarn delete_object(hfile, GenericString("test")) @@ -1161,15 +1171,16 @@ a = create_attribute(hfile, GenericString("a"), dt, ds) for obj in (d, g) @test_nowarn write_attribute(obj, GenericString("a"), 1) @test_nowarn read_attribute(obj, GenericString("a")) - @test_nowarn attrs(obj)["attr1"] = GenericString("b") + @test_nowarn write(obj, GenericString("aa"), 1) + @test_nowarn attributes(obj)["attr1"] = GenericString("b") end -@test_nowarn write_attribute(d, "attr2", GenericString("c")) +@test_nowarn write(d, "attr2", GenericString("c")) @test_nowarn write_dataset(g, GenericString("ag"), GenericString("gg")) @test_nowarn write_dataset(g, GenericString("ag_array"), [GenericString("a1"), GenericString("a2")]) genstrs = GenericString["fee", "fi", "foo"] @test_nowarn write_attribute(d, GenericString("myattr"), genstrs) -@test genstrs == attrs(d)["myattr"] +@test genstrs == read(d["myattr"]) for obj in (hfile,) @test_nowarn open_dataset(obj, GenericString("d")) @@ -1178,7 +1189,7 @@ for obj in (hfile,) @test_nowarn read(obj, GenericString("dd")) @test_nowarn read(obj, GenericString("dd")=>Int) end -@test_nowarn attrs(hfile)[GenericString("a")] +read(attributes(hfile), GenericString("a")) write(hfile, GenericString("ASD"), GenericString("Aa")) write(g, GenericString("ASD"), GenericString("Aa")) @@ -1189,7 +1200,7 @@ write(g, GenericString("ASD1"), [GenericString("Aa")]) # copy methods d1 = create_dataset(hfile, GenericString("d1"), dt, ds) -attrs(d1)["x"] = 32 +d1["x"] = 32 @test_nowarn copy_object(hfile, GenericString("d1"), hfile, GenericString("d1copy1")) @test_nowarn copy_object(d1, hfile, GenericString("d1copy2")) diff --git a/test/properties.jl b/test/properties.jl index 1939a1e0e..bead49338 100644 --- a/test/properties.jl +++ b/test/properties.jl @@ -28,7 +28,7 @@ h5open(fn, "w"; fill_time = :never, obj_track_times = false, kwargs...) - attrs(d)["metadata"] = "test" + attributes(d)["metadata"] = "test" flush(hfile) @@ -36,7 +36,7 @@ h5open(fn, "w"; fapl = HDF5.get_access_properties(hfile) gcpl = HDF5.get_create_properties(hfile["group"]) dcpl = HDF5.get_create_properties(hfile["group/dataset"]) - acpl = HDF5.get_create_properties(open_attribute(hfile["group/dataset"], "metadata")) + acpl = HDF5.get_create_properties(attributes(hfile["group/dataset"])["metadata"]) # Retrievability of properties @test isvalid(fcpl) diff --git a/test/reference.jl b/test/reference.jl index d890356f7..81dcc3de3 100644 --- a/test/reference.jl +++ b/test/reference.jl @@ -15,7 +15,7 @@ using Random, Test, HDF5 g = create_group(f, "sub") g["group_ref"] = HDF5.Reference(f, "group_data") # reference attached to dataset - attrs(f["data"])["attr_ref"] = HDF5.Reference(f, "attr_data") + f["data"]["attr_ref"] = HDF5.Reference(f, "attr_data") close(f) @@ -29,7 +29,7 @@ using Random, Test, HDF5 @test gref isa HDF5.Reference @test data == read(f["sub"][gref]) # read back dataset-attached reference - aref = attrs(f["data"])["attr_ref"] + aref = read(f["data"]["attr_ref"]) @test aref isa HDF5.Reference @test data == read(f["data"][aref]) diff --git a/test/swmr.jl b/test/swmr.jl index 88c127010..0cc6e7e2a 100644 --- a/test/swmr.jl +++ b/test/swmr.jl @@ -92,7 +92,7 @@ end # create datasets and attributes before staring swmr writing function prep_h5_file(h5) d = create_dataset(h5, "foo", datatype(Int), ((1,), (100,)), chunk=(1,)) - attrs(h5)["bar"] = "bar" + attributes(h5)["bar"] = "bar" g = create_group(h5, "group") end From 1c95dffa1a12be316a41e6e525a444f6a5b46316 Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Wed, 8 Jun 2022 14:09:52 -0700 Subject: [PATCH 4/5] Update src/attributes.jl Co-authored-by: Mark Kittisopikul --- src/attributes.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/attributes.jl b/src/attributes.jl index 888d12e38..1447a762e 100644 --- a/src/attributes.jl +++ b/src/attributes.jl @@ -210,10 +210,11 @@ delete!(attrs(object), "name") # delete an attribute keys(attrs(object)) # list the attribute names ``` """ -function attrs(parent::Object) +AttributeDict(file::File) = AttributeDict(open_group(file, ".")) + +function attrs(parent) return AttributeDict(parent) end -attrs(file::File) = attrs(open_group(file, ".")) Base.haskey(attrdict::AttributeDict, path::AbstractString) = API.h5a_exists(checkvalid(attrdict.parent), path) Base.length(attrdict::AttributeDict) = Int(object_info(attrdict.parent).num_attrs) From 875807ad926033f61eae79c019a84d4b951b0051 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Thu, 9 Jun 2022 16:01:05 -0400 Subject: [PATCH 5/5] Test AttributeDict on Group, Dataset, and Datatype --- test/attributes.jl | 90 +++++++++++++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 32 deletions(-) diff --git a/test/attributes.jl b/test/attributes.jl index 2dce603f6..5678c690c 100644 --- a/test/attributes.jl +++ b/test/attributes.jl @@ -1,48 +1,74 @@ using HDF5, Test -@testset "attrs interface" begin - filename = tempname() - f = h5open(filename, "w") +function test_attrs(o::Union{HDF5.File, HDF5.Object}) - @test attrs(f) isa HDF5.AttributeDict + @test attrs(o) isa HDF5.AttributeDict - attrs(f)["a"] = 1 - @test haskey(attrs(f), "a") - @test attrs(f)["a"] == 1 + attrs(o)["a"] = 1 + @test haskey(attrs(o), "a") + @test attrs(o)["a"] == 1 - attrs(f)["b"] = [2,3] - @test attrs(f)["b"] == [2,3] - @test haskey(attrs(f), "a") - @test length(attrs(f)) == 2 - @test sort(keys(attrs(f))) == ["a", "b"] + attrs(o)["b"] = [2,3] + @test attrs(o)["b"] == [2,3] + @test haskey(attrs(o), "a") + @test length(attrs(o)) == 2 + @test sort(keys(attrs(o))) == ["a", "b"] - @test !haskey(attrs(f), "c") + @test !haskey(attrs(o), "c") # overwrite: same type - attrs(f)["a"] = 4 - @test attrs(f)["a"] == 4 - @test get(attrs(f), "a", nothing) == 4 - @test length(attrs(f)) == 2 - @test sort(keys(attrs(f))) == ["a", "b"] + attrs(o)["a"] = 4 + @test attrs(o)["a"] == 4 + @test get(attrs(o), "a", nothing) == 4 + @test length(attrs(o)) == 2 + @test sort(keys(attrs(o))) == ["a", "b"] # overwrite: different size - attrs(f)["b"] = [4,5,6] - @test attrs(f)["b"] == [4,5,6] - @test length(attrs(f)) == 2 - @test sort(keys(attrs(f))) == ["a", "b"] + attrs(o)["b"] = [4,5,6] + @test attrs(o)["b"] == [4,5,6] + @test length(attrs(o)) == 2 + @test sort(keys(attrs(o))) == ["a", "b"] # overwrite: different type - attrs(f)["b"] = "a string" - @test attrs(f)["b"] == "a string" - @test length(attrs(f)) == 2 - @test sort(keys(attrs(f))) == ["a", "b"] + attrs(o)["b"] = "a string" + @test attrs(o)["b"] == "a string" + @test length(attrs(o)) == 2 + @test sort(keys(attrs(o))) == ["a", "b"] # delete a key - delete!(attrs(f), "a") - @test !haskey(attrs(f), "a") - @test length(attrs(f)) == 1 - @test sort(keys(attrs(f))) == ["b"] + delete!(attrs(o), "a") + @test !haskey(attrs(o), "a") + @test length(attrs(o)) == 1 + @test sort(keys(attrs(o))) == ["b"] + + @test_throws KeyError attrs(o)["a"] + @test isnothing(get(attrs(o), "a", nothing)) + +end + + +@testset "attrs interface" begin + filename = tempname() + f = h5open(filename, "w") + + try + # Test attrs on a HDF5.File + test_attrs(f) + + # Test attrs on a HDF5.Group + g = create_group(f, "group_foo") + test_attrs(g) + + # Test attrs on a HDF5.Dataset + d = create_dataset(g, "dataset_bar", Int, (32, 32)) + test_attrs(d) - @test_throws KeyError attrs(f)["a"] - @test isnothing(get(attrs(f), "a", nothing)) + # Test attrs on a HDF5.Datatype + t = commit_datatype(g, "datatype_int16", + HDF5.Datatype(HDF5.API.h5t_copy(HDF5.API.H5T_NATIVE_INT16)) + ) + test_attrs(t) + finally + close(f) + end end \ No newline at end of file