From 04a9ccfea394a8ec141ba3d1be27eb7ae292cd31 Mon Sep 17 00:00:00 2001 From: Justin Willmert Date: Wed, 23 Dec 2020 16:05:33 -0600 Subject: [PATCH 1/3] Grab keys via callback iteration rather than many absolute calls It appears there's significant overhead in directly obtaining the name of every sub-element using a loop of `h5?_get_name_by_idx` calls. For instance, a v7.3-formatted Matlab save file I have access to has deeply nested and large struct which are serialized into a huge number of datasets within a `#refs#` group. ```julia julia> Int(length(fid["#refs#"])) 36236 ``` Before this change, enumerating the keys takes almost a minute: ```julia julia> @time keys(fid["#refs#"]); 54.193914 seconds (72.47 k allocations: 3.594 MiB) ``` but with this change, that's reduced to a fraction of a second: ```julia julia> @time keys(fid["#refs#"]); 0.013353 seconds (36.24 k allocations: 1.383 MiB) ``` --- docs/src/api_bindings.md | 2 ++ gen/api_defs.jl | 2 ++ gen/bind_generator.jl | 2 ++ src/HDF5.jl | 16 ++++++++++++---- src/api.jl | 12 ++++++++++++ src/api_helpers.jl | 14 ++++++++++++++ src/api_types.jl | 8 ++++++++ 7 files changed, 52 insertions(+), 4 deletions(-) diff --git a/docs/src/api_bindings.md b/docs/src/api_bindings.md index 2a396fd48..1890df6f6 100644 --- a/docs/src/api_bindings.md +++ b/docs/src/api_bindings.md @@ -43,6 +43,7 @@ h5a_get_name(attr_id::hid_t, buf_size::Csize_t, buf::Ptr{UInt8}) h5a_get_name_by_idx(loc_id::hid_t, obj_name::Cstring, index_type::Cint, order::Cint, idx::hsize_t, name::Ptr{UInt8}, size::Csize_t, lapl_id::hid_t) h5a_get_space(attr_id::hid_t) h5a_get_type(attr_id::hid_t) +h5a_iterate(obj_id::hid_t, idx_type::Cint, order::Cint, n::Ptr{hsize_t}, op::Ptr{Cvoid}, op_data::Ptr{Cvoid}) h5a_open(obj_id::hid_t, pathname::Ptr{UInt8}, aapl_id::hid_t) h5a_read(attr_id::hid_t, mem_type_id::hid_t, buf::Ptr{Cvoid}) h5a_write(attr_hid::hid_t, mem_type_id::hid_t, buf::Ptr{Cvoid}) @@ -122,6 +123,7 @@ h5l_delete(obj_id::hid_t, pathname::Ptr{UInt8}, lapl_id::hid_t) h5l_exists(loc_id::hid_t, pathname::Ptr{UInt8}, lapl_id::hid_t) h5l_get_info(link_loc_id::hid_t, link_name::Ptr{UInt8}, link_buf::Ptr{H5L_info_t}, lapl_id::hid_t) h5l_get_name_by_idx(loc_id::hid_t, group_name::Ptr{UInt8}, index_field::Cint, order::Cint, n::hsize_t, name::Ptr{UInt8}, size::Csize_t, lapl_id::hid_t) +h5l_iterate(group_id::hid_t, idx_type::Cint, order::Cint, idx::Ptr{hsize_t}, op::Ptr{Cvoid}, op_data::Ptr{Cvoid}) ``` ## [`H5O`](https://portal.hdfgroup.org/display/HDF5/Objects) — Object Interface diff --git a/gen/api_defs.jl b/gen/api_defs.jl index 88a87b5e5..8ff5eac0c 100644 --- a/gen/api_defs.jl +++ b/gen/api_defs.jl @@ -44,6 +44,7 @@ @bind h5a_get_name_by_idx(loc_id::hid_t, obj_name::Cstring, index_type::Cint, order::Cint, idx::hsize_t, name::Ptr{UInt8}, size::Csize_t, lapl_id::hid_t)::Cssize_t "Error getting attribute name" @bind h5a_get_space(attr_id::hid_t)::hid_t "Error getting attribute dataspace" @bind h5a_get_type(attr_id::hid_t)::hid_t "Error getting attribute type" +@bind h5a_iterate(obj_id::hid_t, idx_type::Cint, order::Cint, n::Ptr{hsize_t}, op::Ptr{Cvoid}, op_data::Ptr{Cvoid})::herr_t error("Error iterating attributes in object ", h5i_get_name(obj_id)) @bind h5a_open(obj_id::hid_t, pathname::Ptr{UInt8}, aapl_id::hid_t)::hid_t error("Error opening attribute ", h5i_get_name(obj_id), "/", pathname) @bind h5a_read(attr_id::hid_t, mem_type_id::hid_t, buf::Ptr{Cvoid})::herr_t error("Error reading attribute ", h5a_get_name(attr_id)) @bind h5a_write(attr_hid::hid_t, mem_type_id::hid_t, buf::Ptr{Cvoid})::herr_t "Error writing attribute data" @@ -129,6 +130,7 @@ @bind h5l_exists(loc_id::hid_t, pathname::Ptr{UInt8}, lapl_id::hid_t)::htri_t error("Cannot determine whether ", pathname, " exists") @bind h5l_get_info(link_loc_id::hid_t, link_name::Ptr{UInt8}, link_buf::Ptr{H5L_info_t}, lapl_id::hid_t)::herr_t error("Error getting info for link ", link_name) @bind h5l_get_name_by_idx(loc_id::hid_t, group_name::Ptr{UInt8}, index_field::Cint, order::Cint, n::hsize_t, name::Ptr{UInt8}, size::Csize_t, lapl_id::hid_t)::Cssize_t "Error getting object name" +@bind h5l_iterate(group_id::hid_t, idx_type::Cint, order::Cint, idx::Ptr{hsize_t}, op::Ptr{Cvoid}, op_data::Ptr{Cvoid})::herr_t error("Error iterating through links in group ", h5i_get_name(group_id)) ### ### Object Interface diff --git a/gen/bind_generator.jl b/gen/bind_generator.jl index 9d30171b2..d383b35e8 100644 --- a/gen/bind_generator.jl +++ b/gen/bind_generator.jl @@ -6,12 +6,14 @@ const bind_exceptions = Dict{Symbol,Symbol}() # Version-numbered identifiers push!(bind_exceptions, :h5a_create => :H5Acreate2) +push!(bind_exceptions, :h5a_iterate => :H5Aiterate2) push!(bind_exceptions, :h5d_create => :H5Dcreate2) push!(bind_exceptions, :h5d_open => :H5Dopen2) push!(bind_exceptions, :h5e_get_auto => :H5Eget_auto2) push!(bind_exceptions, :h5e_set_auto => :H5Eset_auto2) push!(bind_exceptions, :h5g_create => :H5Gcreate2) push!(bind_exceptions, :h5g_open => :H5Gopen2) +push!(bind_exceptions, :h5l_iterate => :H5Literate1) push!(bind_exceptions, :h5o_get_info => :H5Oget_info1) push!(bind_exceptions, :h5p_get_filter_by_id => :H5Pget_filter_by_id2) push!(bind_exceptions, :h5r_dereference => :H5Rdereference2) diff --git a/src/HDF5.jl b/src/HDF5.jl index 2b79b6f49..e51e27b86 100644 --- a/src/HDF5.jl +++ b/src/HDF5.jl @@ -934,14 +934,22 @@ name(obj::Union{File,Group,Dataset,Datatype}) = h5i_get_name(checkvalid(obj)) name(attr::Attribute) = h5a_get_name(attr) function Base.keys(x::Union{Group,File}) checkvalid(x) - n = length(x) - return [h5l_get_name_by_idx(x, ".", H5_INDEX_NAME, H5_ITER_INC, i-1, H5P_DEFAULT) for i = 1:n] + children = sizehint!(String[], length(x)) + h5l_iterate(x, H5_INDEX_NAME, H5_ITER_INC) do group, name, info, data + push!(children, unsafe_string(name)) + return herr_t(0) + end + return children end function Base.keys(x::Attributes) checkvalid(x.parent) - n = length(x) - return [h5a_get_name_by_idx(x.parent, ".", H5_INDEX_NAME, H5_ITER_INC, i-1, H5P_DEFAULT) for i = 1:n] + children = sizehint!(String[], length(x)) + h5a_iterate(x.parent, H5_INDEX_NAME, H5_ITER_INC) do loc_id, attr_name, ainfo, data + push!(children, unsafe_string(attr_name)) + return herr_t(0) + end + return children end # iteration by objects diff --git a/src/api.jl b/src/api.jl index 194325bc4..b02b434e1 100644 --- a/src/api.jl +++ b/src/api.jl @@ -129,6 +129,12 @@ function h5a_get_type(attr_id) return var"#status#" end +function h5a_iterate(obj_id, idx_type, order, n, op, op_data) + var"#status#" = ccall((:H5Aiterate2, libhdf5), herr_t, (hid_t, Cint, Cint, Ptr{hsize_t}, Ptr{Cvoid}, Ptr{Cvoid}), obj_id, idx_type, order, n, op, op_data) + var"#status#" < 0 && error("Error iterating attributes in object ", h5i_get_name(obj_id)) + return nothing +end + function h5a_open(obj_id, pathname, aapl_id) var"#status#" = ccall((:H5Aopen, libhdf5), hid_t, (hid_t, Ptr{UInt8}, hid_t), obj_id, pathname, aapl_id) var"#status#" < 0 && error("Error opening attribute ", h5i_get_name(obj_id), "/", pathname) @@ -459,6 +465,12 @@ function h5l_get_name_by_idx(loc_id, group_name, index_field, order, n, name, si return var"#status#" end +function h5l_iterate(group_id, idx_type, order, idx, op, op_data) + var"#status#" = ccall((:H5Literate1, libhdf5), herr_t, (hid_t, Cint, Cint, Ptr{hsize_t}, Ptr{Cvoid}, Ptr{Cvoid}), group_id, idx_type, order, idx, op, op_data) + var"#status#" < 0 && error("Error iterating through links in group ", h5i_get_name(group_id)) + return nothing +end + function h5o_close(object_id) var"#status#" = ccall((:H5Oclose, libhdf5), herr_t, (hid_t,), object_id) var"#status#" < 0 && error("Error closing object") diff --git a/src/api_helpers.jl b/src/api_helpers.jl index 68b8e21d3..ae7e2f6a5 100644 --- a/src/api_helpers.jl +++ b/src/api_helpers.jl @@ -37,6 +37,13 @@ function h5a_get_name_by_idx(loc_id, obj_name, idx_type, order, idx, lapl_id) return String(buf) end +function h5a_iterate(f, obj_id, idx_type, order, idx = 0) + idxref = Ref{hsize_t}(idx) + fptr = @cfunction($f, herr_t, (hid_t, Ptr{Cchar}, Ptr{H5A_info_t}, Ptr{Cvoid})) + h5a_iterate(obj_id, idx_type, order, idxref, fptr, C_NULL) + return idxref[] +end + ### ### Dataset Interface ### @@ -133,6 +140,13 @@ function h5l_get_name_by_idx(loc_id, group_name, idx_type, order, idx, lapl_id) return String(buf) end +function h5l_iterate(f, group_id, idx_type, order, idx = 0) + idxref = Ref{hsize_t}(idx) + fptr = @cfunction($f, herr_t, (hid_t, Ptr{Cchar}, Ptr{H5L_info_t}, Ptr{Cvoid})) + h5l_iterate(group_id, idx_type, order, idxref, fptr, C_NULL) + return idxref[] +end + ### ### Object Interface ### diff --git a/src/api_types.jl b/src/api_types.jl index f35c21268..b55e3fa71 100644 --- a/src/api_types.jl +++ b/src/api_types.jl @@ -37,6 +37,14 @@ end const H5R_REF_T_NULL = H5R_ref_t(ntuple(_ -> 0x0, Val(64))) =# +# For attribute information +struct H5A_info_t + corder_valid::hbool_t + corder::UInt32 # typedef uint32_t H5O_msg_crt_idx_t + cset::Cint + data_size::hsize_t +end + # For group information struct H5G_info_t storage_type::Cint # enum H5G_storage_type_t From 6065167e4d616aed394c09fb5c35706d6f8561de Mon Sep 17 00:00:00 2001 From: Justin Willmert Date: Wed, 23 Dec 2020 19:07:00 -0600 Subject: [PATCH 2/3] Automatically handle versioned libhdf5 naming --- gen/api_defs.jl | 41 +++++++++++++++++++++-------------------- gen/bind_generator.jl | 33 ++++++++++----------------------- 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/gen/api_defs.jl b/gen/api_defs.jl index 8ff5eac0c..7f1b3cd1e 100644 --- a/gen/api_defs.jl +++ b/gen/api_defs.jl @@ -10,9 +10,10 @@ # # The C library names are automatically generated from the Julia function name by # uppercasing the `h5?` and removing the first underscore --- -# e.g. `h5d_close` -> `H5Dclose`. -# For versioned bindings (such as `H5Dopen2`), the mapping between names is given -# explicitly in the `bind_exceptions` Dict in `bind_generator.jl`. +# e.g. `h5d_close` -> `H5Dclose`. Versioned function names (such as +# `h5d_open2` -> `H5Dopen2`) have the trailing number removed for the Julia function +# definition. Other arbitrary mappings may be added by adding an entry to the +# `bind_exceptions` Dict in `bind_generator.jl`. ### ### HDF5 General library functions @@ -32,7 +33,7 @@ ### @bind h5a_close(id::hid_t)::herr_t "Error closing attribute" -@bind h5a_create(loc_id::hid_t, pathname::Ptr{UInt8}, type_id::hid_t, space_id::hid_t, acpl_id::hid_t, aapl_id::hid_t)::hid_t error("Error creating attribute ", h5a_get_name(loc_id), "/", pathname) +@bind h5a_create2(loc_id::hid_t, pathname::Ptr{UInt8}, type_id::hid_t, space_id::hid_t, acpl_id::hid_t, aapl_id::hid_t)::hid_t error("Error creating attribute ", h5a_get_name(loc_id), "/", pathname) @bind h5a_create_by_name(loc_id::hid_t, obj_name::Ptr{UInt8}, attr_name::Ptr{UInt8}, type_id::hid_t, space_id::hid_t, acpl_id::hid_t, aapl_id::hid_t, lapl_id::hid_t)::hid_t error("Error creating attribute ", attr_name, " for object ", obj_name) @bind h5a_delete(loc_id::hid_t, attr_name::Ptr{UInt8})::herr_t error("Error deleting attribute ", attr_name) @bind h5a_delete_by_idx(loc_id::hid_t, obj_name::Ptr{UInt8}, idx_type::Cint, order::Cint, n::hsize_t, lapl_id::hid_t)::herr_t error("Error deleting attribute ", n, " from object ", obj_name) @@ -44,7 +45,7 @@ @bind h5a_get_name_by_idx(loc_id::hid_t, obj_name::Cstring, index_type::Cint, order::Cint, idx::hsize_t, name::Ptr{UInt8}, size::Csize_t, lapl_id::hid_t)::Cssize_t "Error getting attribute name" @bind h5a_get_space(attr_id::hid_t)::hid_t "Error getting attribute dataspace" @bind h5a_get_type(attr_id::hid_t)::hid_t "Error getting attribute type" -@bind h5a_iterate(obj_id::hid_t, idx_type::Cint, order::Cint, n::Ptr{hsize_t}, op::Ptr{Cvoid}, op_data::Ptr{Cvoid})::herr_t error("Error iterating attributes in object ", h5i_get_name(obj_id)) +@bind h5a_iterate2(obj_id::hid_t, idx_type::Cint, order::Cint, n::Ptr{hsize_t}, op::Ptr{Cvoid}, op_data::Ptr{Cvoid})::herr_t error("Error iterating attributes in object ", h5i_get_name(obj_id)) @bind h5a_open(obj_id::hid_t, pathname::Ptr{UInt8}, aapl_id::hid_t)::hid_t error("Error opening attribute ", h5i_get_name(obj_id), "/", pathname) @bind h5a_read(attr_id::hid_t, mem_type_id::hid_t, buf::Ptr{Cvoid})::herr_t error("Error reading attribute ", h5a_get_name(attr_id)) @bind h5a_write(attr_hid::hid_t, mem_type_id::hid_t, buf::Ptr{Cvoid})::herr_t "Error writing attribute data" @@ -54,14 +55,14 @@ ### @bind h5d_close(dataset_id::hid_t)::herr_t "Error closing dataset" -@bind h5d_create(loc_id::hid_t, pathname::Ptr{UInt8}, dtype_id::hid_t, space_id::hid_t, lcpl_id::hid_t, dcpl_id::hid_t, dapl_id::hid_t)::hid_t error("Error creating dataset ", h5i_get_name(loc_id), "/", pathname) +@bind h5d_create2(loc_id::hid_t, pathname::Ptr{UInt8}, dtype_id::hid_t, space_id::hid_t, lcpl_id::hid_t, dcpl_id::hid_t, dapl_id::hid_t)::hid_t error("Error creating dataset ", h5i_get_name(loc_id), "/", pathname) @bind h5d_flush(dataset_id::hid_t)::herr_t "Error flushing dataset" @bind h5d_get_access_plist(dataset_id::hid_t)::hid_t "Error getting dataset access property list" @bind h5d_get_create_plist(dataset_id::hid_t)::hid_t "Error getting dataset create property list" @bind h5d_get_offset(dataset_id::hid_t)::haddr_t "Error getting offset" @bind h5d_get_space(dataset_id::hid_t)::hid_t "Error getting dataspace" @bind h5d_get_type(dataset_id::hid_t)::hid_t "Error getting dataspace type" -@bind h5d_open(loc_id::hid_t, pathname::Ptr{UInt8}, dapl_id::hid_t)::hid_t error("Error opening dataset ", h5i_get_name(loc_id), "/", pathname) +@bind h5d_open2(loc_id::hid_t, pathname::Ptr{UInt8}, dapl_id::hid_t)::hid_t error("Error opening dataset ", h5i_get_name(loc_id), "/", pathname) @bind h5d_read(dataset_id::hid_t, mem_type_id::hid_t, mem_space_id::hid_t, file_space_id::hid_t, xfer_plist_id::hid_t, buf::Ptr{Cvoid})::herr_t error("Error reading dataset ", h5i_get_name(dataset_id)) @bind h5d_refresh(dataset_id::hid_t)::herr_t "Error refreshing dataset" @bind h5d_set_extent(dataset_id::hid_t, new_dims::Ptr{hsize_t})::herr_t "Error extending dataset dimensions" @@ -73,8 +74,8 @@ ### Error Interface ### -@bind h5e_get_auto(estack_id::hid_t, func::Ref{Ptr{Cvoid}}, client_data::Ref{Ptr{Cvoid}})::herr_t "Error getting error reporting behavior" -@bind h5e_set_auto(estack_id::hid_t, func::Ptr{Cvoid}, client_data::Ptr{Cvoid})::herr_t "Error setting error reporting behavior" +@bind h5e_get_auto2(estack_id::hid_t, func::Ref{Ptr{Cvoid}}, client_data::Ref{Ptr{Cvoid}})::herr_t "Error getting error reporting behavior" +@bind h5e_set_auto2(estack_id::hid_t, func::Ptr{Cvoid}, client_data::Ptr{Cvoid})::herr_t "Error setting error reporting behavior" @bind h5e_get_current_stack()::hid_t "Unable to return current error stack" ### @@ -100,12 +101,12 @@ ### @bind h5g_close(group_id::hid_t)::herr_t "Error closing group" -@bind h5g_create(loc_id::hid_t, pathname::Ptr{UInt8}, lcpl_id::hid_t, gcpl_id::hid_t, gapl_id::hid_t)::hid_t error("Error creating group ", h5i_get_name(loc_id), "/", pathname) +@bind h5g_create2(loc_id::hid_t, pathname::Ptr{UInt8}, lcpl_id::hid_t, gcpl_id::hid_t, gapl_id::hid_t)::hid_t error("Error creating group ", h5i_get_name(loc_id), "/", pathname) @bind h5g_get_create_plist(group_id::hid_t)::hid_t "Error getting group create property list" @bind h5g_get_info(group_id::hid_t, buf::Ptr{H5G_info_t})::herr_t "Error getting group info" @bind h5g_get_num_objs(loc_id::hid_t, num_obj::Ptr{hsize_t})::hid_t "Error getting group length" @bind h5g_get_objname_by_idx(loc_id::hid_t, idx::hsize_t, pathname::Ptr{UInt8}, size::Csize_t)::Cssize_t error("Error getting group object name ", h5i_get_name(loc_id), "/", pathname) -@bind h5g_open(loc_id::hid_t, pathname::Ptr{UInt8}, gapl_id::hid_t)::hid_t error("Error opening group ", h5i_get_name(loc_id), "/", pathname) +@bind h5g_open2(loc_id::hid_t, pathname::Ptr{UInt8}, gapl_id::hid_t)::hid_t error("Error opening group ", h5i_get_name(loc_id), "/", pathname) ### ### Identifier Interface @@ -130,7 +131,7 @@ @bind h5l_exists(loc_id::hid_t, pathname::Ptr{UInt8}, lapl_id::hid_t)::htri_t error("Cannot determine whether ", pathname, " exists") @bind h5l_get_info(link_loc_id::hid_t, link_name::Ptr{UInt8}, link_buf::Ptr{H5L_info_t}, lapl_id::hid_t)::herr_t error("Error getting info for link ", link_name) @bind h5l_get_name_by_idx(loc_id::hid_t, group_name::Ptr{UInt8}, index_field::Cint, order::Cint, n::hsize_t, name::Ptr{UInt8}, size::Csize_t, lapl_id::hid_t)::Cssize_t "Error getting object name" -@bind h5l_iterate(group_id::hid_t, idx_type::Cint, order::Cint, idx::Ptr{hsize_t}, op::Ptr{Cvoid}, op_data::Ptr{Cvoid})::herr_t error("Error iterating through links in group ", h5i_get_name(group_id)) +@bind h5l_iterate1(group_id::hid_t, idx_type::Cint, order::Cint, idx::Ptr{hsize_t}, op::Ptr{Cvoid}, op_data::Ptr{Cvoid})::herr_t error("Error iterating through links in group ", h5i_get_name(group_id)) ### ### Object Interface @@ -138,7 +139,7 @@ @bind h5o_close(object_id::hid_t)::herr_t "Error closing object" @bind h5o_copy(src_loc_id::hid_t, src_name::Ptr{UInt8}, dst_loc_id::hid_t, dst_name::Ptr{UInt8}, ocpypl_id::hid_t, lcpl_id::hid_t)::herr_t error("Error copying object ", h5i_get_name(src_loc_id), "/", src_name, " to ", h5i_get_name(dst_loc_id), "/", dst_name) -@bind h5o_get_info(object_id::hid_t, buf::Ptr{H5O_info_t})::herr_t "Error getting object info" +@bind h5o_get_info1(object_id::hid_t, buf::Ptr{H5O_info_t})::herr_t "Error getting object info" @bind h5o_open(loc_id::hid_t, pathname::Ptr{UInt8}, lapl_id::hid_t)::hid_t error("Error opening object ", h5i_get_name(loc_id), "/", pathname) @bind h5o_open_by_addr(loc_id::hid_t, addr::haddr_t)::hid_t error("Error opening object by address") @bind h5o_open_by_idx(loc_id::hid_t, group_name::Ptr{UInt8}, index_type::Cint, order::Cint, n::hsize_t, lapl_id::hid_t)::hid_t error("Error opening object of index ", n) @@ -160,7 +161,7 @@ @bind h5p_get_fapl_mpio32(fapl_id::hid_t, comm::Ptr{Hmpih32}, info::Ptr{Hmpih32})::herr_t "Error getting MPIO properties" @bind h5p_get_fapl_mpio64(fapl_id::hid_t, comm::Ptr{Hmpih64}, info::Ptr{Hmpih64})::herr_t "Error getting MPIO properties" @bind h5p_get_fclose_degree(fapl_id::hid_t, fc_degree::Ref{Cint})::herr_t "Error getting close degree" -@bind h5p_get_filter_by_id(plist_id::hid_t, filter_id::H5Z_filter_t, flags::Ref{Cuint}, cd_nelmts::Ref{Csize_t}, cd_values::Ptr{Cuint}, namelen::Csize_t, name::Ptr{UInt8}, filter_config::Ptr{Cuint})::herr_t "Error getting filter ID" +@bind h5p_get_filter_by_id2(plist_id::hid_t, filter_id::H5Z_filter_t, flags::Ref{Cuint}, cd_nelmts::Ref{Csize_t}, cd_values::Ptr{Cuint}, namelen::Csize_t, name::Ptr{UInt8}, filter_config::Ptr{Cuint})::herr_t "Error getting filter ID" @bind h5p_get_layout(plist_id::hid_t)::Cint error("Error getting layout") @bind h5p_get_libver_bounds(fapl_id::hid_t, low::Ref{Cint}, high::Ref{Cint})::herr_t "Error getting library version bounds" @bind h5p_get_local_heap_size_hint(plist_id::hid_t, size_hint::Ref{Csize_t})::herr_t "Error getting local heap size hint" @@ -193,8 +194,8 @@ ### @bind h5r_create(ref::Ptr{Cvoid}, loc_id::hid_t, pathname::Ptr{UInt8}, ref_type::Cint, space_id::hid_t)::herr_t error("Error creating reference to object ", h5i_get_name(loc_id), "/", pathname) -@bind h5r_dereference(obj_id::hid_t, oapl_id::hid_t, ref_type::Cint, ref::Ptr{Cvoid})::hid_t "Error dereferencing object" -@bind h5r_get_obj_type(loc_id::hid_t, ref_type::Cint, ref::Ptr{Cvoid}, obj_type::Ptr{Cint})::herr_t "Error getting object type" +@bind h5r_dereference2(obj_id::hid_t, oapl_id::hid_t, ref_type::Cint, ref::Ptr{Cvoid})::hid_t "Error dereferencing object" +@bind h5r_get_obj_type2(loc_id::hid_t, ref_type::Cint, ref::Ptr{Cvoid}, obj_type::Ptr{Cint})::herr_t "Error getting object type" @bind h5r_get_region(loc_id::hid_t, ref_type::Cint, ref::Ptr{Cvoid})::hid_t "Error getting region from reference" ### @@ -222,15 +223,15 @@ ### Datatype Interface ### -@bind h5t_array_create(basetype_id::hid_t, ndims::Cuint, sz::Ptr{hsize_t})::hid_t error("Error creating H5T_ARRAY of id ", basetype_id, " and size ", sz) +@bind h5t_array_create2(basetype_id::hid_t, ndims::Cuint, sz::Ptr{hsize_t})::hid_t error("Error creating H5T_ARRAY of id ", basetype_id, " and size ", sz) @bind h5t_close(dtype_id::hid_t)::herr_t "Error closing datatype" @bind h5t_committed(dtype_id::hid_t)::htri_t error("Error determining whether datatype is committed") -@bind h5t_commit(loc_id::hid_t, name::Ptr{UInt8}, dtype_id::hid_t, lcpl_id::hid_t, tcpl_id::hid_t, tapl_id::hid_t)::herr_t "Error committing type" +@bind h5t_commit2(loc_id::hid_t, name::Ptr{UInt8}, dtype_id::hid_t, lcpl_id::hid_t, tcpl_id::hid_t, tapl_id::hid_t)::herr_t "Error committing type" @bind h5t_copy(dtype_id::hid_t)::hid_t "Error copying datatype" @bind h5t_create(class_id::Cint, sz::Csize_t)::hid_t error("Error creating datatype of id ", class_id) @bind h5t_enum_insert(dtype_id::hid_t, name::Cstring, value::Ptr{Cvoid})::herr_t error("Error adding ", name, " to enum datatype") @bind h5t_equal(dtype_id1::hid_t, dtype_id2::hid_t)::htri_t "Error checking datatype equality" -@bind h5t_get_array_dims(dtype_id::hid_t, dims::Ptr{hsize_t})::Cint "Error getting dimensions of array" +@bind h5t_get_array_dims2(dtype_id::hid_t, dims::Ptr{hsize_t})::Cint "Error getting dimensions of array" @bind h5t_get_array_ndims(dtype_id::hid_t)::Cint "Error getting ndims of array" @bind h5t_get_class(dtype_id::hid_t)::Cint "Error getting class" @bind h5t_get_cset(dtype_id::hid_t)::Cint "Error getting character set encoding" @@ -249,7 +250,7 @@ @bind h5t_insert(dtype_id::hid_t, fieldname::Ptr{UInt8}, offset::Csize_t, field_id::hid_t)::herr_t error("Error adding field ", fieldname, " to compound datatype") @bind h5t_is_variable_str(type_id::hid_t)::htri_t "Error determining whether string is of variable length" @bind h5t_lock(type_id::hid_t)::herr_t "Error locking type" -@bind h5t_open(loc_id::hid_t, name::Ptr{UInt8}, tapl_id::hid_t)::hid_t error("Error opening type ", h5i_get_name(loc_id), "/", name) +@bind h5t_open2(loc_id::hid_t, name::Ptr{UInt8}, tapl_id::hid_t)::hid_t error("Error opening type ", h5i_get_name(loc_id), "/", name) @bind h5t_set_cset(dtype_id::hid_t, cset::Cint)::herr_t "Error setting character set in datatype" @bind h5t_set_ebias(dtype_id::hid_t, ebias::Csize_t)::herr_t "Error setting datatype floating point exponent bias" @bind h5t_set_fields(dtype_id::hid_t, spos::Csize_t, epos::Csize_t, esize::Csize_t, mpos::Csize_t, msize::Csize_t)::herr_t "Error setting datatype floating point bit positions" diff --git a/gen/bind_generator.jl b/gen/bind_generator.jl index d383b35e8..d1f514044 100644 --- a/gen/bind_generator.jl +++ b/gen/bind_generator.jl @@ -4,25 +4,6 @@ using Base.Meta: isexpr, quot # of the translations explicitly. const bind_exceptions = Dict{Symbol,Symbol}() -# Version-numbered identifiers -push!(bind_exceptions, :h5a_create => :H5Acreate2) -push!(bind_exceptions, :h5a_iterate => :H5Aiterate2) -push!(bind_exceptions, :h5d_create => :H5Dcreate2) -push!(bind_exceptions, :h5d_open => :H5Dopen2) -push!(bind_exceptions, :h5e_get_auto => :H5Eget_auto2) -push!(bind_exceptions, :h5e_set_auto => :H5Eset_auto2) -push!(bind_exceptions, :h5g_create => :H5Gcreate2) -push!(bind_exceptions, :h5g_open => :H5Gopen2) -push!(bind_exceptions, :h5l_iterate => :H5Literate1) -push!(bind_exceptions, :h5o_get_info => :H5Oget_info1) -push!(bind_exceptions, :h5p_get_filter_by_id => :H5Pget_filter_by_id2) -push!(bind_exceptions, :h5r_dereference => :H5Rdereference2) -push!(bind_exceptions, :h5r_get_obj_type => :H5Rget_obj_type2) -push!(bind_exceptions, :h5t_array_create => :H5Tarray_create2) -push!(bind_exceptions, :h5t_commit => :H5Tcommit2) -push!(bind_exceptions, :h5t_get_array_dims => :H5Tget_array_dims2) -push!(bind_exceptions, :h5t_open => :H5Topen2) - # Distinguishes 32-bit vs 64-bit handle arguments push!(bind_exceptions, :h5p_get_fapl_mpio32 => :H5Pget_fapl_mpio) push!(bind_exceptions, :h5p_get_fapl_mpio64 => :H5Pget_fapl_mpio) @@ -39,10 +20,11 @@ const bound_api = Dict{String,Vector{String}}() A binding generator for translating `@ccall`-like declarations of HDF5 library functions to error-checked `ccall` expressions. -The provided function name is used to define the Julia function. The corresponding C -function name is auto-generated by uppercasing the first few letters (up to the first `_`), -and the first `_` is removed. Explicit name mappings can be made by inserting a -`:jlname => :h5name` pair into the `bind_exceptions` dictionary. +The provided function name is used to define the Julia function with any trailing version +number removed (such as `h5t_open2` -> `h5t_open`). The corresponding C function name is +auto-generated by uppercasing the first few letters (up to the first `_`), the first `_` is +removed. Explicit name mappings can be made by inserting a `:jlname => :h5name` pair into +the `bind_exceptions` dictionary. The optional `ErrorStringOrExpression` can be either a string literal or an arbitrary expression. If not provided, no error check is done. If it is a `String`, the string is used @@ -114,6 +96,11 @@ macro bind(sig::Expr, err::Union{String,Expr,Nothing} = nothing) else # turn e.g. h5f_close into H5Fclose cfuncname = Symbol(uppercase(prefix), rest) + # Remove the version number if present (excluding match to literal "hdf5" suffix) + if occursin(r"\d(? Date: Wed, 23 Dec 2020 20:03:22 -0600 Subject: [PATCH 3/3] Support libhdf5 version-specific bindings --- gen/api_defs.jl | 7 +++++-- gen/bind_generator.jl | 48 ++++++++++++++++++++++++++++++++++++++++--- gen/gen_wrappers.jl | 3 +++ src/api.jl | 39 ++++++++++++++++++++++++++--------- 4 files changed, 82 insertions(+), 15 deletions(-) diff --git a/gen/api_defs.jl b/gen/api_defs.jl index 7f1b3cd1e..701db3b94 100644 --- a/gen/api_defs.jl +++ b/gen/api_defs.jl @@ -131,7 +131,10 @@ @bind h5l_exists(loc_id::hid_t, pathname::Ptr{UInt8}, lapl_id::hid_t)::htri_t error("Cannot determine whether ", pathname, " exists") @bind h5l_get_info(link_loc_id::hid_t, link_name::Ptr{UInt8}, link_buf::Ptr{H5L_info_t}, lapl_id::hid_t)::herr_t error("Error getting info for link ", link_name) @bind h5l_get_name_by_idx(loc_id::hid_t, group_name::Ptr{UInt8}, index_field::Cint, order::Cint, n::hsize_t, name::Ptr{UInt8}, size::Csize_t, lapl_id::hid_t)::Cssize_t "Error getting object name" -@bind h5l_iterate1(group_id::hid_t, idx_type::Cint, order::Cint, idx::Ptr{hsize_t}, op::Ptr{Cvoid}, op_data::Ptr{Cvoid})::herr_t error("Error iterating through links in group ", h5i_get_name(group_id)) +# libhdf5 v1.10 provides the name H5Literate +# libhdf5 v1.12 provides the same under H5Literate1, and a newer interface on H5Literate2 +@bind h5l_iterate(group_id::hid_t, idx_type::Cint, order::Cint, idx::Ptr{hsize_t}, op::Ptr{Cvoid}, op_data::Ptr{Cvoid})::herr_t error("Error iterating through links in group ", h5i_get_name(group_id)) (nothing, v"1.12") +@bind h5l_iterate1(group_id::hid_t, idx_type::Cint, order::Cint, idx::Ptr{hsize_t}, op::Ptr{Cvoid}, op_data::Ptr{Cvoid})::herr_t error("Error iterating through links in group ", h5i_get_name(group_id)) (v"1.12", nothing) ### ### Object Interface @@ -203,7 +206,7 @@ ### @bind h5s_close(space_id::hid_t)::herr_t "Error closing dataspace" -@bind h5s_combine_select(space1_id::hid_t, op::Cint, space2_id::hid_t)::hid_t "Error combining dataspaces" +@bind h5s_combine_select(space1_id::hid_t, op::Cint, space2_id::hid_t)::hid_t "Error combining dataspaces" (v"1.10.7", nothing) @bind h5s_copy(space_id::hid_t)::hid_t "Error copying dataspace" @bind h5s_create(class::Cint)::hid_t "Error creating dataspace" @bind h5s_create_simple(rank::Cint, current_dims::Ptr{hsize_t}, maximum_dims::Ptr{hsize_t})::hid_t "Error creating simple dataspace" diff --git a/gen/bind_generator.jl b/gen/bind_generator.jl index d1f514044..6c14a4b4a 100644 --- a/gen/bind_generator.jl +++ b/gen/bind_generator.jl @@ -10,6 +10,20 @@ push!(bind_exceptions, :h5p_get_fapl_mpio64 => :H5Pget_fapl_mpio) push!(bind_exceptions, :h5p_set_fapl_mpio32 => :H5Pset_fapl_mpio) push!(bind_exceptions, :h5p_set_fapl_mpio64 => :H5Pset_fapl_mpio) +# An expression which is injected at the beginning of the API defitions to aid in doing +# (pre)compile-time conditional compilation based on the libhdf5 version. +_libhdf5_build_ver_expr = quote + _libhdf5_build_ver = let + majnum, minnum, relnum = Ref{Cuint}(), Ref{Cuint}(), Ref{Cuint}() + r = ccall((:H5get_libversion, libhdf5), herr_t, + (Ref{Cuint}, Ref{Cuint}, Ref{Cuint}), + majnum, minnum, relnum) + r < 0 && error("Error getting HDF5 library version") + VersionNumber(majnum[], minnum[], relnum[]) + end +end + + # We'll also use this processing pass to automatically generate documentation that simply # lists all of the bound API functions. const bound_api = Dict{String,Vector{String}}() @@ -64,7 +78,35 @@ It is assumed that the HDF library names are given in global constants named `li and `libhdf5_hl`. The former is used for all `ccall`s, except if the C library name begins with "H5DO" or "H5TB" then the latter library is used. """ -macro bind(sig::Expr, err::Union{String,Expr,Nothing} = nothing) +macro bind(sig::Expr, err::Union{String,Expr,Nothing} = nothing, + vers::Union{Expr,Nothing} = nothing) + expr = _bind(__module__, __source__, sig, err) + isnothing(vers) && return esc(expr) + + isexpr(vers, :tuple) || error("Expected 2-tuple of version bounds, got ", vers) + length(vers.args) == 2 || error("Expected 2-tuple of version bounds, got ", vers) + lb = vers.args[1] + ub = vers.args[2] + + if lb !== :nothing && !(isexpr(lb, :macrocall) && lb.args[1] == Symbol("@v_str")) + error("Lower version bound must be `nothing` or version number literal, got ", lb) + end + if ub !== :nothing && !(isexpr(ub, :macrocall) && ub.args[1] == Symbol("@v_str")) + error("Upper version bound must be `nothing` or version number literal, got ", ub) + end + + if lb === :nothing && ub !== :nothing + conditional = :(_libhdf5_build_ver < $(ub)) + elseif lb !== :nothing && ub === :nothing + conditional = :($(lb) ≤ _libhdf5_build_ver) + else + conditional = :($(lb) ≤ _libhdf5_build_ver < $(ub)) + end + conditional = Expr(:if, conditional, Expr(:block, expr)) + return esc(Expr(:macrocall, Symbol("@static"), nothing, conditional)) +end + +function _bind(__module__, __source__, sig::Expr, err::Union{String,Expr,Nothing}) sig.head === :(::) || error("return type required on function signature") # Pull apart return-type and rest of function declaration @@ -105,7 +147,7 @@ macro bind(sig::Expr, err::Union{String,Expr,Nothing} = nothing) # Store the function prototype in HDF5-module specific lists: funclist = get!(bound_api, uppercase(prefix), Vector{String}(undef, 0)) - push!(funclist, string(funcsig)) + string(funcsig) in funclist || push!(funclist, string(funcsig)) # Determine the underlying C library to call lib = startswith(string(cfuncname), r"H5(DO|DS|LT|TB)") ? :libhdf5_hl : :libhdf5 @@ -159,5 +201,5 @@ macro bind(sig::Expr, err::Union{String,Expr,Nothing} = nothing) end push!(jlfuncbody.args, returnexpr) - return esc(Expr(:function, jlfuncsig, jlfuncbody)) + return Expr(:function, jlfuncsig, jlfuncbody) end diff --git a/gen/gen_wrappers.jl b/gen/gen_wrappers.jl index 0b7306d3e..2ba1d524d 100644 --- a/gen/gen_wrappers.jl +++ b/gen/gen_wrappers.jl @@ -4,6 +4,9 @@ include(joinpath(@__DIR__, "bind_generator.jl")) defs = read(joinpath(@__DIR__, "api_defs.jl"), String) # Have Julia expand/run the @bind macro to generate expressions for all of the functions exprs = Base.include_string(@__MODULE__, "@macroexpand1 begin\n" * defs * "\nend", "api_defs.jl") +# Insert the conditional version helper expression +prepend!(exprs.args, _libhdf5_build_ver_expr.args) + # Definitions which are not automatically generated, but should still be documented as # part of the raw low-level API: append!(bound_api["H5P"], diff --git a/src/api.jl b/src/api.jl index b02b434e1..9703a57ae 100644 --- a/src/api.jl +++ b/src/api.jl @@ -3,6 +3,13 @@ # To add new bindings, define the binding in `gen/api_defs.jl`, re-run # `gen/gen_wrappers.jl`, and commit the updated `src/api.jl`. +_libhdf5_build_ver = let + (majnum, minnum, relnum) = (Ref{Cuint}(), Ref{Cuint}(), Ref{Cuint}()) + r = ccall((:H5get_libversion, libhdf5), herr_t, (Ref{Cuint}, Ref{Cuint}, Ref{Cuint}), majnum, minnum, relnum) + r < 0 && error("Error getting HDF5 library version") + VersionNumber(majnum[], minnum[], relnum[]) + end + function h5_close() var"#status#" = ccall((:H5close, libhdf5), herr_t, ()) var"#status#" < 0 && error("Error closing the HDF5 resources") @@ -465,11 +472,21 @@ function h5l_get_name_by_idx(loc_id, group_name, index_field, order, n, name, si return var"#status#" end -function h5l_iterate(group_id, idx_type, order, idx, op, op_data) - var"#status#" = ccall((:H5Literate1, libhdf5), herr_t, (hid_t, Cint, Cint, Ptr{hsize_t}, Ptr{Cvoid}, Ptr{Cvoid}), group_id, idx_type, order, idx, op, op_data) - var"#status#" < 0 && error("Error iterating through links in group ", h5i_get_name(group_id)) - return nothing -end +@static if _libhdf5_build_ver < v"1.12" + function h5l_iterate(group_id, idx_type, order, idx, op, op_data) + var"#status#" = ccall((:H5Literate, libhdf5), herr_t, (hid_t, Cint, Cint, Ptr{hsize_t}, Ptr{Cvoid}, Ptr{Cvoid}), group_id, idx_type, order, idx, op, op_data) + var"#status#" < 0 && error("Error iterating through links in group ", h5i_get_name(group_id)) + return nothing + end + end + +@static if v"1.12" ≤ _libhdf5_build_ver + function h5l_iterate(group_id, idx_type, order, idx, op, op_data) + var"#status#" = ccall((:H5Literate1, libhdf5), herr_t, (hid_t, Cint, Cint, Ptr{hsize_t}, Ptr{Cvoid}, Ptr{Cvoid}), group_id, idx_type, order, idx, op, op_data) + var"#status#" < 0 && error("Error iterating through links in group ", h5i_get_name(group_id)) + return nothing + end + end function h5o_close(object_id) var"#status#" = ccall((:H5Oclose, libhdf5), herr_t, (hid_t,), object_id) @@ -776,11 +793,13 @@ function h5s_close(space_id) return nothing end -function h5s_combine_select(space1_id, op, space2_id) - var"#status#" = ccall((:H5Scombine_select, libhdf5), hid_t, (hid_t, Cint, hid_t), space1_id, op, space2_id) - var"#status#" < 0 && error("Error combining dataspaces") - return var"#status#" -end +@static if v"1.10.7" ≤ _libhdf5_build_ver + function h5s_combine_select(space1_id, op, space2_id) + var"#status#" = ccall((:H5Scombine_select, libhdf5), hid_t, (hid_t, Cint, hid_t), space1_id, op, space2_id) + var"#status#" < 0 && error("Error combining dataspaces") + return var"#status#" + end + end function h5s_copy(space_id) var"#status#" = ccall((:H5Scopy, libhdf5), hid_t, (hid_t,), space_id)