From 9abd3abcd79fc012eb4ed8597bb5e05fcf1d52cd Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Tue, 27 Sep 2022 22:53:32 +0200 Subject: [PATCH 01/25] Add write_compound function and tests --- src/HDF5.jl | 1 + src/datasets.jl | 56 ++++++++++++++++++++++++++++++++++++++++++++++++ test/compound.jl | 34 +++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+) diff --git a/src/HDF5.jl b/src/HDF5.jl index ced25e703..6fa339578 100644 --- a/src/HDF5.jl +++ b/src/HDF5.jl @@ -28,6 +28,7 @@ export @read, open_dataset, read_dataset, write_dataset, + write_compound, create_group, open_group, copy_object, diff --git a/src/datasets.jl b/src/datasets.jl index d1b1b773c..b24cd4384 100644 --- a/src/datasets.jl +++ b/src/datasets.jl @@ -482,6 +482,62 @@ write_dataset( xfer::DatasetTransferProperties=dataset.xfer ) = nothing +""" + function write_compound(f, name, data::AbstractArray{T}) where T + +Writes an array of structs with primitive field types to an HDF5Compound dataset. + +Examples +======== + +``` +julia> struct Foo + x::Int32 + y::Float64 + end + +julia> foos = [[Foo(1, 2) Foo(3, 4) Foo(5, 6)]; [Foo(7, 8) Foo(9, 10) Foo(11, 12)]] +2×3 Matrix{Foo}: + Foo(1, 2.0) Foo(3, 4.0) Foo(5, 6.0) + Foo(7, 8.0) Foo(9, 10.0) Foo(11, 12.0) + +julia> h5open("foo.h5", "w") do h5f + write_compound(h5f, "the/foo", ts) + end + +julia> thefoo = h5open("foo.h5", "r") do file + read(file, "the/foo") + end +2×3 Matrix{NamedTuple{(:x, :y), Tuple{Int32, Float64}}}: + (x = 1, y = 2.0) (x = 3, y = 4.0) (x = 5, y = 6.0) + (x = 7, y = 8.0) (x = 9, y = 10.0) (x = 11, y = 12.0) +``` + +The data can be reinterpreted to the original data type using `reinterpret()`: + +``` +julia> reinterpret(Foo, thefoo) +2×3 reinterpret(Foo, ::Matrix{NamedTuple{(:x, :y), Tuple{Int32, Float64}}}): + Foo(1, 2.0) Foo(3, 4.0) Foo(5, 6.0) + Foo(7, 8.0) Foo(9, 10.0) Foo(11, 12.0) +``` + +""" +function write_compound(f, name, data::AbstractArray{T}) where T + dtype = HDF5.API.h5t_create(HDF5.API.H5T_COMPOUND, sizeof(T)) + for (idx, fn) ∈ enumerate(fieldnames(T)) + HDF5.API.h5t_insert( + dtype, + fn, + fieldoffset(T, idx), + datatype(fieldtype(T, idx)) + ) + end + dt = HDF5.Datatype(dtype) + dset = create_dataset(f, name, dt, dataspace(data)) + write_dataset(dset, dt, data) +end + """ get_datasets(file::HDF5.File) -> datasets::Vector{HDF5.Dataset} diff --git a/test/compound.jl b/test/compound.jl index d0e76be9d..143b2f6a0 100644 --- a/test/compound.jl +++ b/test/compound.jl @@ -146,3 +146,37 @@ end @test HDF5.do_reclaim(TTTT) == false @test HDF5.do_normalize(TTTT) == true end + + +struct Bar + a::Int32 + b::Float64 + c::Bool +end + +@testset "write_compound" begin + bars = [ + [Bar(1, 1.1, true) Bar(2, 2.1, false) Bar(3, 3.1, true)]; + [Bar(4, 4.1, false) Bar(5, 5.1, true) Bar(6, 6.1, false)]; + ] + + fn = tempname() + h5open(fn, "w") do h5f + write_compound(h5f, "the/bar", bars) + end + + thebars = h5open(fn, "r") do h5f + read(h5f, "the/bar") + end + + @test (2, 3) == size(thebars) + @test thebars[1, 2].b ≈ 2.1 + @test thebars[2, 1].a == 4 + @test thebars[1, 3].c + + thebars_r = reinterpret(Bar, thebars) + @test (2, 3) == size(thebars_r) + @test thebars_r[1, 2].b ≈ 2.1 + @test thebars_r[2, 1].a == 4 + @test thebars_r[1, 3].c +end From aa9c9427073b337316a3d44fcf444a848694824b Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Tue, 27 Sep 2022 23:06:13 +0200 Subject: [PATCH 02/25] Remove HDF5 namespace --- src/datasets.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/datasets.jl b/src/datasets.jl index b24cd4384..088ca9019 100644 --- a/src/datasets.jl +++ b/src/datasets.jl @@ -524,16 +524,16 @@ julia> reinterpret(Foo, thefoo) """ function write_compound(f, name, data::AbstractArray{T}) where T - dtype = HDF5.API.h5t_create(HDF5.API.H5T_COMPOUND, sizeof(T)) + dtype = API.h5t_create(API.H5T_COMPOUND, sizeof(T)) for (idx, fn) ∈ enumerate(fieldnames(T)) - HDF5.API.h5t_insert( + API.h5t_insert( dtype, fn, fieldoffset(T, idx), datatype(fieldtype(T, idx)) ) end - dt = HDF5.Datatype(dtype) + dt = Datatype(dtype) dset = create_dataset(f, name, dt, dataspace(data)) write_dataset(dset, dt, data) end From e216f36024410b8fd9a32073a656572d610141b6 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Wed, 28 Sep 2022 09:04:43 +0200 Subject: [PATCH 03/25] Fix code formatting --- src/datasets.jl | 11 +++-------- test/compound.jl | 5 ++--- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/datasets.jl b/src/datasets.jl index 088ca9019..f426e6de4 100644 --- a/src/datasets.jl +++ b/src/datasets.jl @@ -523,15 +523,10 @@ julia> reinterpret(Foo, thefoo) ``` """ -function write_compound(f, name, data::AbstractArray{T}) where T +function write_compound(f, name, data::AbstractArray{T}) where {T} dtype = API.h5t_create(API.H5T_COMPOUND, sizeof(T)) - for (idx, fn) ∈ enumerate(fieldnames(T)) - API.h5t_insert( - dtype, - fn, - fieldoffset(T, idx), - datatype(fieldtype(T, idx)) - ) + for (idx, fn) in enumerate(fieldnames(T)) + API.h5t_insert(dtype, fn, fieldoffset(T, idx), datatype(fieldtype(T, idx))) end dt = Datatype(dtype) dset = create_dataset(f, name, dt, dataspace(data)) diff --git a/test/compound.jl b/test/compound.jl index 143b2f6a0..f95fe4197 100644 --- a/test/compound.jl +++ b/test/compound.jl @@ -147,7 +147,6 @@ end @test HDF5.do_normalize(TTTT) == true end - struct Bar a::Int32 b::Float64 @@ -156,8 +155,8 @@ end @testset "write_compound" begin bars = [ - [Bar(1, 1.1, true) Bar(2, 2.1, false) Bar(3, 3.1, true)]; - [Bar(4, 4.1, false) Bar(5, 5.1, true) Bar(6, 6.1, false)]; + [Bar(1, 1.1, true) Bar(2, 2.1, false) Bar(3, 3.1, true)] + [Bar(4, 4.1, false) Bar(5, 5.1, true) Bar(6, 6.1, false)] ] fn = tempname() From 8ec5d5b703e084151dad9004613cb00244133106 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Wed, 5 Oct 2022 12:04:40 +0200 Subject: [PATCH 04/25] Add more type annotations --- src/datasets.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/datasets.jl b/src/datasets.jl index f426e6de4..e05b41dbe 100644 --- a/src/datasets.jl +++ b/src/datasets.jl @@ -483,7 +483,7 @@ write_dataset( ) = nothing """ - function write_compound(f, name, data::AbstractArray{T}) where T + function write_compound(f::AbstractString, name::AbstractString, data::AbstractArray{T}) where T Writes an array of structs with primitive field types to an HDF5Compound dataset. @@ -523,7 +523,7 @@ julia> reinterpret(Foo, thefoo) ``` """ -function write_compound(f, name, data::AbstractArray{T}) where {T} +function write_compound(f::AbstractString, name::AbstractString, data::AbstractArray{T}) where {T} dtype = API.h5t_create(API.H5T_COMPOUND, sizeof(T)) for (idx, fn) in enumerate(fieldnames(T)) API.h5t_insert(dtype, fn, fieldoffset(T, idx), datatype(fieldtype(T, idx))) From a49d33da9f4a2e268f39b7eb2d7d1e853b256502 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Wed, 5 Oct 2022 12:05:41 +0200 Subject: [PATCH 05/25] Julia formatter --- src/datasets.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/datasets.jl b/src/datasets.jl index e05b41dbe..9e7c7cc1e 100644 --- a/src/datasets.jl +++ b/src/datasets.jl @@ -523,7 +523,9 @@ julia> reinterpret(Foo, thefoo) ``` """ -function write_compound(f::AbstractString, name::AbstractString, data::AbstractArray{T}) where {T} +function write_compound( + f::AbstractString, name::AbstractString, data::AbstractArray{T} +) where {T} dtype = API.h5t_create(API.H5T_COMPOUND, sizeof(T)) for (idx, fn) in enumerate(fieldnames(T)) API.h5t_insert(dtype, fn, fieldoffset(T, idx), datatype(fieldtype(T, idx))) From 63303733224e57462e3c5418b139312c2c898da2 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Mon, 10 Oct 2022 13:45:02 +0200 Subject: [PATCH 06/25] Rename write_compound --- src/datasets.jl | 11 ++++++++--- test/compound.jl | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/datasets.jl b/src/datasets.jl index 9e7c7cc1e..c7eafeb23 100644 --- a/src/datasets.jl +++ b/src/datasets.jl @@ -483,7 +483,7 @@ write_dataset( ) = nothing """ - function write_compound(f::AbstractString, name::AbstractString, data::AbstractArray{T}) where T + function write_compound_dataset(f:Union{File, Group}, name::AbstractString, data::AbstractArray{T}) where T Writes an array of structs with primitive field types to an HDF5Compound dataset. @@ -523,8 +523,8 @@ julia> reinterpret(Foo, thefoo) ``` """ -function write_compound( - f::AbstractString, name::AbstractString, data::AbstractArray{T} +function write_compound_dataset( + f::Union{File, Group}, name::AbstractString, data::AbstractArray{T} ) where {T} dtype = API.h5t_create(API.H5T_COMPOUND, sizeof(T)) for (idx, fn) in enumerate(fieldnames(T)) @@ -534,6 +534,11 @@ function write_compound( dset = create_dataset(f, name, dt, dataspace(data)) write_dataset(dset, dt, data) end +function write_compound_dataset(f::AbstractString, name::AbstractString, data::AbstractArray{T}) where {T} + h5open(f, "w") do h5f + write_compound_dataset(h5f, name, data) + end +end """ get_datasets(file::HDF5.File) -> datasets::Vector{HDF5.Dataset} diff --git a/test/compound.jl b/test/compound.jl index f95fe4197..cd22cdba1 100644 --- a/test/compound.jl +++ b/test/compound.jl @@ -161,7 +161,7 @@ end fn = tempname() h5open(fn, "w") do h5f - write_compound(h5f, "the/bar", bars) + write_compound_dataset(h5f, "the/bar", bars) end thebars = h5open(fn, "r") do h5f From 89fc449be25dfff3699ccebd89c3c0ded704e92e Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Mon, 10 Oct 2022 12:33:08 +0200 Subject: [PATCH 07/25] Rename write_compound to write_compound_dataset Co-authored-by: Mark Kittisopikul --- src/HDF5.jl | 2 +- src/datasets.jl | 57 ------------------------------------------------- 2 files changed, 1 insertion(+), 58 deletions(-) diff --git a/src/HDF5.jl b/src/HDF5.jl index 6fa339578..410317cc7 100644 --- a/src/HDF5.jl +++ b/src/HDF5.jl @@ -28,7 +28,7 @@ export @read, open_dataset, read_dataset, write_dataset, - write_compound, + write_compound_dataset, create_group, open_group, copy_object, diff --git a/src/datasets.jl b/src/datasets.jl index c7eafeb23..6c67112dd 100644 --- a/src/datasets.jl +++ b/src/datasets.jl @@ -482,63 +482,6 @@ write_dataset( xfer::DatasetTransferProperties=dataset.xfer ) = nothing -""" - function write_compound_dataset(f:Union{File, Group}, name::AbstractString, data::AbstractArray{T}) where T - -Writes an array of structs with primitive field types to an HDF5Compound dataset. - -Examples -======== - -``` -julia> struct Foo - x::Int32 - y::Float64 - end - -julia> foos = [[Foo(1, 2) Foo(3, 4) Foo(5, 6)]; [Foo(7, 8) Foo(9, 10) Foo(11, 12)]] -2×3 Matrix{Foo}: - Foo(1, 2.0) Foo(3, 4.0) Foo(5, 6.0) - Foo(7, 8.0) Foo(9, 10.0) Foo(11, 12.0) - -julia> h5open("foo.h5", "w") do h5f - write_compound(h5f, "the/foo", ts) - end - -julia> thefoo = h5open("foo.h5", "r") do file - read(file, "the/foo") - end -2×3 Matrix{NamedTuple{(:x, :y), Tuple{Int32, Float64}}}: - (x = 1, y = 2.0) (x = 3, y = 4.0) (x = 5, y = 6.0) - (x = 7, y = 8.0) (x = 9, y = 10.0) (x = 11, y = 12.0) -``` - -The data can be reinterpreted to the original data type using `reinterpret()`: - -``` -julia> reinterpret(Foo, thefoo) -2×3 reinterpret(Foo, ::Matrix{NamedTuple{(:x, :y), Tuple{Int32, Float64}}}): - Foo(1, 2.0) Foo(3, 4.0) Foo(5, 6.0) - Foo(7, 8.0) Foo(9, 10.0) Foo(11, 12.0) -``` - -""" -function write_compound_dataset( - f::Union{File, Group}, name::AbstractString, data::AbstractArray{T} -) where {T} - dtype = API.h5t_create(API.H5T_COMPOUND, sizeof(T)) - for (idx, fn) in enumerate(fieldnames(T)) - API.h5t_insert(dtype, fn, fieldoffset(T, idx), datatype(fieldtype(T, idx))) - end - dt = Datatype(dtype) - dset = create_dataset(f, name, dt, dataspace(data)) - write_dataset(dset, dt, data) -end -function write_compound_dataset(f::AbstractString, name::AbstractString, data::AbstractArray{T}) where {T} - h5open(f, "w") do h5f - write_compound_dataset(h5f, name, data) - end -end """ get_datasets(file::HDF5.File) -> datasets::Vector{HDF5.Dataset} From fe273a964e98bff801a325c28440cd4e2ab2268d Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Mon, 10 Oct 2022 16:40:04 +0200 Subject: [PATCH 08/25] Implement proper datatype discovery for compound --- src/datasets.jl | 1 - src/typeconversions.jl | 9 +++++++++ test/compound.jl | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/datasets.jl b/src/datasets.jl index 6c67112dd..d1b1b773c 100644 --- a/src/datasets.jl +++ b/src/datasets.jl @@ -482,7 +482,6 @@ write_dataset( xfer::DatasetTransferProperties=dataset.xfer ) = nothing - """ get_datasets(file::HDF5.File) -> datasets::Vector{HDF5.Dataset} diff --git a/src/typeconversions.jl b/src/typeconversions.jl index 9941b0e8f..d5e2ff40d 100644 --- a/src/typeconversions.jl +++ b/src/typeconversions.jl @@ -60,6 +60,15 @@ function datatype(str::VLen{C}) where {C<:CharType} Datatype(API.h5t_vlen_create(type_id)) end +# Compound types +function datatype(A::AbstractArray{T}) where {T} + dtype = API.h5t_create(API.H5T_COMPOUND, sizeof(T)) + for (idx, fn) in enumerate(fieldnames(T)) + API.h5t_insert(dtype, fn, fieldoffset(T, idx), datatype(fieldtype(T, idx))) + end + Datatype(dtype) +end + # Opaque types struct Opaque data diff --git a/test/compound.jl b/test/compound.jl index cd22cdba1..5d0714812 100644 --- a/test/compound.jl +++ b/test/compound.jl @@ -161,7 +161,7 @@ end fn = tempname() h5open(fn, "w") do h5f - write_compound_dataset(h5f, "the/bar", bars) + write_dataset(h5f, "the/bar", bars) end thebars = h5open(fn, "r") do h5f From 4c66a5c2e3f39705e3c28b3fc410796725ee40c6 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Mon, 10 Oct 2022 17:00:19 +0200 Subject: [PATCH 09/25] Dispatch on struct-type itself --- src/typeconversions.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/typeconversions.jl b/src/typeconversions.jl index d5e2ff40d..d078ce95f 100644 --- a/src/typeconversions.jl +++ b/src/typeconversions.jl @@ -61,7 +61,10 @@ function datatype(str::VLen{C}) where {C<:CharType} end # Compound types -function datatype(A::AbstractArray{T}) where {T} +function datatype(S::T) where {T} + if !isstructtype(T) + return datatype(T) + end dtype = API.h5t_create(API.H5T_COMPOUND, sizeof(T)) for (idx, fn) in enumerate(fieldnames(T)) API.h5t_insert(dtype, fn, fieldoffset(T, idx), datatype(fieldtype(T, idx))) From a7f3b557f420b858a52795f2fba7dcb75206921f Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Wed, 12 Oct 2022 14:08:19 +0200 Subject: [PATCH 10/25] Readd AbstractArray to datatype dispatch --- src/typeconversions.jl | 5 +---- test/compound.jl | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/typeconversions.jl b/src/typeconversions.jl index d078ce95f..d5e2ff40d 100644 --- a/src/typeconversions.jl +++ b/src/typeconversions.jl @@ -61,10 +61,7 @@ function datatype(str::VLen{C}) where {C<:CharType} end # Compound types -function datatype(S::T) where {T} - if !isstructtype(T) - return datatype(T) - end +function datatype(A::AbstractArray{T}) where {T} dtype = API.h5t_create(API.H5T_COMPOUND, sizeof(T)) for (idx, fn) in enumerate(fieldnames(T)) API.h5t_insert(dtype, fn, fieldoffset(T, idx), datatype(fieldtype(T, idx))) diff --git a/test/compound.jl b/test/compound.jl index 5d0714812..311177458 100644 --- a/test/compound.jl +++ b/test/compound.jl @@ -159,9 +159,12 @@ end [Bar(4, 4.1, false) Bar(5, 5.1, true) Bar(6, 6.1, false)] ] + namedtuples = [(a=1, b=2.3), (a=4, b=5.6)] + fn = tempname() h5open(fn, "w") do h5f write_dataset(h5f, "the/bar", bars) + write_dataset(h5f, "the/namedtuples", namedtuples) end thebars = h5open(fn, "r") do h5f @@ -178,4 +181,15 @@ end @test thebars_r[1, 2].b ≈ 2.1 @test thebars_r[2, 1].a == 4 @test thebars_r[1, 3].c + + thenamedtuples = h5open(fn, "r") do h5f + read(h5f, "the/namedtuples") + end + + @test (2,) == size(thenamedtuples) + @test thenamedtuples[1].a == 1 + @test thenamedtuples[1].b ≈ 2.3 + @test thenamedtuples[2].a == 4 + @test thenamedtuples[2].b ≈ 5.6 + end From 5507f6ff654cfdb3f3a152b736914397a1c39545 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Wed, 12 Oct 2022 14:13:33 +0200 Subject: [PATCH 11/25] JuliaFormatter --- test/compound.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/test/compound.jl b/test/compound.jl index 311177458..2b52e86db 100644 --- a/test/compound.jl +++ b/test/compound.jl @@ -191,5 +191,4 @@ end @test thenamedtuples[1].b ≈ 2.3 @test thenamedtuples[2].a == 4 @test thenamedtuples[2].b ≈ 5.6 - end From 856af2fd3d8192d1cd696f7cbf7f4ab4aef41a79 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Wed, 12 Oct 2022 16:27:30 +0200 Subject: [PATCH 12/25] Cleanup --- src/typeconversions.jl | 3 ++- test/compound.jl | 11 ++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/typeconversions.jl b/src/typeconversions.jl index d5e2ff40d..5508e5802 100644 --- a/src/typeconversions.jl +++ b/src/typeconversions.jl @@ -61,7 +61,8 @@ function datatype(str::VLen{C}) where {C<:CharType} end # Compound types -function datatype(A::AbstractArray{T}) where {T} +datatype(x::AbstractArray{T}) where {T} = HDF5.datatype(x, Val(isstructtype(T))) +function datatype(x::AbstractArray{T}, isstruct::Val{true}) where {T} dtype = API.h5t_create(API.H5T_COMPOUND, sizeof(T)) for (idx, fn) in enumerate(fieldnames(T)) API.h5t_insert(dtype, fn, fieldoffset(T, idx), datatype(fieldtype(T, idx))) diff --git a/test/compound.jl b/test/compound.jl index 2b52e86db..1c68a38cd 100644 --- a/test/compound.jl +++ b/test/compound.jl @@ -163,12 +163,17 @@ end fn = tempname() h5open(fn, "w") do h5f - write_dataset(h5f, "the/bar", bars) - write_dataset(h5f, "the/namedtuples", namedtuples) +# write_dataset(h5f, "the/bar", Bar(1, 2.3, false)) + write_dataset(h5f, "the/bars", bars) + write_dataset(h5f, "the/namedtuples", namedtuples) end + # thebar = h5open(fn, "r") do h5f + # read(h5f, "the/bar") + # end + thebars = h5open(fn, "r") do h5f - read(h5f, "the/bar") + read(h5f, "the/bars") end @test (2, 3) == size(thebars) From 0c525fe0fc9916273f0539c7cd84f027655a3002 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Wed, 12 Oct 2022 16:33:05 +0200 Subject: [PATCH 13/25] JuliaFormatter --- test/compound.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/compound.jl b/test/compound.jl index 1c68a38cd..155aec63d 100644 --- a/test/compound.jl +++ b/test/compound.jl @@ -163,9 +163,9 @@ end fn = tempname() h5open(fn, "w") do h5f -# write_dataset(h5f, "the/bar", Bar(1, 2.3, false)) - write_dataset(h5f, "the/bars", bars) - write_dataset(h5f, "the/namedtuples", namedtuples) + # write_dataset(h5f, "the/bar", Bar(1, 2.3, false)) + write_dataset(h5f, "the/bars", bars) + write_dataset(h5f, "the/namedtuples", namedtuples) end # thebar = h5open(fn, "r") do h5f From 106adb9d296c5283ed1c6f54590490b890ccada7 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Thu, 13 Oct 2022 09:35:24 +0200 Subject: [PATCH 14/25] Update src/typeconversions.jl Co-authored-by: Mark Kittisopikul --- src/typeconversions.jl | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/typeconversions.jl b/src/typeconversions.jl index 5508e5802..eee003433 100644 --- a/src/typeconversions.jl +++ b/src/typeconversions.jl @@ -61,14 +61,23 @@ function datatype(str::VLen{C}) where {C<:CharType} end # Compound types -datatype(x::AbstractArray{T}) where {T} = HDF5.datatype(x, Val(isstructtype(T))) -function datatype(x::AbstractArray{T}, isstruct::Val{true}) where {T} + +datatype(::T) where T = Datatype(hdf5_type_id(T), false) +datatype(::Type{T}) where T = Datatype(hdf5_type_id(T), false) +datatype(x::AbstractArray{T}) where T = Datatype(hdf5_type_id(T), false) + + +# Move closer to where hdf5_type_id is defined +hdf5_type_id(::Type{T}) where T = hdf5_type_id(T, Val(isstructtype(T))) +function hdf5_type_id(::Type{T}, isstruct::Val{true}) where T dtype = API.h5t_create(API.H5T_COMPOUND, sizeof(T)) for (idx, fn) in enumerate(fieldnames(T)) - API.h5t_insert(dtype, fn, fieldoffset(T, idx), datatype(fieldtype(T, idx))) + API.h5t_insert(dtype, fn, fieldoffset(T, idx), hdf5_type_id(fieldtype(T, idx))) end - Datatype(dtype) + return dtype end +# Perhaps we need a custom error type here +hdf5_type_id(::Type{T}, isstruct::Val{false}) where T = throw(MethodError(hdf5_type_id, (T, isstruct))) # Opaque types struct Opaque From 05ede0f0ae3ddc77a0868851dd25f9b26e768441 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Thu, 13 Oct 2022 10:03:02 +0200 Subject: [PATCH 15/25] Remove remark --- src/typeconversions.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/typeconversions.jl b/src/typeconversions.jl index eee003433..0c32e30cf 100644 --- a/src/typeconversions.jl +++ b/src/typeconversions.jl @@ -67,7 +67,6 @@ datatype(::Type{T}) where T = Datatype(hdf5_type_id(T), false) datatype(x::AbstractArray{T}) where T = Datatype(hdf5_type_id(T), false) -# Move closer to where hdf5_type_id is defined hdf5_type_id(::Type{T}) where T = hdf5_type_id(T, Val(isstructtype(T))) function hdf5_type_id(::Type{T}, isstruct::Val{true}) where T dtype = API.h5t_create(API.H5T_COMPOUND, sizeof(T)) From 68f42ab826967b43ebc8d00c247824d43365fe99 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Thu, 13 Oct 2022 10:20:32 +0200 Subject: [PATCH 16/25] JuliaFormatter --- src/typeconversions.jl | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/typeconversions.jl b/src/typeconversions.jl index 0c32e30cf..f98604629 100644 --- a/src/typeconversions.jl +++ b/src/typeconversions.jl @@ -62,21 +62,24 @@ end # Compound types -datatype(::T) where T = Datatype(hdf5_type_id(T), false) -datatype(::Type{T}) where T = Datatype(hdf5_type_id(T), false) -datatype(x::AbstractArray{T}) where T = Datatype(hdf5_type_id(T), false) +datatype(::T) where {T} = Datatype(hdf5_type_id(T), false) +datatype(::Type{T}) where {T} = Datatype(hdf5_type_id(T), false) +datatype(x::AbstractArray{T}) where {T} = Datatype(hdf5_type_id(T), false) - -hdf5_type_id(::Type{T}) where T = hdf5_type_id(T, Val(isstructtype(T))) -function hdf5_type_id(::Type{T}, isstruct::Val{true}) where T +hdf5_type_id(::Type{T}) where {T} = hdf5_type_id(T, Val(isstructtype(T))) +function hdf5_type_id(::Type{T}, isstruct::Val{true}) where {T} dtype = API.h5t_create(API.H5T_COMPOUND, sizeof(T)) + offset = 0 for (idx, fn) in enumerate(fieldnames(T)) - API.h5t_insert(dtype, fn, fieldoffset(T, idx), hdf5_type_id(fieldtype(T, idx))) + ftype = fieldtype(T, idx) + API.h5t_insert(dtype, fn, fieldoffset(T, idx), hdf5_type_id(ftype)) + offset += sizeof(ftype) end return dtype end # Perhaps we need a custom error type here -hdf5_type_id(::Type{T}, isstruct::Val{false}) where T = throw(MethodError(hdf5_type_id, (T, isstruct))) +hdf5_type_id(::Type{T}, isstruct::Val{false}) where {T} = + throw(MethodError(hdf5_type_id, (T, isstruct))) # Opaque types struct Opaque From 07509055cda6c0bc27eac4c29e0230f0f977de8d Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Thu, 13 Oct 2022 10:20:58 +0200 Subject: [PATCH 17/25] Cleanup --- src/typeconversions.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/typeconversions.jl b/src/typeconversions.jl index f98604629..b9fd49412 100644 --- a/src/typeconversions.jl +++ b/src/typeconversions.jl @@ -69,11 +69,9 @@ datatype(x::AbstractArray{T}) where {T} = Datatype(hdf5_type_id(T), false) hdf5_type_id(::Type{T}) where {T} = hdf5_type_id(T, Val(isstructtype(T))) function hdf5_type_id(::Type{T}, isstruct::Val{true}) where {T} dtype = API.h5t_create(API.H5T_COMPOUND, sizeof(T)) - offset = 0 for (idx, fn) in enumerate(fieldnames(T)) ftype = fieldtype(T, idx) API.h5t_insert(dtype, fn, fieldoffset(T, idx), hdf5_type_id(ftype)) - offset += sizeof(ftype) end return dtype end From 798e8be4f63a465392c07b986e51e841505f45bc Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Thu, 13 Oct 2022 14:51:51 +0200 Subject: [PATCH 18/25] Update src/HDF5.jl Co-authored-by: Mark Kittisopikul --- src/HDF5.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/HDF5.jl b/src/HDF5.jl index 410317cc7..ced25e703 100644 --- a/src/HDF5.jl +++ b/src/HDF5.jl @@ -28,7 +28,6 @@ export @read, open_dataset, read_dataset, write_dataset, - write_compound_dataset, create_group, open_group, copy_object, From 741e5923dae2acf599a69d6aa2eb28ae0ac16753 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Thu, 13 Oct 2022 14:52:46 +0200 Subject: [PATCH 19/25] Clean up --- test/compound.jl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/compound.jl b/test/compound.jl index 155aec63d..1ae7c47b2 100644 --- a/test/compound.jl +++ b/test/compound.jl @@ -163,15 +163,10 @@ end fn = tempname() h5open(fn, "w") do h5f - # write_dataset(h5f, "the/bar", Bar(1, 2.3, false)) write_dataset(h5f, "the/bars", bars) write_dataset(h5f, "the/namedtuples", namedtuples) end - # thebar = h5open(fn, "r") do h5f - # read(h5f, "the/bar") - # end - thebars = h5open(fn, "r") do h5f read(h5f, "the/bars") end From 987d9c89e43abe68dfeced963314ed09ad4c1803 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Sun, 23 Oct 2022 18:16:02 +0200 Subject: [PATCH 20/25] Add tests for mutable structs --- test/compound.jl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/compound.jl b/test/compound.jl index 1ae7c47b2..d39e999b5 100644 --- a/test/compound.jl +++ b/test/compound.jl @@ -153,6 +153,10 @@ struct Bar c::Bool end +mutable struct MutableBar + x::Int64 +end + @testset "write_compound" begin bars = [ [Bar(1, 1.1, true) Bar(2, 2.1, false) Bar(3, 3.1, true)] @@ -191,4 +195,17 @@ end @test thenamedtuples[1].b ≈ 2.3 @test thenamedtuples[2].a == 4 @test thenamedtuples[2].b ≈ 5.6 + + mutable_bars = [MutableBar(1), MutableBar(2), MutableBar(3)] + + fn = tempname() + h5open(fn, "w") do h5f + write_dataset(h5f, "the/mutable_bars", mutable_bars) + end + + themutablebars = h5open(fn, "r") do h5f + read(h5f, "the/mutable_bars") + end + + @test [1, 2, 3] == [b.x for b ∈ themutablebars] end From 507fd1c894843864dade38a01ab494c8d40a699b Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Mon, 24 Oct 2022 21:39:35 -0400 Subject: [PATCH 21/25] Convert non-bitstypes to a NamedTuple when writing datasets --- docs/make.jl | 9 +++------ src/datasets.jl | 26 +++++++++++++++++++++++--- test/compound.jl | 10 +++++++++- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/docs/make.jl b/docs/make.jl index 1f2cd19ee..0c024ea45 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -27,7 +27,7 @@ makedocs(; prettyurls=get(ENV, "CI", "false") == "true", canonical="https://JuliaIO.github.io/HDF5.jl", assets=String[], - sidebar_sitename=false, + sidebar_sitename=false ), pages=[ "Home" => "index.md", @@ -41,10 +41,7 @@ makedocs(; ], "mpi.md", "Low-level library bindings" => "api_bindings.md", - ], + ] ) -deploydocs(; - repo="github.com/JuliaIO/HDF5.jl.git", - push_preview=true, -) +deploydocs(; repo="github.com/JuliaIO/HDF5.jl.git", push_preview=true) diff --git a/src/datasets.jl b/src/datasets.jl index d1b1b773c..f07e42a29 100644 --- a/src/datasets.jl +++ b/src/datasets.jl @@ -439,11 +439,31 @@ end function write_dataset( dataset::Dataset, memtype::Datatype, - buf::AbstractArray, + buf::AbstractArray{T}, xfer::DatasetTransferProperties=dataset.xfer -) +) where {T} _check_invalid(dataset, buf) - API.h5d_write(dataset, memtype, API.H5S_ALL, API.H5S_ALL, xfer, buf) + if isbitstype(T) + API.h5d_write(dataset, memtype, API.H5S_ALL, API.H5S_ALL, xfer, buf) + else + # For non-bitstypes, we need to convert the buffer to a bitstype + # For mutable structs, this will usually be a NamedTuple. + jl_type = get_mem_compatible_jl_type(memtype) + try + memtype_buf = convert(Array{jl_type}, buf) + API.h5d_write(dataset, memtype, API.H5S_ALL, API.H5S_ALL, xfer, memtype_buf) + catch err + if err isa MethodError + throw( + ArgumentError( + "Could not convert non-bitstype $T to $jl_type for writing to HDF5. Consider implementing `convert(::Type{Array{$jl_type}}, ::$T)`" + ) + ) + else + rethrow() + end + end + end end function write_dataset( dataset::Dataset, diff --git a/test/compound.jl b/test/compound.jl index d39e999b5..ab302df2d 100644 --- a/test/compound.jl +++ b/test/compound.jl @@ -199,6 +199,14 @@ end mutable_bars = [MutableBar(1), MutableBar(2), MutableBar(3)] fn = tempname() + @test_throws ArgumentError begin + h5open(fn, "w") do h5f + write_dataset(h5f, "the/mutable_bars", mutable_bars) + end + end + + Base.convert(::Type{NamedTuple{(:x,),Tuple{Int64}}}, mb::MutableBar) = (x=mb.x,) + h5open(fn, "w") do h5f write_dataset(h5f, "the/mutable_bars", mutable_bars) end @@ -207,5 +215,5 @@ end read(h5f, "the/mutable_bars") end - @test [1, 2, 3] == [b.x for b ∈ themutablebars] + @test [1, 2, 3] == [b.x for b in themutablebars] end From 40cf17b13147db6a366e814d15716cf9877bee51 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 25 Oct 2022 03:37:11 -0400 Subject: [PATCH 22/25] Make attributes compound type compatible --- src/attributes.jl | 67 ++++++++++++++++++++++++++++++++++++++++++++--- src/datasets.jl | 2 +- src/dataspaces.jl | 6 +++++ test/compound.jl | 17 +++++++++--- 4 files changed, 85 insertions(+), 7 deletions(-) diff --git a/src/attributes.jl b/src/attributes.jl index 7ee564d0c..ef32f3fb5 100644 --- a/src/attributes.jl +++ b/src/attributes.jl @@ -98,15 +98,76 @@ function create_attribute( end # generic method -write_attribute(attr::Attribute, memtype::Datatype, x) = API.h5a_write(attr, memtype, x) +function write_attribute(attr::Attribute, memtype::Datatype, x::T) where {T} + if isbitstype(T) + API.h5a_write(attr, memtype, x) + else + jl_type = get_mem_compatible_jl_type(memtype) + try + x_mem = convert(jl_type, x) + API.h5a_write(attr, memtype, Ref(x_mem)) + catch err + if err isa MethodError + throw( + ArgumentError( + "Could not convert non-bitstype $T to $jl_type for writing to HDF5. Consider implementing `convert(::Type{$jl_type}, ::$T)`" + ) + ) + else + rethrow() + end + end + end +end +function write_attribute(attr::Attribute, memtype::Datatype, x::Ref{T}) where {T} + if isbitstype(T) + API.h5a_write(attr, memtype, x) + else + jl_type = get_mem_compatible_jl_type(memtype) + try + x_mem = convert(Ref{jl_type}, x[]) + API.h5a_write(attr, memtype, x_mem) + catch err + if err isa MethodError + throw( + ArgumentError( + "Could not convert non-bitstype $T to $jl_type for writing to HDF5. Consider implementing `convert(::Type{$jl_type}, ::$T)`" + ) + ) + else + rethrow() + end + end + end +end + # specific methods -function write_attribute(attr::Attribute, memtype::Datatype, x::AbstractArray) +write_attribute(attr::Attribute, memtype::Datatype, x::VLen) = API.h5a_write(attr, memtype, x) +function write_attribute(attr::Attribute, memtype::Datatype, x::AbstractArray{T}) where {T} length(x) == length(attr) || throw( ArgumentError( "Invalid length: $(length(x)) != $(length(attr)), for attribute \"$(name(attr))\"" ) ) - API.h5a_write(attr, memtype, x) + if isbitstype(T) + API.h5a_write(attr, memtype, x) + else + jl_type = get_mem_compatible_jl_type(memtype) + try + x_mem = convert(Array{jl_type}, x) + API.h5a_write(attr, memtype, x_mem) + catch err + if err isa MethodError + throw( + ArgumentError( + "Could not convert non-bitstype $T to $jl_type for writing to HDF5. Consider implementing `convert(::Type{$jl_type}, ::$T)`" + ) + ) + else + rethrow() + end + end + end end function write_attribute(attr::Attribute, memtype::Datatype, str::AbstractString) strbuf = Base.cconvert(Cstring, str) diff --git a/src/datasets.jl b/src/datasets.jl index f07e42a29..67f01dcc3 100644 --- a/src/datasets.jl +++ b/src/datasets.jl @@ -456,7 +456,7 @@ function write_dataset( if err isa MethodError throw( ArgumentError( - "Could not convert non-bitstype $T to $jl_type for writing to HDF5. Consider implementing `convert(::Type{Array{$jl_type}}, ::$T)`" + "Could not convert non-bitstype $T to $jl_type for writing to HDF5. Consider implementing `convert(::Type{$jl_type}, ::$T)`" ) ) else diff --git a/src/dataspaces.jl b/src/dataspaces.jl index 2cd1a8e3e..59b5c6fef 100644 --- a/src/dataspaces.jl +++ b/src/dataspaces.jl @@ -46,6 +46,12 @@ The default `Dataspace` used for representing a Julia object `data`: - arrays: a simple `Dataspace` - `nothing` or an `EmptyArray`: a null dataspace """ +dataspace(x::T) where {T} = + if isstructtype(T) + Dataspace(API.h5s_create(API.H5S_SCALAR)) + else + throw(MethodError(dataspace, x)) + end dataspace(x::Union{T,Complex{T}}) where {T<:ScalarType} = Dataspace(API.h5s_create(API.H5S_SCALAR)) dataspace(::AbstractString) = Dataspace(API.h5s_create(API.H5S_SCALAR)) diff --git a/test/compound.jl b/test/compound.jl index ab302df2d..ca73c8fa7 100644 --- a/test/compound.jl +++ b/test/compound.jl @@ -206,14 +206,25 @@ end end Base.convert(::Type{NamedTuple{(:x,),Tuple{Int64}}}, mb::MutableBar) = (x=mb.x,) + Base.unsafe_convert(::Type{Ptr{Nothing}}, mb::MutableBar) = pointer_from_objref(mb) h5open(fn, "w") do h5f write_dataset(h5f, "the/mutable_bars", mutable_bars) + write_dataset(h5f, "the/mutable_bar", first(mutable_bars)) end - themutablebars = h5open(fn, "r") do h5f - read(h5f, "the/mutable_bars") + h5open(fn, "r") do h5f + @test [1, 2, 3] == [b.x for b in read(h5f, "the/mutable_bars")] + @test 1 == read(h5f, "the/mutable_bar").x end - @test [1, 2, 3] == [b.x for b in themutablebars] + h5open(fn, "w") do h5f + write_attribute(h5f, "mutable_bars", mutable_bars) + write_attribute(h5f, "mutable_bar", first(mutable_bars)) + end + + h5open(fn, "r") do h5f + @test [1, 2, 3] == [b.x for b in attrs(h5f)["mutable_bars"]] + @test 1 == attrs(h5f)["mutable_bar"].x + end end From 8eacf5f31505c6a84965bd17f4d531857ee7b400 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 25 Oct 2022 03:40:04 -0400 Subject: [PATCH 23/25] Garbage collect compound types --- src/typeconversions.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/typeconversions.jl b/src/typeconversions.jl index b9fd49412..ad8de4868 100644 --- a/src/typeconversions.jl +++ b/src/typeconversions.jl @@ -62,9 +62,10 @@ end # Compound types -datatype(::T) where {T} = Datatype(hdf5_type_id(T), false) -datatype(::Type{T}) where {T} = Datatype(hdf5_type_id(T), false) -datatype(x::AbstractArray{T}) where {T} = Datatype(hdf5_type_id(T), false) +# These will use finalizers. Close them eagerly to avoid issues. +datatype(::T) where {T} = Datatype(hdf5_type_id(T), true) +datatype(::Type{T}) where {T} = Datatype(hdf5_type_id(T), true) +datatype(x::AbstractArray{T}) where {T} = Datatype(hdf5_type_id(T), true) hdf5_type_id(::Type{T}) where {T} = hdf5_type_id(T, Val(isstructtype(T))) function hdf5_type_id(::Type{T}, isstruct::Val{true}) where {T} From a5b16898945263322e96d71e6f1beba4ead5c716 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 25 Oct 2022 03:44:53 -0400 Subject: [PATCH 24/25] Formatting --- src/attributes.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/attributes.jl b/src/attributes.jl index ef32f3fb5..5fe9824bd 100644 --- a/src/attributes.jl +++ b/src/attributes.jl @@ -142,7 +142,8 @@ function write_attribute(attr::Attribute, memtype::Datatype, x::Ref{T}) where {T end # specific methods -write_attribute(attr::Attribute, memtype::Datatype, x::VLen) = API.h5a_write(attr, memtype, x) +write_attribute(attr::Attribute, memtype::Datatype, x::VLen) = + API.h5a_write(attr, memtype, x) function write_attribute(attr::Attribute, memtype::Datatype, x::AbstractArray{T}) where {T} length(x) == length(attr) || throw( ArgumentError( From cff8206fac5d2560f882f9d628adeac199261bd5 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Fri, 9 Dec 2022 01:17:44 -0500 Subject: [PATCH 25/25] Apply suggestions from code review Co-authored-by: Mustafa M --- src/dataspaces.jl | 1 + src/typeconversions.jl | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dataspaces.jl b/src/dataspaces.jl index 59b5c6fef..f5040a184 100644 --- a/src/dataspaces.jl +++ b/src/dataspaces.jl @@ -44,6 +44,7 @@ dataspace(ds::Dataspace) = ds The default `Dataspace` used for representing a Julia object `data`: - strings or numbers: a scalar `Dataspace` - arrays: a simple `Dataspace` + - `struct` types: a scalar `Dataspace` - `nothing` or an `EmptyArray`: a null dataspace """ dataspace(x::T) where {T} = diff --git a/src/typeconversions.jl b/src/typeconversions.jl index ad8de4868..315a550f7 100644 --- a/src/typeconversions.jl +++ b/src/typeconversions.jl @@ -64,7 +64,6 @@ end # These will use finalizers. Close them eagerly to avoid issues. datatype(::T) where {T} = Datatype(hdf5_type_id(T), true) -datatype(::Type{T}) where {T} = Datatype(hdf5_type_id(T), true) datatype(x::AbstractArray{T}) where {T} = Datatype(hdf5_type_id(T), true) hdf5_type_id(::Type{T}) where {T} = hdf5_type_id(T, Val(isstructtype(T)))