From ef57f4d3b3f637db2abf4440407cdc6f6775213a Mon Sep 17 00:00:00 2001 From: Grisha Szep Date: Sun, 11 Sep 2022 23:21:48 +0100 Subject: [PATCH 01/16] add ros3 functions --- docs/src/api_bindings.md | 4 ++++ docs/src/interface/properties.md | 1 + gen/api_defs.jl | 2 ++ src/api/functions.jl | 22 +++++++++++++++++++++ src/api/types.jl | 8 ++++++++ src/drivers/drivers.jl | 33 ++++++++++++++++++++++++++++++-- 6 files changed, 68 insertions(+), 2 deletions(-) diff --git a/docs/src/api_bindings.md b/docs/src/api_bindings.md index 6f98de5b2..19dae7d28 100644 --- a/docs/src/api_bindings.md +++ b/docs/src/api_bindings.md @@ -478,6 +478,7 @@ h5pl_size - [`h5p_get_fapl_mpio64`](@ref h5p_get_fapl_mpio64) - [`h5p_get_fapl_multi`](@ref h5p_get_fapl_multi) - [`h5p_get_fapl_splitter`](@ref h5p_get_fapl_splitter) +- [`h5p_get_fapl_ros3`](@ref h5p_get_fapl_ros3) - [`h5p_get_fclose_degree`](@ref h5p_get_fclose_degree) - [`h5p_get_file_image`](@ref h5p_get_file_image) - [`h5p_get_file_image_callbacks`](@ref h5p_get_file_image_callbacks) @@ -580,6 +581,7 @@ h5pl_size - [`h5p_set_fapl_mpio64`](@ref h5p_set_fapl_mpio64) - [`h5p_set_fapl_multi`](@ref h5p_set_fapl_multi) - [`h5p_set_fapl_sec2`](@ref h5p_set_fapl_sec2) +- [`h5p_set_fapl_ros3`](@ref h5p_set_fapl_ros3) - [`h5p_set_fapl_split`](@ref h5p_set_fapl_split) - [`h5p_set_fapl_splitter`](@ref h5p_set_fapl_splitter) - [`h5p_set_fapl_stdio`](@ref h5p_set_fapl_stdio) @@ -694,6 +696,7 @@ h5p_get_fapl_mpio32 h5p_get_fapl_mpio64 h5p_get_fapl_multi h5p_get_fapl_splitter +h5p_get_fapl_ros3 h5p_get_fclose_degree h5p_get_file_image h5p_get_file_image_callbacks @@ -796,6 +799,7 @@ h5p_set_fapl_mpio32 h5p_set_fapl_mpio64 h5p_set_fapl_multi h5p_set_fapl_sec2 +h5p_set_fapl_ros3 h5p_set_fapl_split h5p_set_fapl_splitter h5p_set_fapl_stdio diff --git a/docs/src/interface/properties.md b/docs/src/interface/properties.md index a2e67ac22..0968fd476 100644 --- a/docs/src/interface/properties.md +++ b/docs/src/interface/properties.md @@ -44,5 +44,6 @@ CurrentModule = HDF5.Drivers ```@docs Core POSIX +ROS3 MPIO ``` diff --git a/gen/api_defs.jl b/gen/api_defs.jl index 4bff4b1e5..926839880 100644 --- a/gen/api_defs.jl +++ b/gen/api_defs.jl @@ -285,6 +285,7 @@ @bind h5p_get_fapl_mpio64(fapl_id::hid_t, comm::Ptr{Hmpih64}, info::Ptr{Hmpih64})::herr_t "Error getting MPIO properties" @bind h5p_get_fapl_multi(fapl_id::hid_t, memb_map::Ptr{H5FD_mem_t}, memb_fapl::Ptr{hid_t}, memb_name::Ptr{Ptr{Cchar}}, memb_addr::Ptr{haddr_t}, relax::Ptr{hbool_t})::herr_t "Error in h5p_get_fapl_multi (not annotated)" @bind h5p_get_fapl_splitter(fapl_id::hid_t, config_ptr::Ptr{H5FD_splitter_vfd_config_t})::herr_t "Error in h5p_get_fapl_splitter (not annotated)" +@bind h5p_get_fapl_ros3(fapl_id::hid_t, fa_out::Ptr{H5FD_ros3_fapl_t})::herr_t "Error in getting ros3 properties" @bind h5p_get_fclose_degree(fapl_id::hid_t, fc_degree::Ref{Cint})::herr_t "Error getting close degree" @bind h5p_get_file_image(fapl_id::hid_t, buf_ptr_ptr::Ptr{Ptr{Cvoid}}, buf_len_ptr::Ptr{Csize_t})::herr_t "Error in h5p_get_file_image (not annotated)" @bind h5p_get_file_image_callbacks(fapl_id::hid_t, callbacks_ptr::Ptr{H5FD_file_image_callbacks_t})::herr_t "Error in h5p_get_file_image_callbacks (not annotated)" @@ -382,6 +383,7 @@ @bind h5p_set_fapl_mpio64(fapl_id::hid_t, comm::Hmpih64, info::Hmpih64)::herr_t "Error setting MPIO properties" @bind h5p_set_fapl_multi(fapl_id::hid_t, memb_map::Ptr{H5FD_mem_t}, memb_fapl::Ptr{hid_t}, memb_name::Ptr{Ptr{Cchar}}, memb_addr::Ptr{haddr_t}, relax::hbool_t)::herr_t "Error in h5p_set_fapl_multi (not annotated)" @bind h5p_set_fapl_sec2(fapl_id::hid_t)::herr_t "Error setting Sec2 properties" +@bind h5p_set_fapl_ros3(fapl_id::hid_t, fa::Ptr{H5FD_ros3_fapl_t})::herr_t "Error in setting ros3 properties" @bind h5p_set_fapl_split(fapl::hid_t, meta_ext::Ptr{Cchar}, meta_plist_id::hid_t, raw_ext::Ptr{Cchar}, raw_plist_id::hid_t)::herr_t "Error in h5p_set_fapl_split (not annotated)" @bind h5p_set_fapl_splitter(fapl_id::hid_t, config_ptr::Ptr{H5FD_splitter_vfd_config_t})::herr_t "Error in h5p_set_fapl_splitter (not annotated)" @bind h5p_set_fapl_stdio(fapl_id::hid_t)::herr_t "Error in h5p_set_fapl_stdio (not annotated)" diff --git a/src/api/functions.jl b/src/api/functions.jl index 736fab890..66d63173c 100644 --- a/src/api/functions.jl +++ b/src/api/functions.jl @@ -2366,6 +2366,17 @@ function h5p_get_fapl_splitter(fapl_id, config_ptr) return nothing end +""" + h5p_get_fapl_ros3(fapl_id::hid_t, fa_out::Ptr{H5FD_hdfs_fapl_t}) + +See `libhdf5` documentation for [`H5Pget_fapl_ros3`](https://portal.hdfgroup.org/display/HDF5/H5P_GET_FAPL_ROS3). +""" +function h5p_get_fapl_ros3(fapl_id, fa_out) + var"#status#" = ccall((:H5Pget_fapl_ros3, libhdf5), herr_t, (hid_t, Ptr{H5FD_ros3_fapl_t}), fapl_id, fa_out) + var"#status#" < 0 && @h5error("Error in getting ros3 properties") + return nothing +end + """ h5p_get_fclose_degree(fapl_id::hid_t, fc_degree::Ref{Cint}) @@ -3411,6 +3422,17 @@ function h5p_set_fapl_sec2(fapl_id) return nothing end +""" + h5p_set_fapl_ros3(fapl_id::hid_t, fa::Ptr{H5FD_ros3_fapl_t}) + +See `libhdf5` documentation for [`H5Pset_fapl_ros3`](https://portal.hdfgroup.org/display/HDF5/H5P_SET_FAPL_ROS3). +""" +function h5p_set_fapl_ros3(fapl_id, fa) + var"#status#" = ccall((:H5Pset_fapl_ros3, libhdf5), herr_t, (hid_t, Ptr{H5FD_ros3_fapl_t}), fapl_id, fa) + var"#status#" < 0 && @h5error("Error in setting ros3 properties") + return nothing +end + """ h5p_set_fapl_split(fapl::hid_t, meta_ext::Ptr{Cchar}, meta_plist_id::hid_t, raw_ext::Ptr{Cchar}, raw_plist_id::hid_t) diff --git a/src/api/types.jl b/src/api/types.jl index 14c19abcb..4e95af298 100644 --- a/src/api/types.jl +++ b/src/api/types.jl @@ -222,6 +222,14 @@ struct H5FD_hdfs_fapl_t stream_buffer_size::Int32 end +struct H5FD_ros3_fapl_t + version::Int32 + authenticate::hbool_t + aws_region::NTuple{33,Cchar} + secret_id::NTuple{129,Cchar} + secret_key::NTuple{129,Cchar} +end + struct H5FD_splitter_vfd_config_t magic::Int32 version::Cuint diff --git a/src/drivers/drivers.jl b/src/drivers/drivers.jl index 43bb11483..222cf00e1 100644 --- a/src/drivers/drivers.jl +++ b/src/drivers/drivers.jl @@ -1,6 +1,6 @@ module Drivers -export POSIX +export POSIX, ROS3 import ..API import ..HDF5: HDF5, Properties, h5doc @@ -79,10 +79,39 @@ function set_driver!(fapl::Properties, ::POSIX) return nothing end +""" + ROS3() + +This is the read-only virtual driver that enables access to HDF5 objects stored in AWS S3 +""" +struct ROS3 <: Driver + fa::API.H5FD_ros3_fapl_t +end +ROS3(region::T, id::T, key::T) where {T<:AbstractString} = (ROS3 ∘ API.H5FD_ros3_fapl_t)( + 1, true, cconvert(region, 33), cconvert(id, 129), cconvert(key, 129) +) +ROS3() = (ROS3 ∘ API.H5FD_ros3_fapl_t)( + 1, false, cconvert("", 33), cconvert("", 129), cconvert("", 129) +) + +cconvert(s::AbstractString, N::Integer) = Tuple(i ≤ length(s) ? s[i] : '\0' for i in 1:N) + +function get_driver(fapl::Properties, ::Type{ROS3}) + ROS3() +end + +function set_driver!(fapl::Properties, driver::ROS3) + HDF5.init!(fapl) + API.h5p_set_fapl_ros3(fapl, driver.fa) + DRIVERS[API.h5p_get_driver(fapl)] = ROS3 + return nothing +end + function __init__() - # Initialize POSIX key in DRIVERS + # Initialize POSIX,ROS3 keys in DRIVERS HDF5.FileAccessProperties() do fapl set_driver!(fapl, POSIX()) + set_driver!(fapl, ROS3()) end # Check whether the libhdf5 was compiled with parallel support. From 007b49def2818341ac0f2027bc77a121dd312f93 Mon Sep 17 00:00:00 2001 From: Grisha Szep Date: Mon, 12 Sep 2022 00:35:37 +0100 Subject: [PATCH 02/16] fix julia julia to C struct passing --- src/api/functions.jl | 4 ++-- test/mpio.jl | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/api/functions.jl b/src/api/functions.jl index 66d63173c..a020c84a9 100644 --- a/src/api/functions.jl +++ b/src/api/functions.jl @@ -2372,7 +2372,7 @@ end See `libhdf5` documentation for [`H5Pget_fapl_ros3`](https://portal.hdfgroup.org/display/HDF5/H5P_GET_FAPL_ROS3). """ function h5p_get_fapl_ros3(fapl_id, fa_out) - var"#status#" = ccall((:H5Pget_fapl_ros3, libhdf5), herr_t, (hid_t, Ptr{H5FD_ros3_fapl_t}), fapl_id, fa_out) + var"#status#" = ccall((:H5Pget_fapl_ros3, libhdf5), herr_t, (hid_t, Ref{H5FD_ros3_fapl_t}), fapl_id, fa_out) var"#status#" < 0 && @h5error("Error in getting ros3 properties") return nothing end @@ -3428,7 +3428,7 @@ end See `libhdf5` documentation for [`H5Pset_fapl_ros3`](https://portal.hdfgroup.org/display/HDF5/H5P_SET_FAPL_ROS3). """ function h5p_set_fapl_ros3(fapl_id, fa) - var"#status#" = ccall((:H5Pset_fapl_ros3, libhdf5), herr_t, (hid_t, Ptr{H5FD_ros3_fapl_t}), fapl_id, fa) + var"#status#" = ccall((:H5Pset_fapl_ros3, libhdf5), herr_t, (hid_t, Ref{H5FD_ros3_fapl_t}), fapl_id, fa) var"#status#" < 0 && @h5error("Error in setting ros3 properties") return nothing end diff --git a/test/mpio.jl b/test/mpio.jl index 801f50853..ba4b48b9e 100644 --- a/test/mpio.jl +++ b/test/mpio.jl @@ -20,6 +20,7 @@ using Test # Check that serial drivers are still there after loading MPI (#928) @test Drivers.Core ∈ values(Drivers.DRIVERS) @test Drivers.POSIX ∈ values(Drivers.DRIVERS) + @test Drivers.ROS3 ∈ values(Drivers.DRIVERS) let fileprop = HDF5.FileAccessProperties() fileprop.driver = HDF5.Drivers.MPIO(comm, info) From ff4517cd3cc735e607ef0022b3d8bbe659aff1f1 Mon Sep 17 00:00:00 2001 From: Grisha Szep Date: Mon, 12 Sep 2022 10:37:02 +0100 Subject: [PATCH 03/16] Update src/drivers/drivers.jl Co-authored-by: Mark Kittisopikul --- src/drivers/drivers.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/drivers/drivers.jl b/src/drivers/drivers.jl index 222cf00e1..0d4439680 100644 --- a/src/drivers/drivers.jl +++ b/src/drivers/drivers.jl @@ -102,7 +102,7 @@ end function set_driver!(fapl::Properties, driver::ROS3) HDF5.init!(fapl) - API.h5p_set_fapl_ros3(fapl, driver.fa) + API.h5p_set_fapl_ros3(fapl, Ref(driver.fa)) DRIVERS[API.h5p_get_driver(fapl)] = ROS3 return nothing end From 29f99d5a2452fdede5e04646aca791e7d4a69dea Mon Sep 17 00:00:00 2001 From: Grisha Szep Date: Mon, 12 Sep 2022 10:37:09 +0100 Subject: [PATCH 04/16] Update src/drivers/drivers.jl Co-authored-by: Mark Kittisopikul --- src/drivers/drivers.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/drivers/drivers.jl b/src/drivers/drivers.jl index 0d4439680..26b2feaee 100644 --- a/src/drivers/drivers.jl +++ b/src/drivers/drivers.jl @@ -97,7 +97,9 @@ ROS3() = (ROS3 ∘ API.H5FD_ros3_fapl_t)( cconvert(s::AbstractString, N::Integer) = Tuple(i ≤ length(s) ? s[i] : '\0' for i in 1:N) function get_driver(fapl::Properties, ::Type{ROS3}) - ROS3() + r_fa = Ref{H5FD_ros3_fapl_t}() + H5Pget_fapl_ros3(fapl, r_fa) + return ROS3(r_fa[]) end function set_driver!(fapl::Properties, driver::ROS3) From b7b4c18890ab003cf36753899bb6f0bf8f8dd846 Mon Sep 17 00:00:00 2001 From: Grisha Szep Date: Mon, 12 Sep 2022 11:16:59 +0100 Subject: [PATCH 05/16] re-generate functions.jl and remove cchar hardcode --- docs/src/api_bindings.md | 8 ++++---- src/api/functions.jl | 12 ++++++------ src/api/types.jl | 6 +++--- src/drivers/drivers.jl | 24 +++++++++++++++++------- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/docs/src/api_bindings.md b/docs/src/api_bindings.md index 19dae7d28..12a87b70d 100644 --- a/docs/src/api_bindings.md +++ b/docs/src/api_bindings.md @@ -477,8 +477,8 @@ h5pl_size - [`h5p_get_fapl_mpio32`](@ref h5p_get_fapl_mpio32) - [`h5p_get_fapl_mpio64`](@ref h5p_get_fapl_mpio64) - [`h5p_get_fapl_multi`](@ref h5p_get_fapl_multi) +- [`h5p_get_fapl_ros`](@ref h5p_get_fapl_ros) - [`h5p_get_fapl_splitter`](@ref h5p_get_fapl_splitter) -- [`h5p_get_fapl_ros3`](@ref h5p_get_fapl_ros3) - [`h5p_get_fclose_degree`](@ref h5p_get_fclose_degree) - [`h5p_get_file_image`](@ref h5p_get_file_image) - [`h5p_get_file_image_callbacks`](@ref h5p_get_file_image_callbacks) @@ -580,8 +580,8 @@ h5pl_size - [`h5p_set_fapl_mpio32`](@ref h5p_set_fapl_mpio32) - [`h5p_set_fapl_mpio64`](@ref h5p_set_fapl_mpio64) - [`h5p_set_fapl_multi`](@ref h5p_set_fapl_multi) +- [`h5p_set_fapl_ros`](@ref h5p_set_fapl_ros) - [`h5p_set_fapl_sec2`](@ref h5p_set_fapl_sec2) -- [`h5p_set_fapl_ros3`](@ref h5p_set_fapl_ros3) - [`h5p_set_fapl_split`](@ref h5p_set_fapl_split) - [`h5p_set_fapl_splitter`](@ref h5p_set_fapl_splitter) - [`h5p_set_fapl_stdio`](@ref h5p_set_fapl_stdio) @@ -695,8 +695,8 @@ h5p_get_fapl_hdfs h5p_get_fapl_mpio32 h5p_get_fapl_mpio64 h5p_get_fapl_multi +h5p_get_fapl_ros h5p_get_fapl_splitter -h5p_get_fapl_ros3 h5p_get_fclose_degree h5p_get_file_image h5p_get_file_image_callbacks @@ -798,8 +798,8 @@ h5p_set_fapl_log h5p_set_fapl_mpio32 h5p_set_fapl_mpio64 h5p_set_fapl_multi +h5p_set_fapl_ros h5p_set_fapl_sec2 -h5p_set_fapl_ros3 h5p_set_fapl_split h5p_set_fapl_splitter h5p_set_fapl_stdio diff --git a/src/api/functions.jl b/src/api/functions.jl index a020c84a9..4ec8c0e3c 100644 --- a/src/api/functions.jl +++ b/src/api/functions.jl @@ -2367,12 +2367,12 @@ function h5p_get_fapl_splitter(fapl_id, config_ptr) end """ - h5p_get_fapl_ros3(fapl_id::hid_t, fa_out::Ptr{H5FD_hdfs_fapl_t}) + h5p_get_fapl_ros(fapl_id::hid_t, fa_out::Ptr{H5FD_ros3_fapl_t}) See `libhdf5` documentation for [`H5Pget_fapl_ros3`](https://portal.hdfgroup.org/display/HDF5/H5P_GET_FAPL_ROS3). """ -function h5p_get_fapl_ros3(fapl_id, fa_out) - var"#status#" = ccall((:H5Pget_fapl_ros3, libhdf5), herr_t, (hid_t, Ref{H5FD_ros3_fapl_t}), fapl_id, fa_out) +function h5p_get_fapl_ros(fapl_id, fa_out) + var"#status#" = ccall((:H5Pget_fapl_ros3, libhdf5), herr_t, (hid_t, Ptr{H5FD_ros3_fapl_t}), fapl_id, fa_out) var"#status#" < 0 && @h5error("Error in getting ros3 properties") return nothing end @@ -3423,12 +3423,12 @@ function h5p_set_fapl_sec2(fapl_id) end """ - h5p_set_fapl_ros3(fapl_id::hid_t, fa::Ptr{H5FD_ros3_fapl_t}) + h5p_set_fapl_ros(fapl_id::hid_t, fa::Ptr{H5FD_ros3_fapl_t}) See `libhdf5` documentation for [`H5Pset_fapl_ros3`](https://portal.hdfgroup.org/display/HDF5/H5P_SET_FAPL_ROS3). """ -function h5p_set_fapl_ros3(fapl_id, fa) - var"#status#" = ccall((:H5Pset_fapl_ros3, libhdf5), herr_t, (hid_t, Ref{H5FD_ros3_fapl_t}), fapl_id, fa) +function h5p_set_fapl_ros(fapl_id, fa) + var"#status#" = ccall((:H5Pset_fapl_ros3, libhdf5), herr_t, (hid_t, Ptr{H5FD_ros3_fapl_t}), fapl_id, fa) var"#status#" < 0 && @h5error("Error in setting ros3 properties") return nothing end diff --git a/src/api/types.jl b/src/api/types.jl index 4e95af298..bb2c16d89 100644 --- a/src/api/types.jl +++ b/src/api/types.jl @@ -225,9 +225,9 @@ end struct H5FD_ros3_fapl_t version::Int32 authenticate::hbool_t - aws_region::NTuple{33,Cchar} - secret_id::NTuple{129,Cchar} - secret_key::NTuple{129,Cchar} + aws_region::Cstring + secret_id::Cstring + secret_key::Cstring end struct H5FD_splitter_vfd_config_t diff --git a/src/drivers/drivers.jl b/src/drivers/drivers.jl index 26b2feaee..76e038009 100644 --- a/src/drivers/drivers.jl +++ b/src/drivers/drivers.jl @@ -87,24 +87,34 @@ This is the read-only virtual driver that enables access to HDF5 objects stored struct ROS3 <: Driver fa::API.H5FD_ros3_fapl_t end -ROS3(region::T, id::T, key::T) where {T<:AbstractString} = (ROS3 ∘ API.H5FD_ros3_fapl_t)( - 1, true, cconvert(region, 33), cconvert(id, 129), cconvert(key, 129) -) -ROS3() = (ROS3 ∘ API.H5FD_ros3_fapl_t)( - 1, false, cconvert("", 33), cconvert("", 129), cconvert("", 129) +ROS3( + version::Integer, + auth::Bool, + region::AbstractString, + id::AbstractString, + key::AbstractString +) = (ROS3 ∘ API.H5FD_ros3_fapl_t)( + version, + auth, + Base.unsafe_convert(Cstring, region), + Base.unsafe_convert(Cstring, id), + Base.unsafe_convert(Cstring, key) ) +ROS3(region::AbstractString, id::AbstractString, key::AbstractString) = + ROS3(1, true, region, id, key) +ROS3() = ROS3(1, false, "", "", "") cconvert(s::AbstractString, N::Integer) = Tuple(i ≤ length(s) ? s[i] : '\0' for i in 1:N) function get_driver(fapl::Properties, ::Type{ROS3}) r_fa = Ref{H5FD_ros3_fapl_t}() - H5Pget_fapl_ros3(fapl, r_fa) + API.h5p_get_fapl_ros(fapl, r_fa) return ROS3(r_fa[]) end function set_driver!(fapl::Properties, driver::ROS3) HDF5.init!(fapl) - API.h5p_set_fapl_ros3(fapl, Ref(driver.fa)) + API.h5p_set_fapl_ros(fapl, Ref(driver.fa)) DRIVERS[API.h5p_get_driver(fapl)] = ROS3 return nothing end From bbbb1e410a122f3cf6a3a0921718c081d21171cc Mon Sep 17 00:00:00 2001 From: Grisha Szep Date: Mon, 12 Sep 2022 11:22:26 +0100 Subject: [PATCH 06/16] remove local file existance check --- src/file.jl | 3 --- test/drivers.jl | 5 +++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/file.jl b/src/file.jl index 6674122b7..b06bd7b2a 100644 --- a/src/file.jl +++ b/src/file.jl @@ -42,9 +42,6 @@ function h5open( flag = swmr ? API.H5F_ACC_TRUNC | API.H5F_ACC_SWMR_WRITE : API.H5F_ACC_TRUNC fid = API.h5f_create(filename, flag, fcpl, fapl) else - ishdf5(filename) || error( - "unable to determine if $filename is accessible in the HDF5 format (file may not exist)" - ) if wr flag = swmr ? API.H5F_ACC_RDWR | API.H5F_ACC_SWMR_WRITE : API.H5F_ACC_RDWR else diff --git a/test/drivers.jl b/test/drivers.jl index 709871b60..98b0f0d9d 100644 --- a/test/drivers.jl +++ b/test/drivers.jl @@ -18,4 +18,9 @@ using Test ds = write_dataset(f, "core_dataset", A) end @test !isfile(fn) + + s3 = Drivers.ROS3() + h5open("http://s3.us-east-2.amazonaws.com/hdf5ros3/GMODO-SVM01.h5"; driver=s3) do f + @test keys(f) == ["All_Data", "Data_Products"] + end end From 02eff7def40f625887910a14e080048c31527d95 Mon Sep 17 00:00:00 2001 From: Grisha Szep Date: Mon, 12 Sep 2022 13:58:04 +0100 Subject: [PATCH 07/16] remove user-defined cconvert --- src/drivers/drivers.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/drivers/drivers.jl b/src/drivers/drivers.jl index 76e038009..a4499530e 100644 --- a/src/drivers/drivers.jl +++ b/src/drivers/drivers.jl @@ -104,8 +104,6 @@ ROS3(region::AbstractString, id::AbstractString, key::AbstractString) = ROS3(1, true, region, id, key) ROS3() = ROS3(1, false, "", "", "") -cconvert(s::AbstractString, N::Integer) = Tuple(i ≤ length(s) ? s[i] : '\0' for i in 1:N) - function get_driver(fapl::Properties, ::Type{ROS3}) r_fa = Ref{H5FD_ros3_fapl_t}() API.h5p_get_fapl_ros(fapl, r_fa) From 692ded091e10f41a43ff2d4de4e8bf1a626671e9 Mon Sep 17 00:00:00 2001 From: Grisha Szep Date: Mon, 12 Sep 2022 14:22:55 +0100 Subject: [PATCH 08/16] remove h5file existance test --- test/plain.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/test/plain.jl b/test/plain.jl index 991264205..e0be79963 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -1546,7 +1546,6 @@ end @test !HDF5.ishdf5(fn1) # check that a non-hdf5 file retuns false @test !HDF5.ishdf5(fn2) # checks that a file that does not exist returns false - @test_throws ErrorException h5write(fn1, "x", 1) # non hdf5 file throws h5write(fn2, "x", 1) @test HDF5.ishdf5(fn2) From aa89a7faa2037acff7806f9bc41437d4acd0fd01 Mon Sep 17 00:00:00 2001 From: Grisha Szep Date: Mon, 12 Sep 2022 22:01:06 +0100 Subject: [PATCH 09/16] lazy tests --- src/HDF5.jl | 8 ++++++++ src/drivers/drivers.jl | 43 +++--------------------------------------- src/drivers/ros3.jl | 32 +++++++++++++++++++++++++++++++ test/drivers.jl | 5 ----- test/mpio.jl | 1 - test/ros3.jl | 10 ++++++++++ test/runtests.jl | 4 ++++ 7 files changed, 57 insertions(+), 46 deletions(-) create mode 100644 src/drivers/ros3.jl create mode 100644 test/ros3.jl diff --git a/src/HDF5.jl b/src/HDF5.jl index ced25e703..f3f2526f1 100644 --- a/src/HDF5.jl +++ b/src/HDF5.jl @@ -85,6 +85,7 @@ include("highlevel.jl") const libversion = API.h5_get_libversion() const HAS_PARALLEL = Ref(false) +const HAS_ROS3 = Ref(false) """ has_parallel() @@ -96,6 +97,13 @@ For the second condition to be true, MPI.jl must be imported before HDF5.jl. """ has_parallel() = HAS_PARALLEL[] +""" + has_ros3() + +Returns `true` if the HDF5 libraries were compiled with ros3 support +""" +has_ros3() = HAS_ROS3[] + function __init__() # HDF5.API.__init__() is run first # diff --git a/src/drivers/drivers.jl b/src/drivers/drivers.jl index a4499530e..8e0ad4481 100644 --- a/src/drivers/drivers.jl +++ b/src/drivers/drivers.jl @@ -79,57 +79,20 @@ function set_driver!(fapl::Properties, ::POSIX) return nothing end -""" - ROS3() - -This is the read-only virtual driver that enables access to HDF5 objects stored in AWS S3 -""" -struct ROS3 <: Driver - fa::API.H5FD_ros3_fapl_t -end -ROS3( - version::Integer, - auth::Bool, - region::AbstractString, - id::AbstractString, - key::AbstractString -) = (ROS3 ∘ API.H5FD_ros3_fapl_t)( - version, - auth, - Base.unsafe_convert(Cstring, region), - Base.unsafe_convert(Cstring, id), - Base.unsafe_convert(Cstring, key) -) -ROS3(region::AbstractString, id::AbstractString, key::AbstractString) = - ROS3(1, true, region, id, key) -ROS3() = ROS3(1, false, "", "", "") - -function get_driver(fapl::Properties, ::Type{ROS3}) - r_fa = Ref{H5FD_ros3_fapl_t}() - API.h5p_get_fapl_ros(fapl, r_fa) - return ROS3(r_fa[]) -end - -function set_driver!(fapl::Properties, driver::ROS3) - HDF5.init!(fapl) - API.h5p_set_fapl_ros(fapl, Ref(driver.fa)) - DRIVERS[API.h5p_get_driver(fapl)] = ROS3 - return nothing -end - function __init__() - # Initialize POSIX,ROS3 keys in DRIVERS + # Initialize POSIX key in DRIVERS HDF5.FileAccessProperties() do fapl set_driver!(fapl, POSIX()) - set_driver!(fapl, ROS3()) end # Check whether the libhdf5 was compiled with parallel support. HDF5.HAS_PARALLEL[] = API._has_symbol(:H5Pset_fapl_mpio) + HDF5.HAS_ROS3[] = API._has_symbol(:H5Pset_fapl_ros3) @require MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" ( HDF5.has_parallel() && include("mpio.jl") ) end +include("ros3.jl") end # module diff --git a/src/drivers/ros3.jl b/src/drivers/ros3.jl new file mode 100644 index 000000000..aa256f360 --- /dev/null +++ b/src/drivers/ros3.jl @@ -0,0 +1,32 @@ +""" + ROS3() + +This is the read-only virtual driver that enables access to HDF5 objects stored in AWS S3 +""" +struct ROS3 <: Driver + fa::API.H5FD_ros3_fapl_t +end + +function ROS3( version::Integer, auth::Bool, region::AbstractString, id::AbstractString, key::AbstractString ) + return (ROS3 ∘ API.H5FD_ros3_fapl_t)( version, auth, Base.unsafe_convert(Cstring, region), Base.unsafe_convert(Cstring, id), Base.unsafe_convert(Cstring, key)) +end + +ROS3(region::AbstractString, id::AbstractString, key::AbstractString) = ROS3(1, true, region, id, key) +ROS3() = ROS3(1, false, "", "", "") + +function get_driver(fapl::Properties, ::Type{ROS3}) + r_fa = Ref{H5FD_ros3_fapl_t}() + API.h5p_get_fapl_ros(fapl, r_fa) + return ROS3(r_fa[]) +end + +function set_driver!(fapl::Properties, driver::ROS3) + HDF5.has_ros3() || error( + "HDF5.jl has no ros3 support." * + " Make sure that you're using ROS3-enabled HDF5 libraries" + ) + HDF5.init!(fapl) + API.h5p_set_fapl_ros(fapl, Ref(driver.fa)) + DRIVERS[API.h5p_get_driver(fapl)] = ROS3 + return nothing +end \ No newline at end of file diff --git a/test/drivers.jl b/test/drivers.jl index 98b0f0d9d..709871b60 100644 --- a/test/drivers.jl +++ b/test/drivers.jl @@ -18,9 +18,4 @@ using Test ds = write_dataset(f, "core_dataset", A) end @test !isfile(fn) - - s3 = Drivers.ROS3() - h5open("http://s3.us-east-2.amazonaws.com/hdf5ros3/GMODO-SVM01.h5"; driver=s3) do f - @test keys(f) == ["All_Data", "Data_Products"] - end end diff --git a/test/mpio.jl b/test/mpio.jl index ba4b48b9e..801f50853 100644 --- a/test/mpio.jl +++ b/test/mpio.jl @@ -20,7 +20,6 @@ using Test # Check that serial drivers are still there after loading MPI (#928) @test Drivers.Core ∈ values(Drivers.DRIVERS) @test Drivers.POSIX ∈ values(Drivers.DRIVERS) - @test Drivers.ROS3 ∈ values(Drivers.DRIVERS) let fileprop = HDF5.FileAccessProperties() fileprop.driver = HDF5.Drivers.MPIO(comm, info) diff --git a/test/ros3.jl b/test/ros3.jl new file mode 100644 index 000000000..e8ccc8e5d --- /dev/null +++ b/test/ros3.jl @@ -0,0 +1,10 @@ +using HDF5 +using Test + +@testset "ros3" begin + @test HDF5.has_ros3() + + h5open("http://s3.us-east-2.amazonaws.com/hdf5ros3/GMODO-SVM01.h5"; driver=HDF5.Drivers.ROS3()) do f + @test keys(f) == ["All_Data", "Data_Products"] + end +end diff --git a/test/runtests.jl b/test/runtests.jl index eb05d9352..dd5a1d6a3 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -69,6 +69,10 @@ end include("mpio.jl") end + if HDF5.has_ros3() + include("ros3.jl") + end + # Clean up after all resources HDF5.API.h5_close() end From 97f01134e8e7c27102af7df79fae35883d65ec23 Mon Sep 17 00:00:00 2001 From: Grisha Szep Date: Mon, 12 Sep 2022 22:15:58 +0100 Subject: [PATCH 10/16] obey formatter --- src/drivers/ros3.jl | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/drivers/ros3.jl b/src/drivers/ros3.jl index aa256f360..33eb54171 100644 --- a/src/drivers/ros3.jl +++ b/src/drivers/ros3.jl @@ -7,11 +7,24 @@ struct ROS3 <: Driver fa::API.H5FD_ros3_fapl_t end -function ROS3( version::Integer, auth::Bool, region::AbstractString, id::AbstractString, key::AbstractString ) - return (ROS3 ∘ API.H5FD_ros3_fapl_t)( version, auth, Base.unsafe_convert(Cstring, region), Base.unsafe_convert(Cstring, id), Base.unsafe_convert(Cstring, key)) +function ROS3( + version::Integer, + auth::Bool, + region::AbstractString, + id::AbstractString, + key::AbstractString +) + return (ROS3 ∘ API.H5FD_ros3_fapl_t)( + version, + auth, + Base.unsafe_convert(Cstring, region), + Base.unsafe_convert(Cstring, id), + Base.unsafe_convert(Cstring, key) + ) end -ROS3(region::AbstractString, id::AbstractString, key::AbstractString) = ROS3(1, true, region, id, key) +ROS3(region::AbstractString, id::AbstractString, key::AbstractString) = + ROS3(1, true, region, id, key) ROS3() = ROS3(1, false, "", "", "") function get_driver(fapl::Properties, ::Type{ROS3}) From 6aa4810fb18cbccc04a45dccb88ca125e2ae9fc8 Mon Sep 17 00:00:00 2001 From: Grisha Szep Date: Mon, 12 Sep 2022 22:19:25 +0100 Subject: [PATCH 11/16] obey formatter --- test/ros3.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/ros3.jl b/test/ros3.jl index e8ccc8e5d..5a4d8e8ac 100644 --- a/test/ros3.jl +++ b/test/ros3.jl @@ -4,7 +4,10 @@ using Test @testset "ros3" begin @test HDF5.has_ros3() - h5open("http://s3.us-east-2.amazonaws.com/hdf5ros3/GMODO-SVM01.h5"; driver=HDF5.Drivers.ROS3()) do f + h5open( + "http://s3.us-east-2.amazonaws.com/hdf5ros3/GMODO-SVM01.h5"; + driver=HDF5.Drivers.ROS3() + ) do f @test keys(f) == ["All_Data", "Data_Products"] end end From 877f0ac830f57ca629a17b1dcc7e36da37d46dc8 Mon Sep 17 00:00:00 2001 From: Grisha Szep Date: Mon, 12 Sep 2022 23:14:11 +0100 Subject: [PATCH 12/16] restore file existance tests --- src/file.jl | 5 +++++ test/plain.jl | 1 + 2 files changed, 6 insertions(+) diff --git a/src/file.jl b/src/file.jl index b06bd7b2a..c2f3afa9d 100644 --- a/src/file.jl +++ b/src/file.jl @@ -42,6 +42,11 @@ function h5open( flag = swmr ? API.H5F_ACC_TRUNC | API.H5F_ACC_SWMR_WRITE : API.H5F_ACC_TRUNC fid = API.h5f_create(filename, flag, fcpl, fapl) else + occursin(r"(s3a?|https?)://", filename) || + ishdf5(filename) || + error( + "unable to determine if $filename is accessible in the HDF5 format (file may not exist)" + ) if wr flag = swmr ? API.H5F_ACC_RDWR | API.H5F_ACC_SWMR_WRITE : API.H5F_ACC_RDWR else diff --git a/test/plain.jl b/test/plain.jl index e0be79963..991264205 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -1546,6 +1546,7 @@ end @test !HDF5.ishdf5(fn1) # check that a non-hdf5 file retuns false @test !HDF5.ishdf5(fn2) # checks that a file that does not exist returns false + @test_throws ErrorException h5write(fn1, "x", 1) # non hdf5 file throws h5write(fn2, "x", 1) @test HDF5.ishdf5(fn2) From 6fdeaa7a4d2e5485d87c8f06181fddd66d29f52e Mon Sep 17 00:00:00 2001 From: Grisha Szep Date: Mon, 12 Sep 2022 23:19:14 +0100 Subject: [PATCH 13/16] obey formatter --- src/drivers/ros3.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/drivers/ros3.jl b/src/drivers/ros3.jl index 33eb54171..8c9e9d83b 100644 --- a/src/drivers/ros3.jl +++ b/src/drivers/ros3.jl @@ -42,4 +42,4 @@ function set_driver!(fapl::Properties, driver::ROS3) API.h5p_set_fapl_ros(fapl, Ref(driver.fa)) DRIVERS[API.h5p_get_driver(fapl)] = ROS3 return nothing -end \ No newline at end of file +end From cc6625e33e6bce33b5ef0cc5d3ee43d14a0e795a Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Wed, 14 Sep 2022 17:15:43 -0400 Subject: [PATCH 14/16] Make a bind exception for h5p_get_fapl_ros3 and h5p_set_fapl_ros3 --- docs/src/api_bindings.md | 8 ++++---- gen/bind_generator.jl | 2 ++ src/api/functions.jl | 8 ++++---- src/drivers/ros3.jl | 4 ++-- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/src/api_bindings.md b/docs/src/api_bindings.md index 12a87b70d..d2c56f49d 100644 --- a/docs/src/api_bindings.md +++ b/docs/src/api_bindings.md @@ -477,7 +477,7 @@ h5pl_size - [`h5p_get_fapl_mpio32`](@ref h5p_get_fapl_mpio32) - [`h5p_get_fapl_mpio64`](@ref h5p_get_fapl_mpio64) - [`h5p_get_fapl_multi`](@ref h5p_get_fapl_multi) -- [`h5p_get_fapl_ros`](@ref h5p_get_fapl_ros) +- [`h5p_get_fapl_ros3`](@ref h5p_get_fapl_ros3) - [`h5p_get_fapl_splitter`](@ref h5p_get_fapl_splitter) - [`h5p_get_fclose_degree`](@ref h5p_get_fclose_degree) - [`h5p_get_file_image`](@ref h5p_get_file_image) @@ -580,7 +580,7 @@ h5pl_size - [`h5p_set_fapl_mpio32`](@ref h5p_set_fapl_mpio32) - [`h5p_set_fapl_mpio64`](@ref h5p_set_fapl_mpio64) - [`h5p_set_fapl_multi`](@ref h5p_set_fapl_multi) -- [`h5p_set_fapl_ros`](@ref h5p_set_fapl_ros) +- [`h5p_set_fapl_ros3`](@ref h5p_set_fapl_ros3) - [`h5p_set_fapl_sec2`](@ref h5p_set_fapl_sec2) - [`h5p_set_fapl_split`](@ref h5p_set_fapl_split) - [`h5p_set_fapl_splitter`](@ref h5p_set_fapl_splitter) @@ -695,7 +695,7 @@ h5p_get_fapl_hdfs h5p_get_fapl_mpio32 h5p_get_fapl_mpio64 h5p_get_fapl_multi -h5p_get_fapl_ros +h5p_get_fapl_ros3 h5p_get_fapl_splitter h5p_get_fclose_degree h5p_get_file_image @@ -798,7 +798,7 @@ h5p_set_fapl_log h5p_set_fapl_mpio32 h5p_set_fapl_mpio64 h5p_set_fapl_multi -h5p_set_fapl_ros +h5p_set_fapl_ros3 h5p_set_fapl_sec2 h5p_set_fapl_split h5p_set_fapl_splitter diff --git a/gen/bind_generator.jl b/gen/bind_generator.jl index a8b958874..695d16592 100644 --- a/gen/bind_generator.jl +++ b/gen/bind_generator.jl @@ -12,6 +12,8 @@ bind_exceptions[:h5p_set_fapl_mpio64] = :H5Pset_fapl_mpio # have numbers at the end bind_exceptions[:h5p_set_fletcher32] = :H5Pset_fletcher32 bind_exceptions[:h5p_set_fapl_sec2] = :H5Pset_fapl_sec2 +bind_exceptions[:h5p_get_fapl_ros3] = :H5Pget_fapl_ros3 +bind_exceptions[:h5p_set_fapl_ros3] = :H5Pset_fapl_ros3 # underscore separator not removed bind_exceptions[:h5fd_core_init] = :H5FD_core_init bind_exceptions[:h5fd_family_init] = :H5FD_family_init diff --git a/src/api/functions.jl b/src/api/functions.jl index 4ec8c0e3c..4d744e9df 100644 --- a/src/api/functions.jl +++ b/src/api/functions.jl @@ -2367,11 +2367,11 @@ function h5p_get_fapl_splitter(fapl_id, config_ptr) end """ - h5p_get_fapl_ros(fapl_id::hid_t, fa_out::Ptr{H5FD_ros3_fapl_t}) + h5p_get_fapl_ros3(fapl_id::hid_t, fa_out::Ptr{H5FD_ros3_fapl_t}) See `libhdf5` documentation for [`H5Pget_fapl_ros3`](https://portal.hdfgroup.org/display/HDF5/H5P_GET_FAPL_ROS3). """ -function h5p_get_fapl_ros(fapl_id, fa_out) +function h5p_get_fapl_ros3(fapl_id, fa_out) var"#status#" = ccall((:H5Pget_fapl_ros3, libhdf5), herr_t, (hid_t, Ptr{H5FD_ros3_fapl_t}), fapl_id, fa_out) var"#status#" < 0 && @h5error("Error in getting ros3 properties") return nothing @@ -3423,11 +3423,11 @@ function h5p_set_fapl_sec2(fapl_id) end """ - h5p_set_fapl_ros(fapl_id::hid_t, fa::Ptr{H5FD_ros3_fapl_t}) + h5p_set_fapl_ros3(fapl_id::hid_t, fa::Ptr{H5FD_ros3_fapl_t}) See `libhdf5` documentation for [`H5Pset_fapl_ros3`](https://portal.hdfgroup.org/display/HDF5/H5P_SET_FAPL_ROS3). """ -function h5p_set_fapl_ros(fapl_id, fa) +function h5p_set_fapl_ros3(fapl_id, fa) var"#status#" = ccall((:H5Pset_fapl_ros3, libhdf5), herr_t, (hid_t, Ptr{H5FD_ros3_fapl_t}), fapl_id, fa) var"#status#" < 0 && @h5error("Error in setting ros3 properties") return nothing diff --git a/src/drivers/ros3.jl b/src/drivers/ros3.jl index 8c9e9d83b..739802773 100644 --- a/src/drivers/ros3.jl +++ b/src/drivers/ros3.jl @@ -29,7 +29,7 @@ ROS3() = ROS3(1, false, "", "", "") function get_driver(fapl::Properties, ::Type{ROS3}) r_fa = Ref{H5FD_ros3_fapl_t}() - API.h5p_get_fapl_ros(fapl, r_fa) + API.h5p_get_fapl_ros3(fapl, r_fa) return ROS3(r_fa[]) end @@ -39,7 +39,7 @@ function set_driver!(fapl::Properties, driver::ROS3) " Make sure that you're using ROS3-enabled HDF5 libraries" ) HDF5.init!(fapl) - API.h5p_set_fapl_ros(fapl, Ref(driver.fa)) + API.h5p_set_fapl_ros3(fapl, Ref(driver.fa)) DRIVERS[API.h5p_get_driver(fapl)] = ROS3 return nothing end From b904ce634a85e27b8704206c793bb031cae9312a Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Fri, 23 Sep 2022 03:27:15 -0400 Subject: [PATCH 15/16] Refactor ROS3 driver interface to match C definitions --- src/api/types.jl | 12 ++++++--- src/drivers/ros3.jl | 60 +++++++++++++++++++++++++++++++-------------- 2 files changed, 49 insertions(+), 23 deletions(-) diff --git a/src/api/types.jl b/src/api/types.jl index bb2c16d89..84f32e7b3 100644 --- a/src/api/types.jl +++ b/src/api/types.jl @@ -3,7 +3,7 @@ const Ctime_t = Int ## HDF5 types and constants const haddr_t = UInt64 -const hbool_t = Cuint +const hbool_t = Bool const herr_t = Cint const hid_t = Int64 const hsize_t = UInt64 @@ -222,12 +222,16 @@ struct H5FD_hdfs_fapl_t stream_buffer_size::Int32 end +const H5FD_ROS3_MAX_REGION_LEN = 32 +const H5FD_ROS3_MAX_SECRET_ID_LEN = 128 +const H5FD_ROS3_MAX_SECRET_KEY_LEN = 128 + struct H5FD_ros3_fapl_t version::Int32 authenticate::hbool_t - aws_region::Cstring - secret_id::Cstring - secret_key::Cstring + aws_region::NTuple{H5FD_ROS3_MAX_REGION_LEN + 1,Cchar} + secret_id::NTuple{H5FD_ROS3_MAX_SECRET_ID_LEN + 1,Cchar} + secret_key::NTuple{H5FD_ROS3_MAX_SECRET_KEY_LEN + 1,Cchar} end struct H5FD_splitter_vfd_config_t diff --git a/src/drivers/ros3.jl b/src/drivers/ros3.jl index 739802773..ae714bbf0 100644 --- a/src/drivers/ros3.jl +++ b/src/drivers/ros3.jl @@ -4,28 +4,50 @@ This is the read-only virtual driver that enables access to HDF5 objects stored in AWS S3 """ struct ROS3 <: Driver - fa::API.H5FD_ros3_fapl_t -end - -function ROS3( - version::Integer, - auth::Bool, - region::AbstractString, - id::AbstractString, - key::AbstractString -) - return (ROS3 ∘ API.H5FD_ros3_fapl_t)( - version, - auth, - Base.unsafe_convert(Cstring, region), - Base.unsafe_convert(Cstring, id), - Base.unsafe_convert(Cstring, key) - ) + version::Int32 + authenticate::Bool + aws_region::String + secret_id::String + secret_key::String + function ROS3(version, authenticate, aws_region, secret_id, secret_key) + length(aws_region) <= API.H5FD_ROS3_MAX_REGION_LEN || + "length(aws_region), $(length(aws_region)), must be less than or equal to $(API.H5FD_ROS3_MAX_REGION_LEN)" + length(secret_id) <= API.H5FD_ROS3_MAX_SECRET_ID_LEN || + "length(secret_id), $(length(secret_id)), must be less than or equal to $(API.H5FD_ROS3_MAX_SECRET_ID_LEN)" + length(secret_key) <= API.H5FD_ROS3_MAX_SECRET_KEY_LEN || + "length(secret_key), $(length(secret_key)), must be less than or equal to $(API.H5FD_ROS3_MAX_SECRET_KEY_LEN)" + new(version, authenticate, aws_region, secret_id, secret_key) + end end +ROS3() = ROS3(1, false, "", "", "") ROS3(region::AbstractString, id::AbstractString, key::AbstractString) = ROS3(1, true, region, id, key) -ROS3() = ROS3(1, false, "", "", "") +_ntuple_to_string(x) = unsafe_string(Ptr{Cchar}(pointer_from_objref(Ref(x))), length(x)) +function ROS3(driver::API.H5FD_ros3_fapl_t) + aws_region = _ntuple_to_string(driver.aws_region) + secret_id = _ntuple_to_string(driver.secret_id) + secret_key = _ntuple_to_string(driver.secret_key) + ROS3(driver.version, driver.authenticate, aws_region, secret_id, secret_key) +end + +function Base.convert(::Type{API.H5FD_ros3_fapl_t}, driver::ROS3) + aws_region = ntuple( + i -> i <= length(driver.aws_region) ? Cchar(driver.aws_region[i]) : Cchar(0x0), + API.H5FD_ROS3_MAX_REGION_LEN + 1 + ) + secret_id = ntuple( + i -> i <= length(driver.secret_id) ? Cchar(driver.secret_id[i]) : Cchar(0x0), + API.H5FD_ROS3_MAX_SECRET_ID_LEN + 1 + ) + secret_key = ntuple( + i -> i <= length(driver.secret_key) ? Cchar(driver.secret_key[i]) : Cchar(0x0), + API.H5FD_ROS3_MAX_SECRET_KEY_LEN + 1 + ) + s = API.H5FD_ros3_fapl_t( + driver.version, driver.authenticate, aws_region, secret_id, secret_key, + ) +end function get_driver(fapl::Properties, ::Type{ROS3}) r_fa = Ref{H5FD_ros3_fapl_t}() @@ -39,7 +61,7 @@ function set_driver!(fapl::Properties, driver::ROS3) " Make sure that you're using ROS3-enabled HDF5 libraries" ) HDF5.init!(fapl) - API.h5p_set_fapl_ros3(fapl, Ref(driver.fa)) + API.h5p_set_fapl_ros3(fapl, Ref{API.H5FD_ros3_fapl_t}(driver)) DRIVERS[API.h5p_get_driver(fapl)] = ROS3 return nothing end From 6d41c990decaddf3845c04e34dd41228d688fb01 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Fri, 23 Sep 2022 04:24:19 -0400 Subject: [PATCH 16/16] Update docstring, reorganize --- src/drivers/ros3.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/drivers/ros3.jl b/src/drivers/ros3.jl index ae714bbf0..85f762a26 100644 --- a/src/drivers/ros3.jl +++ b/src/drivers/ros3.jl @@ -1,5 +1,7 @@ """ ROS3() + ROS3(aws_region::String, secret_id::String, secret_key::String) + ROS3(version::Int32, authenticate::Bool, aws_region::String, secret_id::String, secret_key::String) This is the read-only virtual driver that enables access to HDF5 objects stored in AWS S3 """ @@ -21,15 +23,17 @@ struct ROS3 <: Driver end ROS3() = ROS3(1, false, "", "", "") + ROS3(region::AbstractString, id::AbstractString, key::AbstractString) = ROS3(1, true, region, id, key) -_ntuple_to_string(x) = unsafe_string(Ptr{Cchar}(pointer_from_objref(Ref(x))), length(x)) + function ROS3(driver::API.H5FD_ros3_fapl_t) aws_region = _ntuple_to_string(driver.aws_region) secret_id = _ntuple_to_string(driver.secret_id) secret_key = _ntuple_to_string(driver.secret_key) ROS3(driver.version, driver.authenticate, aws_region, secret_id, secret_key) end +_ntuple_to_string(x) = unsafe_string(Ptr{Cchar}(pointer_from_objref(Ref(x))), length(x)) function Base.convert(::Type{API.H5FD_ros3_fapl_t}, driver::ROS3) aws_region = ntuple(