add support for read-only s3 virtual driver

docs/src/api_bindings.md

@@ -477,6 +477,7 @@ h5pl_size
 - [`h5p_get_fapl_mpio32`](@ref h5p_get_fapl_mpio32)
 - [`h5p_get_fapl_mpio64`](@ref h5p_get_fapl_mpio64)
 - [`h5p_get_fapl_multi`](@ref h5p_get_fapl_multi)
+- [`h5p_get_fapl_ros3`](@ref h5p_get_fapl_ros3)
 - [`h5p_get_fapl_splitter`](@ref h5p_get_fapl_splitter)
 - [`h5p_get_fclose_degree`](@ref h5p_get_fclose_degree)
 - [`h5p_get_file_image`](@ref h5p_get_file_image)
@@ -579,6 +580,7 @@ h5pl_size
 - [`h5p_set_fapl_mpio32`](@ref h5p_set_fapl_mpio32)
 - [`h5p_set_fapl_mpio64`](@ref h5p_set_fapl_mpio64)
 - [`h5p_set_fapl_multi`](@ref h5p_set_fapl_multi)
+- [`h5p_set_fapl_ros3`](@ref h5p_set_fapl_ros3)
 - [`h5p_set_fapl_sec2`](@ref h5p_set_fapl_sec2)
 - [`h5p_set_fapl_split`](@ref h5p_set_fapl_split)
 - [`h5p_set_fapl_splitter`](@ref h5p_set_fapl_splitter)
@@ -693,6 +695,7 @@ h5p_get_fapl_hdfs
 h5p_get_fapl_mpio32
 h5p_get_fapl_mpio64
 h5p_get_fapl_multi
+h5p_get_fapl_ros3
 h5p_get_fapl_splitter
 h5p_get_fclose_degree
 h5p_get_file_image
@@ -795,6 +798,7 @@ h5p_set_fapl_log
 h5p_set_fapl_mpio32
 h5p_set_fapl_mpio64
 h5p_set_fapl_multi
+h5p_set_fapl_ros3
 h5p_set_fapl_sec2
 h5p_set_fapl_split
 h5p_set_fapl_splitter

docs/src/interface/properties.md

@@ -44,5 +44,6 @@ CurrentModule = HDF5.Drivers
 ```@docs
 Core
 POSIX
+ROS3
 MPIO
 ```

gen/api_defs.jl

@@ -285,6 +285,7 @@
 @bind h5p_get_fapl_mpio64(fapl_id::hid_t, comm::Ptr{Hmpih64}, info::Ptr{Hmpih64})::herr_t "Error getting MPIO properties"
 @bind h5p_get_fapl_multi(fapl_id::hid_t, memb_map::Ptr{H5FD_mem_t}, memb_fapl::Ptr{hid_t}, memb_name::Ptr{Ptr{Cchar}}, memb_addr::Ptr{haddr_t}, relax::Ptr{hbool_t})::herr_t "Error in h5p_get_fapl_multi (not annotated)"
 @bind h5p_get_fapl_splitter(fapl_id::hid_t, config_ptr::Ptr{H5FD_splitter_vfd_config_t})::herr_t "Error in h5p_get_fapl_splitter (not annotated)"
+@bind h5p_get_fapl_ros3(fapl_id::hid_t, fa_out::Ptr{H5FD_ros3_fapl_t})::herr_t "Error in getting ros3 properties"
 @bind h5p_get_fclose_degree(fapl_id::hid_t, fc_degree::Ref{Cint})::herr_t "Error getting close degree"
 @bind h5p_get_file_image(fapl_id::hid_t, buf_ptr_ptr::Ptr{Ptr{Cvoid}}, buf_len_ptr::Ptr{Csize_t})::herr_t "Error in h5p_get_file_image (not annotated)"
 @bind h5p_get_file_image_callbacks(fapl_id::hid_t, callbacks_ptr::Ptr{H5FD_file_image_callbacks_t})::herr_t "Error in h5p_get_file_image_callbacks (not annotated)"
@@ -382,6 +383,7 @@
 @bind h5p_set_fapl_mpio64(fapl_id::hid_t, comm::Hmpih64, info::Hmpih64)::herr_t "Error setting MPIO properties"
 @bind h5p_set_fapl_multi(fapl_id::hid_t, memb_map::Ptr{H5FD_mem_t}, memb_fapl::Ptr{hid_t}, memb_name::Ptr{Ptr{Cchar}}, memb_addr::Ptr{haddr_t}, relax::hbool_t)::herr_t "Error in h5p_set_fapl_multi (not annotated)"
 @bind h5p_set_fapl_sec2(fapl_id::hid_t)::herr_t "Error setting Sec2 properties"
+@bind h5p_set_fapl_ros3(fapl_id::hid_t, fa::Ptr{H5FD_ros3_fapl_t})::herr_t "Error in setting ros3 properties"
 @bind h5p_set_fapl_split(fapl::hid_t, meta_ext::Ptr{Cchar}, meta_plist_id::hid_t, raw_ext::Ptr{Cchar}, raw_plist_id::hid_t)::herr_t "Error in h5p_set_fapl_split (not annotated)"
 @bind h5p_set_fapl_splitter(fapl_id::hid_t, config_ptr::Ptr{H5FD_splitter_vfd_config_t})::herr_t "Error in h5p_set_fapl_splitter (not annotated)"
 @bind h5p_set_fapl_stdio(fapl_id::hid_t)::herr_t "Error in h5p_set_fapl_stdio (not annotated)"

gen/bind_generator.jl

@@ -12,6 +12,8 @@ bind_exceptions[:h5p_set_fapl_mpio64] = :H5Pset_fapl_mpio
 # have numbers at the end
 bind_exceptions[:h5p_set_fletcher32] = :H5Pset_fletcher32
 bind_exceptions[:h5p_set_fapl_sec2]  = :H5Pset_fapl_sec2
+bind_exceptions[:h5p_get_fapl_ros3]  = :H5Pget_fapl_ros3
+bind_exceptions[:h5p_set_fapl_ros3]  = :H5Pset_fapl_ros3
 # underscore separator not removed
 bind_exceptions[:h5fd_core_init]   = :H5FD_core_init
 bind_exceptions[:h5fd_family_init] = :H5FD_family_init

src/HDF5.jl

@@ -85,6 +85,7 @@ include("highlevel.jl")
 
 const libversion = API.h5_get_libversion()
 const HAS_PARALLEL = Ref(false)
+const HAS_ROS3 = Ref(false)
 
 """
     has_parallel()
@@ -96,6 +97,13 @@ For the second condition to be true, MPI.jl must be imported before HDF5.jl.
 """
 has_parallel() = HAS_PARALLEL[]
 
+"""
+    has_ros3()
+
+Returns `true` if the HDF5 libraries were compiled with ros3 support
+"""
+has_ros3() = HAS_ROS3[]
+
 function __init__()
     # HDF5.API.__init__() is run first
     #

src/api/functions.jl

@@ -2366,6 +2366,17 @@ function h5p_get_fapl_splitter(fapl_id, config_ptr)
     return nothing
 end
 
+"""
+    h5p_get_fapl_ros3(fapl_id::hid_t, fa_out::Ptr{H5FD_ros3_fapl_t})
+
+See `libhdf5` documentation for [`H5Pget_fapl_ros3`](https://portal.hdfgroup.org/display/HDF5/H5P_GET_FAPL_ROS3).
+"""
+function h5p_get_fapl_ros3(fapl_id, fa_out)
+    var"#status#" = ccall((:H5Pget_fapl_ros3, libhdf5), herr_t, (hid_t, Ptr{H5FD_ros3_fapl_t}), fapl_id, fa_out)
+    var"#status#" < 0 && @h5error("Error in getting ros3 properties")
+    return nothing
+end
+
 """
     h5p_get_fclose_degree(fapl_id::hid_t, fc_degree::Ref{Cint})
 
@@ -3411,6 +3422,17 @@ function h5p_set_fapl_sec2(fapl_id)
     return nothing
 end
 
+"""
+    h5p_set_fapl_ros3(fapl_id::hid_t, fa::Ptr{H5FD_ros3_fapl_t})
+
+See `libhdf5` documentation for [`H5Pset_fapl_ros3`](https://portal.hdfgroup.org/display/HDF5/H5P_SET_FAPL_ROS3).
+"""
+function h5p_set_fapl_ros3(fapl_id, fa)
+    var"#status#" = ccall((:H5Pset_fapl_ros3, libhdf5), herr_t, (hid_t, Ptr{H5FD_ros3_fapl_t}), fapl_id, fa)
+    var"#status#" < 0 && @h5error("Error in setting ros3 properties")
+    return nothing
+end
+
 """
     h5p_set_fapl_split(fapl::hid_t, meta_ext::Ptr{Cchar}, meta_plist_id::hid_t, raw_ext::Ptr{Cchar}, raw_plist_id::hid_t)
 

src/api/types.jl

@@ -3,7 +3,7 @@ const Ctime_t = Int
 
 ## HDF5 types and constants
 const haddr_t  = UInt64
-const hbool_t  = Cuint
+const hbool_t  = Bool
 const herr_t   = Cint
 const hid_t    = Int64
 const hsize_t  = UInt64
@@ -222,6 +222,18 @@ struct H5FD_hdfs_fapl_t
     stream_buffer_size::Int32
 end
 
+const H5FD_ROS3_MAX_REGION_LEN = 32
+const H5FD_ROS3_MAX_SECRET_ID_LEN = 128
+const H5FD_ROS3_MAX_SECRET_KEY_LEN = 128
+
+struct H5FD_ros3_fapl_t
+    version::Int32
+    authenticate::hbool_t
+    aws_region::NTuple{H5FD_ROS3_MAX_REGION_LEN + 1,Cchar}
+    secret_id::NTuple{H5FD_ROS3_MAX_SECRET_ID_LEN + 1,Cchar}
+    secret_key::NTuple{H5FD_ROS3_MAX_SECRET_KEY_LEN + 1,Cchar}
+end
+
 struct H5FD_splitter_vfd_config_t
     magic::Int32
     version::Cuint

src/drivers/drivers.jl

@@ -1,6 +1,6 @@
 module Drivers
 
-export POSIX
+export POSIX, ROS3
 
 import ..API
 import ..HDF5: HDF5, Properties, h5doc
@@ -87,10 +87,12 @@ function __init__()
 
     # Check whether the libhdf5 was compiled with parallel support.
     HDF5.HAS_PARALLEL[] = API._has_symbol(:H5Pset_fapl_mpio)
+    HDF5.HAS_ROS3[] = API._has_symbol(:H5Pset_fapl_ros3)
 
     @require MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" (
         HDF5.has_parallel() && include("mpio.jl")
     )
 end
 
+include("ros3.jl")
 end # module

src/drivers/ros3.jl

@@ -0,0 +1,71 @@
+"""
+    ROS3()
+    ROS3(aws_region::String, secret_id::String, secret_key::String)
+    ROS3(version::Int32, authenticate::Bool, aws_region::String, secret_id::String, secret_key::String)
+
+This is the read-only virtual driver that enables access to HDF5 objects stored in AWS S3
+"""
+struct ROS3 <: Driver
+    version::Int32
+    authenticate::Bool
+    aws_region::String
+    secret_id::String
+    secret_key::String
+    function ROS3(version, authenticate, aws_region, secret_id, secret_key)
+        length(aws_region) <= API.H5FD_ROS3_MAX_REGION_LEN ||
+            "length(aws_region), $(length(aws_region)), must be less than or equal to $(API.H5FD_ROS3_MAX_REGION_LEN)"
+        length(secret_id) <= API.H5FD_ROS3_MAX_SECRET_ID_LEN ||
+            "length(secret_id), $(length(secret_id)), must be less than or equal to $(API.H5FD_ROS3_MAX_SECRET_ID_LEN)"
+        length(secret_key) <= API.H5FD_ROS3_MAX_SECRET_KEY_LEN ||
+            "length(secret_key), $(length(secret_key)), must be less than or equal to $(API.H5FD_ROS3_MAX_SECRET_KEY_LEN)"
+        new(version, authenticate, aws_region, secret_id, secret_key)
+    end
+end
+
+ROS3() = ROS3(1, false, "", "", "")
+
+ROS3(region::AbstractString, id::AbstractString, key::AbstractString) =
+    ROS3(1, true, region, id, key)
+
+function ROS3(driver::API.H5FD_ros3_fapl_t)
+    aws_region = _ntuple_to_string(driver.aws_region)
+    secret_id = _ntuple_to_string(driver.secret_id)
+    secret_key = _ntuple_to_string(driver.secret_key)
+    ROS3(driver.version, driver.authenticate, aws_region, secret_id, secret_key)
+end
+_ntuple_to_string(x) = unsafe_string(Ptr{Cchar}(pointer_from_objref(Ref(x))), length(x))
+
+function Base.convert(::Type{API.H5FD_ros3_fapl_t}, driver::ROS3)
+    aws_region = ntuple(
+        i -> i <= length(driver.aws_region) ? Cchar(driver.aws_region[i]) : Cchar(0x0),
+        API.H5FD_ROS3_MAX_REGION_LEN + 1
+    )
+    secret_id = ntuple(
+        i -> i <= length(driver.secret_id) ? Cchar(driver.secret_id[i]) : Cchar(0x0),
+        API.H5FD_ROS3_MAX_SECRET_ID_LEN + 1
+    )
+    secret_key = ntuple(
+        i -> i <= length(driver.secret_key) ? Cchar(driver.secret_key[i]) : Cchar(0x0),
+        API.H5FD_ROS3_MAX_SECRET_KEY_LEN + 1
+    )
+    s = API.H5FD_ros3_fapl_t(
+        driver.version, driver.authenticate, aws_region, secret_id, secret_key,
+    )
+end
+
+function get_driver(fapl::Properties, ::Type{ROS3})
+    r_fa = Ref{H5FD_ros3_fapl_t}()
+    API.h5p_get_fapl_ros3(fapl, r_fa)
+    return ROS3(r_fa[])
+end
+
+function set_driver!(fapl::Properties, driver::ROS3)
+    HDF5.has_ros3() || error(
+        "HDF5.jl has no ros3 support." *
+        " Make sure that you're using ROS3-enabled HDF5 libraries"
+    )
+    HDF5.init!(fapl)
+    API.h5p_set_fapl_ros3(fapl, Ref{API.H5FD_ros3_fapl_t}(driver))
+    DRIVERS[API.h5p_get_driver(fapl)] = ROS3
+    return nothing
+end

src/file.jl

@@ -42,9 +42,11 @@ function h5open(
         flag = swmr ? API.H5F_ACC_TRUNC | API.H5F_ACC_SWMR_WRITE : API.H5F_ACC_TRUNC
         fid = API.h5f_create(filename, flag, fcpl, fapl)
     else
-        ishdf5(filename) || error(
-            "unable to determine if $filename is accessible in the HDF5 format (file may not exist)"
-        )
+        occursin(r"(s3a?|https?)://", filename) ||
+            ishdf5(filename) ||
+            error(
+                "unable to determine if $filename is accessible in the HDF5 format (file may not exist)"
+            )
         if wr
             flag = swmr ? API.H5F_ACC_RDWR | API.H5F_ACC_SWMR_WRITE : API.H5F_ACC_RDWR
         else

test/ros3.jl

@@ -0,0 +1,13 @@
+using HDF5
+using Test
+
+@testset "ros3" begin
+    @test HDF5.has_ros3()
+
+    h5open(
+        "http://s3.us-east-2.amazonaws.com/hdf5ros3/GMODO-SVM01.h5";
+        driver=HDF5.Drivers.ROS3()
+    ) do f
+        @test keys(f) == ["All_Data", "Data_Products"]
+    end
+end

test/runtests.jl

@@ -69,6 +69,10 @@ end
         include("mpio.jl")
     end
 
+    if HDF5.has_ros3()
+        include("ros3.jl")
+    end
+
     # Clean up after all resources
     HDF5.API.h5_close()
 end