diff --git a/docs/make.jl b/docs/make.jl index 36356ff9e..286f5f020 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -6,6 +6,8 @@ using H5Zlz4 using H5Zzstd using MPI # needed to generate docs for parallel HDF5 API +DocMeta.setdocmeta!(HDF5, :DocTestSetup, :(using HDF5); recursive=true) + makedocs(; sitename="HDF5.jl", modules=[HDF5, H5Zblosc, H5Zbzip2, H5Zlz4, H5Zzstd], diff --git a/docs/src/interface/filters.md b/docs/src/interface/filters.md index 88af03692..99611a677 100644 --- a/docs/src/interface/filters.md +++ b/docs/src/interface/filters.md @@ -125,6 +125,12 @@ h5open(filename, "w") do h5f end ``` +### Registered Filter Helpers + +The HDF Group maintains a list of registered filters which have been assigned a filter ID number. +The module [`Filters.Registered`](@ref) contains information about registered filters including functions +to create an `ExternalFilter` for each registered filter. + ### Creating a new Filter type Examining the [bitshuffle filter source code](https://github.com/kiyo-masui/bitshuffle/blob/0aee87e142c71407aa097c660727f2621c71c493/src/bshuf_h5filter.c#L47-L64) we see that three additional data components get prepended to the options. These are @@ -192,6 +198,12 @@ filter_cfunc register_filter ``` +## Registered Filters + +```@autodocs +Modules = [Registered] +``` + ## External Links * A [list of registered filter plugins](https://portal.hdfgroup.org/display/support/Registered+Filter+Plugins) can be found on the HDF Group website. diff --git a/src/attributes.jl b/src/attributes.jl index d5ed7f8a0..0c683224a 100644 --- a/src/attributes.jl +++ b/src/attributes.jl @@ -242,7 +242,7 @@ function h5writeattr(filename, name::AbstractString, data::Dict) end """ - h5readattr(filename, name::AbstractString, data::Dict) + h5readattr(filename, name::AbstractString) Read the attributes of the object at `name` in the HDF5 file `filename`, returning a `Dict`. """ diff --git a/src/filters/filters.jl b/src/filters/filters.jl index dd6db29c9..540603fa7 100644 --- a/src/filters/filters.jl +++ b/src/filters/filters.jl @@ -428,6 +428,9 @@ end function Base.convert(::Type{I}, ::Type{F}) where {I<:Integer,F<:Filter} Base.convert(I, filterid(F)) end +function Base.convert(::Type{I}, f::Filter) where {I<:Integer} + Base.convert(I, filterid(f)) +end """ EXTERNAL_FILTER_JULIA_PACKAGES @@ -479,5 +482,6 @@ end include("builtin.jl") include("filters_midlevel.jl") +include("registered.jl") end # module diff --git a/src/filters/registered.jl b/src/filters/registered.jl new file mode 100644 index 000000000..9d065bb6f --- /dev/null +++ b/src/filters/registered.jl @@ -0,0 +1,137 @@ +""" + HDF5.Filters.Registered + +Module containing convenience methods to create `ExternalFilter` instances +of [HDF5 registered filters](https://portal.hdfgroup.org/display/support/Registered+Filter+Plugins). + +This module does not implement any filter or guarantee filter availability. +Rather the functions within this module create `ExternalFilter` instances for convenience. +These instances can be used to determine if a filter is available. They can also +be incorporated as part of a filter pipeline. + +Examine `REGISTERED_FILTERS`, a `Dict{H5Z_filter_t, Function}`, for a list of +filter functions contained within this module, which are exported. + +```jldoctest +julia> println.(values(HDF5.Filters.Registered.REGISTERED_FILTERS)); +FCIDECOMPFilter +LZOFilter +BitGroomFilter +SZ3Filter +Delta_RiceFilter +fpzipFilter +LPC_RiceFilter +LZFFilter +FLACFilter +VBZFilter +FAPECFilter +zfpFilter +CBFFilter +JPEG_XRFilter +LZ4Filter +BLOSC2Filter +ZstandardFilter +SZFilter +Granular_BitRoundFilter +JPEGFilter +SnappyFilter +B³DFilter +APAXFilter +BLOSCFilter +SPDPFilter +bitshuffleFilter +MAFISCFilter +BZIP2Filter +CCSDS_123Filter +JPEG_LSFilter +``` + +""" +module Registered + +using HDF5.Filters: + Filters, Filter, ExternalFilter, EXTERNAL_FILTER_JULIA_PACKAGES, isavailable +using HDF5.API: API, H5Z_filter_t, H5Z_FLAG_MANDATORY + +const _REGISTERED_FILTERIDS_DICT = Dict{H5Z_filter_t,Symbol}( + 305 => :LZO, + 307 => :BZIP2, + 32000 => :LZF, + 32001 => :BLOSC, + 32002 => :MAFISC, + 32003 => :Snappy, + 32004 => :LZ4, + 32005 => :APAX, + 32006 => :CBF, + 32007 => :JPEG_XR, + 32008 => :bitshuffle, + 32009 => :SPDP, + 32010 => :LPC_Rice, + 32011 => :CCSDS_123, + 32012 => :JPEG_LS, + 32013 => :zfp, + 32014 => :fpzip, + 32015 => :Zstandard, + 32016 => :B³D, + 32017 => :SZ, + 32018 => :FCIDECOMP, + 32019 => :JPEG, + 32020 => :VBZ, + 32021 => :FAPEC, + 32022 => :BitGroom, + 32023 => :Granular_BitRound, + 32024 => :SZ3, + 32025 => :Delta_Rice, + 32026 => :BLOSC2, + 32027 => :FLAC +) + +const REGISTERED_FILTERS = Dict{H5Z_filter_t,Function}() + +for (filter_id, filter_name) in _REGISTERED_FILTERIDS_DICT + fn_string = String(filter_name) * "Filter" + fn = Symbol(fn_string) + filter_name_string = replace(String(filter_name), "_" => raw"\_") + @eval begin + @doc """ + $($fn_string)(flags=API.H5Z_FLAG_MANDATORY, data::AbstractVector{<: Integer}=Cuint[], config::Cuint=0) + $($fn_string)(flags=API.H5Z_FLAG_MANDATORY, data::Integer...) + + Create an [`ExternalFilter`](@ref) for $($filter_name_string) with filter id $($filter_id). + $(haskey(EXTERNAL_FILTER_JULIA_PACKAGES, $filter_id) ? + "Users are instead encouraged to use the Julia package $(EXTERNAL_FILTER_JULIA_PACKAGES[$filter_id])." : + "Users should consider defining a subtype of [`Filter`](@ref) to specify the data." + ) + + # Fields / Arguments + * `flags` - (optional) bit vector describing general properties of the filter. Defaults to `API.H5Z_FLAG_MANDATORY` + * `data` - (optional) auxillary data for the filter. See [`cd_values`](@ref API.h5p_set_filter). Defaults to `Cuint[]` + * `config` - (optional) bit vector representing information about the filter regarding whether it is able to encode data, decode data, neither, or both. Defaults to `0`. + + See [`ExternalFilter`](@ref) for valid argument values. + """ $fn + export $fn + $fn(flags, data::AbstractVector{<:Integer}) = + ExternalFilter($filter_id, flags, Cuint.(data), $filter_name_string, 0) + $fn(flags, data::Integer...) = + ExternalFilter($filter_id, flags, Cuint[data...], $filter_name_string, 0) + $fn(data::AbstractVector{<:Integer}=Cuint[]) = ExternalFilter( + $filter_id, H5Z_FLAG_MANDATORY, Cuint.(data), $filter_name_string, 0 + ) + $fn(flags, data, config) = + ExternalFilter($filter_id, flags, data, $filter_name_string, config) + REGISTERED_FILTERS[$filter_id] = $fn + end +end + +""" + available_registered_filters()::Dict{H5Z_filter_t, Function} + +Return a `Dict{H5Z_filter_t, Function}` listing the available filter ids and +their corresponding convenience function. +""" +function available_registered_filters() + filter(p -> isavailable(first(p)), REGISTERED_FILTERS) +end + +end diff --git a/test/filter.jl b/test/filter.jl index 9aadebc48..7cefa4f60 100644 --- a/test/filter.jl +++ b/test/filter.jl @@ -1,5 +1,6 @@ using HDF5 using HDF5.Filters +import HDF5.Filters.Registered using Test using H5Zblosc, H5Zlz4, H5Zbzip2, H5Zzstd using Preferences @@ -253,6 +254,23 @@ using HDF5.Filters: ExternalFilter, isavailable, isencoderenabled, isdecoderenab @test HDF5.API.h5z_filter_avail(H5Z_FILTER_LZ4) @test HDF5.API.h5z_filter_avail(H5Z_FILTER_ZSTD) @test HDF5.API.h5z_filter_avail(H5Z_FILTER_BLOSC) + + # Test the RegisteredFilter module for filters we know to be loaded + reg_loaded = [ + Registered.BZIP2Filter, + Registered.LZ4Filter, + Registered.ZstandardFilter, + Registered.BLOSCFilter + ] + for func in reg_loaded + f = func() + @test HDF5.API.h5z_filter_avail(f) + @test (Filters.filterid(f) => func) in Registered.available_registered_filters() + @test func(HDF5.API.H5Z_FLAG_OPTIONAL) isa ExternalFilter + @test func( + HDF5.API.H5Z_FLAG_OPTIONAL, Cuint[], HDF5.API.H5Z_FILTER_CONFIG_ENCODE_ENABLED + ) isa ExternalFilter + end HDF5.API.h5z_unregister(H5Z_FILTER_LZ4) HDF5.Filters.register_filter(H5Zlz4.Lz4Filter) @test isavailable(H5Z_FILTER_LZ4)