diff --git a/src/Compressors.jl b/src/Compressors.jl index e6db06e7..b54e97ac 100644 --- a/src/Compressors.jl +++ b/src/Compressors.jl @@ -2,6 +2,8 @@ import Blosc import CodecZlib import JSON +_reinterpret(::Type{T}, x::AbstractArray{S, 0}) where {T, S} = reinterpret(T, reshape(x, 1)) +_reinterpret(::Type{T}, x::AbstractArray) where T = reinterpret(T, x) abstract type Compressor end getCompressor(compdict::Dict) = getCompressor(compressortypes[compdict["id"]],compdict) @@ -105,11 +107,11 @@ Creates an object that can be passed to ZArray constructors without compression. struct NoCompressor <: Compressor end function zuncompress(a, ::NoCompressor, T) - reinterpret(T,a) + _reinterpret(T,a) end function zcompress(a, ::NoCompressor) - reinterpret(UInt8,a) + _reinterpret(UInt8,a) end JSON.lower(::NoCompressor) = nothing @@ -136,11 +138,11 @@ end function zuncompress(a, ::ZlibCompressor, T) result = transcode(CodecZlib.ZlibDecompressor,a) - reinterpret(Base.nonmissingtype(T),result) + _reinterpret(Base.nonmissingtype(T),result) end function zcompress(a, ::ZlibCompressor) - a_uint8 = reinterpret(UInt8,a)[:] + a_uint8 = _reinterpret(UInt8,a)[:] transcode(CodecZlib.ZlibCompressor, a_uint8) end diff --git a/src/Storage/Storage.jl b/src/Storage/Storage.jl index 1e7730a0..d1c69142 100644 --- a/src/Storage/Storage.jl +++ b/src/Storage/Storage.jl @@ -48,6 +48,7 @@ Deletes the given key from the store. """ citostring(i::CartesianIndex) = join(reverse((i - oneunit(i)).I), '.') +citostring(::CartesianIndex{0}) = "0" _concatpath(p,s) = isempty(p) ? s : rstrip(p,'/') * '/' * s Base.getindex(s::AbstractStore, p, i::CartesianIndex) = s[p, citostring(i)] diff --git a/src/ZArray.jl b/src/ZArray.jl index 4e7b3002..ee74215d 100644 --- a/src/ZArray.jl +++ b/src/ZArray.jl @@ -159,11 +159,11 @@ function readblock!(aout::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::Cartes # Now loop through the chunks c = Channel{Pair{eltype(blockr),Union{Nothing,Vector{UInt8}}}}(channelsize(z.storage)) - task = @async begin + task = @async begin read_items!($z.storage,c, $z.path, $blockr) end bind(c,task) - + try for i in 1:length(blockr) @@ -268,7 +268,7 @@ function uncompress_to_output!(aout,output_base_offsets,z,chunk_compressed,curre uncompress_raw!(a,z,chunk_compressed) if length.(indranges) == size(a) - aout[dotminus.(indranges, output_base_offsets)...] = a + aout[dotminus.(indranges, output_base_offsets)...] = ndims(a) == 0 ? a[1] : a else curchunk = a[dotminus.(indranges,current_chunk_offsets)...] aout[dotminus.(indranges, output_base_offsets)...] = curchunk diff --git a/test/python.jl b/test/python.jl index 03ad6631..6ff15d76 100644 --- a/test/python.jl +++ b/test/python.jl @@ -36,12 +36,16 @@ compressors = ( "blosc_bitshuffle"=>BloscCompressor(cname="zstd",shuffle=2), "zlib"=>ZlibCompressor()) testarrays = Dict(t=>(t<:AbstractString) ? [randstring(maximum(i.I)) for i in CartesianIndices((1:10,1:6,1:2))] : rand(t,10,6,2) for t in dtypes) +testzerodimarrays = Dict(t=>(t<:AbstractString) ? randstring(10) : rand(t) for t in dtypes) for t in dtypes, co in compressors compstr, comp = co att = Dict("This is a nested attribute"=>Dict("a"=>5)) a = zcreate(t, g,string("a",t,compstr),10,6,2,attrs=att, chunks = (5,2,2),compressor=comp) a[:,:,:] = testarrays[t] + + a = zcreate(t, g,string("azerodim",t,compstr), compressor=comp) + a[] = testzerodimarrays[t] end # Test reading in python py""" @@ -83,6 +87,30 @@ for i=1:length(dtypes), co in compressors end end +for i=1:length(dtypes), co in compressors + compstr,comp = co + t = dtypes[i] + tp = dtypesp[i] + if t == UInt64 + continue + # need to exclude UInt64: + # need explicit conversion because of https://github.com/JuliaPy/PyCall.jl/issues/744 + # but explicit conversion uses PyLong_AsLongLongAndOverflow, which converts everything + # to a signed 64-bit integer, which can error out if the UInt64 is too large. + # Adding an overload to PyCall for unsigned ints doesn't work with NumPy scalars because + # they are not subtypes of integer: https://stackoverflow.com/a/58816671 + end + + arname = string("azerodim",t,compstr) + py""" + ar=g[$arname] + """ + + @test py"ar.dtype==$tp" + @test py"ar.shape" == () + @test convert(t, py"ar[()]") == testzerodimarrays[t] +end + ## Now the other way around, we create a zarr array using the python lib and read back into julia data = rand(Int32,2,6,10) py"""