From 84300a6ac29d414775644a97d52f15fb645adc91 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Tue, 13 Jun 2017 10:50:53 -0400 Subject: [PATCH] clean up and export crc32c function (#22274) * clean up and export crc32c function * added PR to NEWS * restore crc32 of String, add crc32c(io) to read all of a stream, add optimized open(crc32c, filename), make IOBuffer checksums consistent with other streams * use crc32c block size of 8192*3, matching the underling C library * optimized IOBuffer crc32c --- NEWS.md | 2 ++ base/exports.jl | 1 + base/iobuffer.jl | 12 +++++++++ base/loading.jl | 13 +++------- base/util.jl | 48 +++++++++++++++++++++++++++++++----- doc/src/stdlib/arrays.md | 1 + doc/src/stdlib/io-network.md | 1 + test/misc.jl | 34 ++++++++++++++++++++++--- 8 files changed, 93 insertions(+), 19 deletions(-) diff --git a/NEWS.md b/NEWS.md index cff38eca2c720..24aaa02f22dcf 100644 --- a/NEWS.md +++ b/NEWS.md @@ -59,6 +59,8 @@ Library improvements * `resize!` and `sizehint!` methods no longer over-reserve memory when the requested array size is more than double of its current size ([#22038]). + * The `crc32c` function for CRC-32c checksums is now exported ([#22274]). + * The output of `versioninfo()` is now controlled with keyword arguments ([#21974]). * The function `LibGit2.set_remote_url` now always sets both the fetch and push URLs for a diff --git a/base/exports.jl b/base/exports.jl index d88133ca324d7..0aeba4d7b2eb0 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -1047,6 +1047,7 @@ export atexit, atreplinit, clipboard, + crc32c, exit, ntuple, quit, diff --git a/base/iobuffer.jl b/base/iobuffer.jl index c9d4226462dac..854e39cbc4f17 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -412,3 +412,15 @@ function readuntil(io::AbstractIOBuffer, delim::UInt8) end A end + +# copy-free crc32c of IOBuffer: +function crc32c(io::IOBuffer, nb::Integer, crc::UInt32=0x00000000) + nb < 0 && throw(ArgumentError("number of bytes to checksum must be ≥ 0")) + io.readable || throw(ArgumentError("read failed, IOBuffer is not readable")) + n = min(nb, nb_available(io)) + n == 0 && return crc + crc = unsafe_crc32c(pointer(io.data, io.ptr), n, crc) + io.ptr += n + return crc +end +crc32c(io::IOBuffer, crc::UInt32=0x00000000) = crc32c(io, nb_available(io), crc) diff --git a/base/loading.jl b/base/loading.jl index 1a6b5671342d0..1382f4f2a00ed 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -683,10 +683,7 @@ function compilecache(name::String) if success(create_expr_cache(path, cachefile, concrete_deps)) # append checksum to the end of the .ji file: open(cachefile, "a+") do f - data = Mmap.mmap(f, Vector{UInt8}, filesize(f), 0) - checksum = crc32c(data) - finalize(data) - write(f, hton(checksum)) + write(f, hton(crc32c(seekstart(f)))) end else error("Failed to precompile $name to $cachefile.") @@ -809,12 +806,8 @@ function stale_cachefile(modpath::String, cachefile::String) end # finally, verify that the cache file has a valid checksum - data = Mmap.mmap(io, Vector{UInt8}, filesize(io), 0) - # checksum = UInt32 read in bigendian format from the last 4 bytes: - checksum = UInt32(data[end]) + UInt32(data[end-1])<<8 + UInt32(data[end-2])<<16 + UInt32(data[end-3])<<24 - crc = crc32c(@view(data[1:end-4])) - finalize(data) - if checksum != crc + crc = crc32c(seekstart(io), filesize(io)-4) + if crc != ntoh(read(io, UInt32)) DEBUG_LOADING[] && info("JL_DEBUG_LOADING: Rejecting cache file $cachefile because it has an invalid checksum.") return true end diff --git a/base/util.jl b/base/util.jl index a909b8355cf7a..1617b3d6c5df1 100644 --- a/base/util.jl +++ b/base/util.jl @@ -765,10 +765,6 @@ if is_windows() end -# compute sizeof correctly for strings, arrays, and subarrays of bytes -_sizeof(a) = sizeof(a) -_sizeof(a::FastContiguousSubArray{UInt8,N,<:Array{UInt8}} where N) = length(a) - """ crc32c(data, crc::UInt32=0x00000000) @@ -778,9 +774,49 @@ a starting `crc` integer to be mixed in with the checksum. The `crc` parameter can be used to compute a checksum on data divided into chunks: performing `crc32c(data2, crc32c(data1))` is equivalent to the checksum of `[data1; data2]`. (Technically, a little-endian checksum is computed.) + +There is also a method `crc32c(io, nb, crc)` to checksum `nb` bytes from +a stream `io`, or `crc32c(io, crc)` to checksum all the remaining bytes. +Hence you can do [`open(crc32c, filename)`](@ref) to checksum an entire file, +or `crc32c(seekstart(buf))` to checksum an [`IOBuffer`](@ref) without +calling [`take!`](@ref). + +For a `String`, note that the result is specific to the UTF-8 encoding +(a different checksum would be obtained from a different Unicode encoding). +To checksum an `a::Array` of some other bitstype, you can do `crc32c(reinterpret(UInt8,a))`, +but note that the result may be endian-dependent. """ -crc32c(a::Union{Array{UInt8},FastContiguousSubArray{UInt8,N,<:Array{UInt8}} where N,String}, crc::UInt32=0x00000000) = - ccall(:jl_crc32c, UInt32, (UInt32, Ptr{UInt8}, Csize_t), crc, a, _sizeof(a)) +function crc32c end + +unsafe_crc32c(a, n, crc) = ccall(:jl_crc32c, UInt32, (UInt32, Ptr{UInt8}, Csize_t), crc, a, n) + +crc32c(a::Union{Array{UInt8},FastContiguousSubArray{UInt8,N,<:Array{UInt8}} where N}, crc::UInt32=0x00000000) = + unsafe_crc32c(a, length(a), crc) + +crc32c(s::String, crc::UInt32=0x00000000) = unsafe_crc32c(s, sizeof(s), crc) + +""" + crc32c(io::IO, [nb::Integer,] crc::UInt32=0x00000000) + +Read up to `nb` bytes from `io` and return the CRC-32c checksum, optionally +mixed with a starting `crc` integer. If `nb` is not supplied, then +`io` will be read until the end of the stream. +""" +function crc32c(io::IO, nb::Integer, crc::UInt32=0x00000000) + nb < 0 && throw(ArgumentError("number of bytes to checksum must be ≥ 0")) + # use block size 24576=8192*3, since that is the threshold for + # 3-way parallel SIMD code in the underlying jl_crc32c C function. + buf = Array{UInt8}(min(nb, 24576)) + while !eof(io) && nb > 24576 + n = readbytes!(io, buf) + crc = unsafe_crc32c(buf, n, crc) + nb -= n + end + return unsafe_crc32c(buf, readbytes!(io, buf, min(nb, length(buf))), crc) +end +crc32c(io::IO, crc::UInt32=0x00000000) = crc32c(io, typemax(Int64), crc) +crc32c(io::IOStream, crc::UInt32=0x00000000) = crc32c(io, filesize(io)-position(io), crc) + """ @kwdef typedef diff --git a/doc/src/stdlib/arrays.md b/doc/src/stdlib/arrays.md index 2fc43f52bee53..9221735386c58 100644 --- a/doc/src/stdlib/arrays.md +++ b/doc/src/stdlib/arrays.md @@ -131,6 +131,7 @@ Base.cumprod! Base.cumsum Base.cumsum! Base.cumsum_kbn +Base.crc32c Base.LinAlg.diff Base.LinAlg.gradient Base.rot180 diff --git a/doc/src/stdlib/io-network.md b/doc/src/stdlib/io-network.md index 88fd2b2cf3bb0..173261912b02e 100644 --- a/doc/src/stdlib/io-network.md +++ b/doc/src/stdlib/io-network.md @@ -12,6 +12,7 @@ Base.take!(::Base.AbstractIOBuffer) Base.fdio Base.flush Base.close +Base.crc32c(::IO, ::Integer, ::UInt32) Base.write Base.read Base.read! diff --git a/test/misc.jl b/test/misc.jl index f6cc13c538f78..4a0b481d56acd 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -566,14 +566,42 @@ end for force_software_crc in (1,0) ccall(:jl_crc32c_init, Void, (Cint,), force_software_crc) for (n,crc) in [(0,0x00000000),(1,0xa016d052),(2,0x03f89f52),(3,0xf130f21e),(4,0x29308cf4),(5,0x53518fab),(6,0x4f4dfbab),(7,0xbd3a64dc),(8,0x46891f81),(9,0x5a14b9f9),(10,0xb219db69),(11,0xd232a91f),(12,0x51a15563),(13,0x9f92de41),(14,0x4d8ae017),(15,0xc8b74611),(16,0xa0de6714),(17,0x672c992a),(18,0xe8206eb6),(19,0xc52fd285),(20,0x327b0397),(21,0x318263dd),(22,0x08485ccd),(23,0xea44d29e),(24,0xf6c0cb13),(25,0x3969bba2),(26,0x6a8810ec),(27,0x75b3d0df),(28,0x82d535b1),(29,0xbdf7fc12),(30,0x1f836b7d),(31,0xd29f33af),(32,0x8e4acb3e),(33,0x1cbee2d1),(34,0xb25f7132),(35,0xb0fa484c),(36,0xb9d262b4),(37,0x3207fe27),(38,0xa024d7ac),(39,0x49a2e7c5),(40,0x0e2c157f),(41,0x25f7427f),(42,0x368c6adc),(43,0x75efd4a5),(44,0xa84c5c31),(45,0x0fc817b2),(46,0x8d99a881),(47,0x5cc3c078),(48,0x9983d5e2),(49,0x9267c2db),(50,0xc96d4745),(51,0x058d8df3),(52,0x453f9cf3),(53,0xb714ade1),(54,0x55d3c2bc),(55,0x495710d0),(56,0x3bddf494),(57,0x4f2577d0),(58,0xdae0f604),(59,0x3c57c632),(60,0xfe39bbb0),(61,0x6f5d1d41),(62,0x7d996665),(63,0x68c738dc),(64,0x8dfea7ae)] - @test Base.crc32c(UInt8[1:n;]) == crc + @test crc32c(UInt8[1:n;]) == crc == crc32c(String(UInt8[1:n;])) end # test that crc parameter is equivalent to checksum of concatenated data, # and test crc of subarrays: a = UInt8[1:255;] - crc_256 = Base.crc32c(UInt8[1:255;]) + crc_256 = crc32c(a) @views for n = 1:255 - @test Base.crc32c(a[n+1:end], Base.crc32c(a[1:n])) == crc_256 + @test crc32c(a[n+1:end], crc32c(a[1:n])) == crc_256 + end + + @test crc32c(IOBuffer(a)) == crc_256 + let buf = IOBuffer() + write(buf, a[1:3]) + @test crc32c(seekstart(buf)) == crc32c(a[1:3]) + @test crc32c(buf) == 0x00000000 + @test crc32c(seek(buf, 1)) == crc32c(a[2:3]) + @test crc32c(seek(buf, 0), 2) == crc32c(a[1:2]) + @test crc32c(buf) == crc32c(a[3:3]) + end + + let f = tempname() + try + write(f, a) + @test open(crc32c, f) == crc_256 + open(f, "r") do io + @test crc32c(io, 16) == crc32c(a[1:16]) + @test crc32c(io, 16) == crc32c(a[17:32]) + @test crc32c(io) == crc32c(a[33:end]) + @test crc32c(io, 1000) == 0x00000000 + end + a = rand(UInt8, 30000) + write(f, a) + @test open(crc32c, f) == crc32c(a) == open(io -> crc32c(io, 10^6), f) + finally + rm(f, force=true) + end end end