diff --git a/third_party/zlib/BUILD.gn b/third_party/zlib/BUILD.gn index 190e7bbc32..722687bf93 100644 --- a/third_party/zlib/BUILD.gn +++ b/third_party/zlib/BUILD.gn @@ -2,43 +2,228 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + +if (current_cpu == "arm" || current_cpu == "arm64") { + import("//build/config/arm.gni") +} + config("zlib_config") { include_dirs = [ "." ] } -static_library("zlib_x86_simd") { - if (!is_ios && (current_cpu == "x86" || current_cpu == "x64")) { +config("zlib_internal_config") { + defines = [ "ZLIB_IMPLEMENTATION" ] +} + +use_arm_neon_optimizations = false +if ((current_cpu == "arm" || current_cpu == "arm64") && + !(is_win && !is_clang)) { + # TODO(richard.townsend@arm.com): Optimizations temporarily disabled for + # Windows on Arm MSVC builds, see http://crbug.com/v8/10012. + if (arm_use_neon) { + use_arm_neon_optimizations = true + } +} + +use_x86_x64_optimizations = + (current_cpu == "x86" || current_cpu == "x64") && !is_ios + +config("zlib_adler32_simd_config") { + if (use_x86_x64_optimizations) { + defines = [ "ADLER32_SIMD_SSSE3" ] + if (is_win) { + defines += [ "X86_WINDOWS" ] + } else { + defines += [ "X86_NOT_WINDOWS" ] + } + } + + if (use_arm_neon_optimizations) { + defines = [ "ADLER32_SIMD_NEON" ] + } +} + +source_set("zlib_adler32_simd") { + visibility = [ ":*" ] + + if (use_x86_x64_optimizations) { sources = [ - "crc_folding.c", - "fill_window_sse.c", + "adler32_simd.c", + "adler32_simd.h", ] + if (!is_win || is_clang) { - cflags = [ - "-msse4.2", - "-mpclmul", + cflags = [ "-mssse3" ] + } + } + + if (use_arm_neon_optimizations) { + sources = [ + "adler32_simd.c", + "adler32_simd.h", + ] + } + + configs += [ ":zlib_internal_config" ] + + public_configs = [ ":zlib_adler32_simd_config" ] +} + +if (use_arm_neon_optimizations) { + config("zlib_arm_crc32_config") { + # Disabled for iPhone, as described in DDI0487C_a_armv8_arm: + # "All implementations of the ARMv8.1 architecture are required to + # implement the CRC32* instructions. These are optional in ARMv8.0." + if (!is_ios) { + defines = [ "CRC32_ARMV8_CRC32" ] + if (is_android) { + defines += [ "ARMV8_OS_ANDROID" ] + } else if (is_linux || is_chromeos) { + defines += [ "ARMV8_OS_LINUX" ] + } else if (is_mac) { + defines += [ "ARMV8_OS_MACOS" ] + } else if (is_fuchsia) { + defines += [ "ARMV8_OS_FUCHSIA" ] + } else if (is_win) { + defines += [ "ARMV8_OS_WINDOWS" ] + } else { + assert(false, "Unsupported ARM OS") + } + } + } + + source_set("zlib_arm_crc32") { + visibility = [ ":*" ] + + if (!is_ios) { + include_dirs = [ "." ] + + if (!is_win && !is_clang) { + assert(!use_thin_lto, + "ThinLTO fails mixing different module-level targets") + cflags_c = [ "-march=armv8-a+crc" ] + } + + sources = [ + "crc32_simd.c", + "crc32_simd.h", ] } - } else { + + configs += [ ":zlib_internal_config" ] + + public_configs = [ ":zlib_arm_crc32_config" ] + } +} + +config("zlib_inflate_chunk_simd_config") { + if (use_x86_x64_optimizations) { + defines = [ "INFLATE_CHUNK_SIMD_SSE2" ] + + if (current_cpu == "x64") { + defines += [ "INFLATE_CHUNK_READ_64LE" ] + } + } + + if (use_arm_neon_optimizations) { + defines = [ "INFLATE_CHUNK_SIMD_NEON" ] + + if (current_cpu == "arm64") { + defines += [ "INFLATE_CHUNK_READ_64LE" ] + } + } +} + +source_set("zlib_inflate_chunk_simd") { + visibility = [ ":*" ] + + if (use_x86_x64_optimizations || use_arm_neon_optimizations) { + include_dirs = [ "." ] + sources = [ - "simd_stub.c", + "contrib/optimizations/chunkcopy.h", + "contrib/optimizations/inffast_chunk.c", + "contrib/optimizations/inffast_chunk.h", + "contrib/optimizations/inflate.c", ] } + configs += [ ":zlib_internal_config" ] + + # Needed for MSVC, which is still supported by V8 and PDFium. zlib uses K&R C + # style function declarations, which triggers warning C4131. configs -= [ "//build/config/compiler:chromium_code" ] configs += [ "//build/config/compiler:no_chromium_code" ] + + public_configs = [ ":zlib_inflate_chunk_simd_config" ] } -config("zlib_warnings") { +config("zlib_crc32_simd_config") { + if (use_x86_x64_optimizations) { + defines = [ "CRC32_SIMD_SSE42_PCLMUL" ] + } +} + +source_set("zlib_crc32_simd") { visibility = [ ":*" ] - if (is_clang) { - cflags = [ - "-Wno-incompatible-pointer-types", - "-Wno-shift-negative-value", + + if (use_x86_x64_optimizations) { + sources = [ + "crc32_simd.c", + "crc32_simd.h", ] + + if (!is_win || is_clang) { + cflags = [ + "-msse4.2", + "-mpclmul", + ] + } + } + + configs += [ ":zlib_internal_config" ] + + public_configs = [ ":zlib_crc32_simd_config" ] +} + +config("zlib_x86_simd_config") { + if (use_x86_x64_optimizations) { + defines = [ + "CRC32_SIMD_SSE42_PCLMUL", + "DEFLATE_FILL_WINDOW_SSE2", + ] + } +} + +source_set("zlib_x86_simd") { + visibility = [ ":*" ] + + if (use_x86_x64_optimizations) { + sources = [ + "crc_folding.c", + "fill_window_sse.c", + ] + + if (!is_win || is_clang) { + cflags = [ + "-msse4.2", + "-mpclmul", + ] + } + } + + configs += [ ":zlib_internal_config" ] + + public_configs = [ ":zlib_x86_simd_config" ] +} + +config("zlib_warnings") { + if (is_clang && use_x86_x64_optimizations) { + cflags = [ "-Wno-incompatible-pointer-types" ] } } -static_library("zlib") { +component("zlib") { if (!is_win) { # Don't stomp on "libzlib" on other platforms. output_name = "chrome_zlib" @@ -46,7 +231,11 @@ static_library("zlib") { sources = [ "adler32.c", + "chromeconf.h", "compress.c", + "contrib/optimizations/insert_string.h", + "cpu_features.c", + "cpu_features.h", "crc32.c", "crc32.h", "deflate.c", @@ -60,37 +249,80 @@ static_library("zlib") { "inffast.c", "inffast.h", "inffixed.h", - "inflate.c", "inflate.h", "inftrees.c", "inftrees.h", - "mozzconf.h", "trees.c", "trees.h", "uncompr.c", - "x86.h", "zconf.h", "zlib.h", "zutil.c", "zutil.h", ] - if (!is_ios && (current_cpu == "x86" || current_cpu == "x64")) { - sources += [ "x86.c" ] + defines = [] + deps = [] + + if (!use_x86_x64_optimizations && !use_arm_neon_optimizations) { + # Apparently android_cronet bot builds with NEON disabled and + # we also should disable optimizations for iOS@x86 (a.k.a. simulator). + defines += [ "CPU_NO_SIMD" ] + } + + if (is_ios) { + # iOS@ARM is a special case where we always have NEON but don't check + # for crypto extensions. + # TODO(cavalcantii): verify what is the current state of CPU features + # shipped on latest iOS devices. + defines += [ "ARM_OS_IOS" ] + } + + if (use_x86_x64_optimizations || use_arm_neon_optimizations) { + deps += [ + ":zlib_adler32_simd", + ":zlib_inflate_chunk_simd", + ] + + if (use_x86_x64_optimizations) { + deps += [ ":zlib_crc32_simd" ] + } else if (use_arm_neon_optimizations) { + sources += [ "contrib/optimizations/slide_hash_neon.h" ] + deps += [ ":zlib_arm_crc32" ] + } + } else { + sources += [ "inflate.c" ] + } + + deps += [ ":zlib_x86_simd" ] + + if (is_android) { + import("//build/config/android/config.gni") + if (defined(android_ndk_root) && android_ndk_root != "") { + deps += [ "//third_party/android_tools:cpu_features" ] + } else { + assert(false, "CPU detection requires the Android NDK") + } } - configs += [ ":zlib_warnings" ] configs -= [ "//build/config/compiler:chromium_code" ] configs += [ "//build/config/compiler:no_chromium_code" ] public_configs = [ ":zlib_config" ] - deps = [ - ":zlib_x86_simd", + + configs += [ + ":zlib_internal_config", + + # Must be after no_chromium_code for warning flags to be ordered correctly. + ":zlib_warnings", ] + + allow_circular_includes_from = deps } config("minizip_warnings") { visibility = [ ":*" ] + if (is_clang) { # zlib uses `if ((a == b))` for some reason. cflags = [ "-Wno-parentheses-equality" ] @@ -115,32 +347,33 @@ static_library("minizip") { "contrib/minizip/iowin32.h", ] } + if (is_mac || is_ios || is_android || is_fuchsia) { # Mac, Android and the BSDs don't have fopen64, ftello64, or fseeko64. We # use fopen, ftell, and fseek instead on these systems. defines = [ "USE_FILE32API" ] } - deps = [ - ":zlib", - ] + deps = [ ":zlib" ] configs -= [ "//build/config/compiler:chromium_code" ] configs += [ "//build/config/compiler:no_chromium_code" ] - configs += [ ":minizip_warnings" ] + public_configs = [ ":zlib_config" ] -} -static_library("zip") { - sources = [ - "google/zip.cc", - "google/zip.h", - "google/zip_internal.cc", - "google/zip_internal.h", - "google/zip_reader.cc", - "google/zip_reader.h", - ] - deps = [ - ":minizip", + configs += [ + # Must be after no_chromium_code for warning flags to be ordered correctly. + ":minizip_warnings", ] } + +executable("zlib_bench") { + include_dirs = [ "." ] + + sources = [ "contrib/bench/zlib_bench.cc" ] + + deps = [ ":zlib" ] + + configs -= [ "//build/config/compiler:chromium_code" ] + configs += [ "//build/config/compiler:no_chromium_code" ] +} diff --git a/third_party/zlib/DEPS b/third_party/zlib/DEPS new file mode 100644 index 0000000000..b6dcfc6bc1 --- /dev/null +++ b/third_party/zlib/DEPS @@ -0,0 +1,3 @@ +include_rules = [ + "+testing/gtest", +] \ No newline at end of file diff --git a/third_party/zlib/LICENSE b/third_party/zlib/LICENSE index 23e5e693cf..9f056865b5 100644 --- a/third_party/zlib/LICENSE +++ b/third_party/zlib/LICENSE @@ -1,25 +1,19 @@ -/* zlib.h -- interface of the 'zlib' general purpose compression library - version 1.2.4, March 14th, 2010 +version 1.2.11, January 15th, 2017 - Copyright (C) 1995-2010 Jean-loup Gailly and Mark Adler +Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler - This software is provided 'as-is', without any express or implied - warranty. In no event will the authors be held liable for any damages - arising from the use of this software. +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - 3. This notice may not be removed or altered from any source distribution. - - Jean-loup Gailly - Mark Adler - -*/ +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. +3. This notice may not be removed or altered from any source distribution. diff --git a/third_party/zlib/README.chromium b/third_party/zlib/README.chromium index b90bcff42f..c3c1ef69ad 100644 --- a/third_party/zlib/README.chromium +++ b/third_party/zlib/README.chromium @@ -1,37 +1,29 @@ Name: zlib +Short Name: zlib URL: http://zlib.net/ -Version: 1.2.5 +Version: 1.2.11 +CPEPrefix: cpe:/a:zlib:zlib:1.2.11 Security Critical: yes License: Custom license +License File: LICENSE License Android Compatible: yes Description: -General purpose compression library +"A massively spiffy yet delicately unobtrusive compression library." -Local Modifications: -A few minor changes, all marked with "Google": -- Added #ifdefs to avoid compile warnings when NO_GZCOMPRESS is defined. -- Removed use of strerror for WinCE in gzio.c. -- Added 'int z_errno' global for WinCE, to which 'errno' is defined in zutil.h. -- Added 'mozzconf.h' to mangle the function names. -- Added logic in zlib.h to undef our earlier mangles when defaulting to 64 bit offset versions of API. -The 'google.patch' file represents our changes from the original zlib-1.2.5. - -A more significant change to support mixed-source data compression. See -crbug.com/139744 and mixed-source.patch. - -Integrated Intel SIMD optimisations from: https://github.com/jtkukunas/zlib/ -and modified to accomodate the older version and existing changes in tree. +zlib is a free, general-purpose, legally unencumbered lossless data-compression +library. zlib implements the "deflate" compression algorithm described by RFC +1951, which combines the LZ77 (Lempel-Ziv) algorithm with Huffman coding. zlib +also implements the zlib (RFC 1950) and gzip (RFC 1952) wrapper formats. -This introduces new files: simd_stub.c, crc_folding.c, fill_window_sse.c and -x86.[ch]. All but the latter are built into a static library to allow the -compiler to use the desired instructions only when valid. The latter version is -only built on x86 (32-bit and 64-bit) systems with it's functionality stubbed -on the others. - -Other changes to accomodate: -- fill_window() implementation calls into _sse() variant when supported and the - original implementation renamed to _c() -- read_buf was moved from local to ZLIB_INTERNAL for fill_window_sse.c to use -- INSERT_STRING macro was made a function, insert_string() and an implementation using CRC instruction added -- some crc funcionality moved into crc32.c +Local Modifications: + - Only source code from the zlib distribution used to build the zlib and + minizip libraries are present. Many other files have been omitted. Only *.c + and *.h files from the upstream root directory and contrib/minizip were + imported. + - The contents of the google directory are original Chromium-specific + additions. + - Added chromeconf.h + - Plus the changes in 'patches' folder. + - Code in contrib/ other than contrib/minizip was added to match zlib's + contributor layout. diff --git a/third_party/zlib/adler32.c b/third_party/zlib/adler32.c index 65ad6a5adc..696773a09d 100644 --- a/third_party/zlib/adler32.c +++ b/third_party/zlib/adler32.c @@ -1,5 +1,5 @@ /* adler32.c -- compute the Adler-32 checksum of a data stream - * Copyright (C) 1995-2007 Mark Adler + * Copyright (C) 1995-2011, 2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -7,11 +7,9 @@ #include "zutil.h" -#define local static +local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2)); -local uLong adler32_combine_(uLong adler1, uLong adler2, z_off64_t len2); - -#define BASE 65521UL /* largest prime smaller than 65536 */ +#define BASE 65521U /* largest prime smaller than 65536 */ #define NMAX 5552 /* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ @@ -21,50 +19,68 @@ local uLong adler32_combine_(uLong adler1, uLong adler2, z_off64_t len2); #define DO8(buf,i) DO4(buf,i); DO4(buf,i+4); #define DO16(buf) DO8(buf,0); DO8(buf,8); -/* use NO_DIVIDE if your processor does not do division in hardware */ +/* use NO_DIVIDE if your processor does not do division in hardware -- + try it both ways to see which is faster */ #ifdef NO_DIVIDE -# define MOD(a) \ +/* note that this assumes BASE is 65521, where 65536 % 65521 == 15 + (thank you to John Reiser for pointing this out) */ +# define CHOP(a) \ + do { \ + unsigned long tmp = a >> 16; \ + a &= 0xffffUL; \ + a += (tmp << 4) - tmp; \ + } while (0) +# define MOD28(a) \ do { \ - if (a >= (BASE << 16)) a -= (BASE << 16); \ - if (a >= (BASE << 15)) a -= (BASE << 15); \ - if (a >= (BASE << 14)) a -= (BASE << 14); \ - if (a >= (BASE << 13)) a -= (BASE << 13); \ - if (a >= (BASE << 12)) a -= (BASE << 12); \ - if (a >= (BASE << 11)) a -= (BASE << 11); \ - if (a >= (BASE << 10)) a -= (BASE << 10); \ - if (a >= (BASE << 9)) a -= (BASE << 9); \ - if (a >= (BASE << 8)) a -= (BASE << 8); \ - if (a >= (BASE << 7)) a -= (BASE << 7); \ - if (a >= (BASE << 6)) a -= (BASE << 6); \ - if (a >= (BASE << 5)) a -= (BASE << 5); \ - if (a >= (BASE << 4)) a -= (BASE << 4); \ - if (a >= (BASE << 3)) a -= (BASE << 3); \ - if (a >= (BASE << 2)) a -= (BASE << 2); \ - if (a >= (BASE << 1)) a -= (BASE << 1); \ + CHOP(a); \ if (a >= BASE) a -= BASE; \ } while (0) -# define MOD4(a) \ +# define MOD(a) \ do { \ - if (a >= (BASE << 4)) a -= (BASE << 4); \ - if (a >= (BASE << 3)) a -= (BASE << 3); \ - if (a >= (BASE << 2)) a -= (BASE << 2); \ - if (a >= (BASE << 1)) a -= (BASE << 1); \ + CHOP(a); \ + MOD28(a); \ + } while (0) +# define MOD63(a) \ + do { /* this assumes a is not negative */ \ + z_off64_t tmp = a >> 32; \ + a &= 0xffffffffL; \ + a += (tmp << 8) - (tmp << 5) + tmp; \ + tmp = a >> 16; \ + a &= 0xffffL; \ + a += (tmp << 4) - tmp; \ + tmp = a >> 16; \ + a &= 0xffffL; \ + a += (tmp << 4) - tmp; \ if (a >= BASE) a -= BASE; \ } while (0) #else # define MOD(a) a %= BASE -# define MOD4(a) a %= BASE +# define MOD28(a) a %= BASE +# define MOD63(a) a %= BASE +#endif + +#include "cpu_features.h" +#if defined(ADLER32_SIMD_SSSE3) || defined(ADLER32_SIMD_NEON) +#include "adler32_simd.h" #endif /* ========================================================================= */ -uLong ZEXPORT adler32(adler, buf, len) +uLong ZEXPORT adler32_z(adler, buf, len) uLong adler; const Bytef *buf; - uInt len; + z_size_t len; { unsigned long sum2; unsigned n; +#if defined(ADLER32_SIMD_SSSE3) + if (x86_cpu_enable_ssse3 && buf && len >= 64) + return adler32_simd_(adler, buf, len); +#elif defined(ADLER32_SIMD_NEON) + if (buf && len >= 64) + return adler32_simd_(adler, buf, len); +#endif + /* split Adler-32 into component sums */ sum2 = (adler >> 16) & 0xffff; adler &= 0xffff; @@ -80,9 +96,24 @@ uLong ZEXPORT adler32(adler, buf, len) return adler | (sum2 << 16); } +#if defined(ADLER32_SIMD_SSSE3) + /* + * Use SSSE3 to compute the adler32. Since this routine can be + * freely used, check CPU features here. zlib convention is to + * call adler32(0, NULL, 0), before making calls to adler32(). + * So this is a good early (and infrequent) place to cache CPU + * features for those later, more interesting adler32() calls. + */ + if (buf == Z_NULL) { + if (!len) /* Assume user is calling adler32(0, NULL, 0); */ + cpu_check_features(); + return 1L; + } +#else /* initial Adler-32 value (deferred check for len == 1 speed) */ if (buf == Z_NULL) return 1L; +#endif /* in case short lengths are provided, keep it somewhat fast */ if (len < 16) { @@ -92,7 +123,7 @@ uLong ZEXPORT adler32(adler, buf, len) } if (adler >= BASE) adler -= BASE; - MOD4(sum2); /* only added so many BASE's */ + MOD28(sum2); /* only added so many BASE's */ return adler | (sum2 << 16); } @@ -127,6 +158,15 @@ uLong ZEXPORT adler32(adler, buf, len) return adler | (sum2 << 16); } +/* ========================================================================= */ +uLong ZEXPORT adler32(adler, buf, len) + uLong adler; + const Bytef *buf; + uInt len; +{ + return adler32_z(adler, buf, len); +} + /* ========================================================================= */ local uLong adler32_combine_(adler1, adler2, len2) uLong adler1; @@ -137,8 +177,13 @@ local uLong adler32_combine_(adler1, adler2, len2) unsigned long sum2; unsigned rem; + /* for negative len, return invalid adler32 as a clue for debugging */ + if (len2 < 0) + return 0xffffffffUL; + /* the derivation of this formula is left as an exercise for the reader */ - rem = (unsigned)(len2 % BASE); + MOD63(len2); /* assumes len2 >= 0 */ + rem = (unsigned)len2; sum1 = adler1 & 0xffff; sum2 = rem * sum1; MOD(sum2); @@ -146,7 +191,7 @@ local uLong adler32_combine_(adler1, adler2, len2) sum2 += ((adler1 >> 16) & 0xffff) + ((adler2 >> 16) & 0xffff) + BASE - rem; if (sum1 >= BASE) sum1 -= BASE; if (sum1 >= BASE) sum1 -= BASE; - if (sum2 >= (BASE << 1)) sum2 -= (BASE << 1); + if (sum2 >= ((unsigned long)BASE << 1)) sum2 -= ((unsigned long)BASE << 1); if (sum2 >= BASE) sum2 -= BASE; return sum1 | (sum2 << 16); } diff --git a/third_party/zlib/adler32_simd.c b/third_party/zlib/adler32_simd.c new file mode 100644 index 0000000000..1354915cc0 --- /dev/null +++ b/third_party/zlib/adler32_simd.c @@ -0,0 +1,366 @@ +/* adler32_simd.c + * + * Copyright 2017 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + * + * Per http://en.wikipedia.org/wiki/Adler-32 the adler32 A value (aka s1) is + * the sum of N input data bytes D1 ... DN, + * + * A = A0 + D1 + D2 + ... + DN + * + * where A0 is the initial value. + * + * SSE2 _mm_sad_epu8() can be used for byte sums (see http://bit.ly/2wpUOeD, + * for example) and accumulating the byte sums can use SSE shuffle-adds (see + * the "Integer" section of http://bit.ly/2erPT8t for details). Arm NEON has + * similar instructions. + * + * The adler32 B value (aka s2) sums the A values from each step: + * + * B0 + (A0 + D1) + (A0 + D1 + D2) + ... + (A0 + D1 + D2 + ... + DN) or + * + * B0 + N.A0 + N.D1 + (N-1).D2 + (N-2).D3 + ... + (N-(N-1)).DN + * + * B0 being the initial value. For 32 bytes (ideal for garden-variety SIMD): + * + * B = B0 + 32.A0 + [D1 D2 D3 ... D32] x [32 31 30 ... 1]. + * + * Adjacent blocks of 32 input bytes can be iterated with the expressions to + * compute the adler32 s1 s2 of M >> 32 input bytes [1]. + * + * As M grows, the s1 s2 sums grow. If left unchecked, they would eventually + * overflow the precision of their integer representation (bad). However, s1 + * and s2 also need to be computed modulo the adler BASE value (reduced). If + * at most NMAX bytes are processed before a reduce, s1 s2 _cannot_ overflow + * a uint32_t type (the NMAX constraint) [2]. + * + * [1] the iterative equations for s2 contain constant factors; these can be + * hoisted from the n-blocks do loop of the SIMD code. + * + * [2] zlib adler32_z() uses this fact to implement NMAX-block-based updates + * of the adler s1 s2 of uint32_t type (see adler32.c). + */ + +#include "adler32_simd.h" + +/* Definitions from adler32.c: largest prime smaller than 65536 */ +#define BASE 65521U +/* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ +#define NMAX 5552 + +#if defined(ADLER32_SIMD_SSSE3) + +#include + +uint32_t ZLIB_INTERNAL adler32_simd_( /* SSSE3 */ + uint32_t adler, + const unsigned char *buf, + z_size_t len) +{ + /* + * Split Adler-32 into component sums. + */ + uint32_t s1 = adler & 0xffff; + uint32_t s2 = adler >> 16; + + /* + * Process the data in blocks. + */ + const unsigned BLOCK_SIZE = 1 << 5; + + z_size_t blocks = len / BLOCK_SIZE; + len -= blocks * BLOCK_SIZE; + + while (blocks) + { + unsigned n = NMAX / BLOCK_SIZE; /* The NMAX constraint. */ + if (n > blocks) + n = (unsigned) blocks; + blocks -= n; + + const __m128i tap1 = + _mm_setr_epi8(32,31,30,29,28,27,26,25,24,23,22,21,20,19,18,17); + const __m128i tap2 = + _mm_setr_epi8(16,15,14,13,12,11,10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + const __m128i zero = + _mm_setr_epi8( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + const __m128i ones = + _mm_set_epi16( 1, 1, 1, 1, 1, 1, 1, 1); + + /* + * Process n blocks of data. At most NMAX data bytes can be + * processed before s2 must be reduced modulo BASE. + */ + __m128i v_ps = _mm_set_epi32(0, 0, 0, s1 * n); + __m128i v_s2 = _mm_set_epi32(0, 0, 0, s2); + __m128i v_s1 = _mm_set_epi32(0, 0, 0, 0); + + do { + /* + * Load 32 input bytes. + */ + const __m128i bytes1 = _mm_loadu_si128((__m128i*)(buf)); + const __m128i bytes2 = _mm_loadu_si128((__m128i*)(buf + 16)); + + /* + * Add previous block byte sum to v_ps. + */ + v_ps = _mm_add_epi32(v_ps, v_s1); + + /* + * Horizontally add the bytes for s1, multiply-adds the + * bytes by [ 32, 31, 30, ... ] for s2. + */ + v_s1 = _mm_add_epi32(v_s1, _mm_sad_epu8(bytes1, zero)); + const __m128i mad1 = _mm_maddubs_epi16(bytes1, tap1); + v_s2 = _mm_add_epi32(v_s2, _mm_madd_epi16(mad1, ones)); + + v_s1 = _mm_add_epi32(v_s1, _mm_sad_epu8(bytes2, zero)); + const __m128i mad2 = _mm_maddubs_epi16(bytes2, tap2); + v_s2 = _mm_add_epi32(v_s2, _mm_madd_epi16(mad2, ones)); + + buf += BLOCK_SIZE; + + } while (--n); + + v_s2 = _mm_add_epi32(v_s2, _mm_slli_epi32(v_ps, 5)); + + /* + * Sum epi32 ints v_s1(s2) and accumulate in s1(s2). + */ + +#define S23O1 _MM_SHUFFLE(2,3,0,1) /* A B C D -> B A D C */ +#define S1O32 _MM_SHUFFLE(1,0,3,2) /* A B C D -> C D A B */ + + v_s1 = _mm_add_epi32(v_s1, _mm_shuffle_epi32(v_s1, S23O1)); + v_s1 = _mm_add_epi32(v_s1, _mm_shuffle_epi32(v_s1, S1O32)); + + s1 += _mm_cvtsi128_si32(v_s1); + + v_s2 = _mm_add_epi32(v_s2, _mm_shuffle_epi32(v_s2, S23O1)); + v_s2 = _mm_add_epi32(v_s2, _mm_shuffle_epi32(v_s2, S1O32)); + + s2 = _mm_cvtsi128_si32(v_s2); + +#undef S23O1 +#undef S1O32 + + /* + * Reduce. + */ + s1 %= BASE; + s2 %= BASE; + } + + /* + * Handle leftover data. + */ + if (len) { + if (len >= 16) { + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + len -= 16; + } + + while (len--) { + s2 += (s1 += *buf++); + } + + if (s1 >= BASE) + s1 -= BASE; + s2 %= BASE; + } + + /* + * Return the recombined sums. + */ + return s1 | (s2 << 16); +} + +#elif defined(ADLER32_SIMD_NEON) + +#include + +uint32_t ZLIB_INTERNAL adler32_simd_( /* NEON */ + uint32_t adler, + const unsigned char *buf, + z_size_t len) +{ + /* + * Split Adler-32 into component sums. + */ + uint32_t s1 = adler & 0xffff; + uint32_t s2 = adler >> 16; + + /* + * Serially compute s1 & s2, until the data is 16-byte aligned. + */ + if ((uintptr_t)buf & 15) { + while ((uintptr_t)buf & 15) { + s2 += (s1 += *buf++); + --len; + } + + if (s1 >= BASE) + s1 -= BASE; + s2 %= BASE; + } + + /* + * Process the data in blocks. + */ + const unsigned BLOCK_SIZE = 1 << 5; + + z_size_t blocks = len / BLOCK_SIZE; + len -= blocks * BLOCK_SIZE; + + while (blocks) + { + unsigned n = NMAX / BLOCK_SIZE; /* The NMAX constraint. */ + if (n > blocks) + n = (unsigned) blocks; + blocks -= n; + + /* + * Process n blocks of data. At most NMAX data bytes can be + * processed before s2 must be reduced modulo BASE. + */ + uint32x4_t v_s2 = (uint32x4_t) { 0, 0, 0, s1 * n }; + uint32x4_t v_s1 = (uint32x4_t) { 0, 0, 0, 0 }; + + uint16x8_t v_column_sum_1 = vdupq_n_u16(0); + uint16x8_t v_column_sum_2 = vdupq_n_u16(0); + uint16x8_t v_column_sum_3 = vdupq_n_u16(0); + uint16x8_t v_column_sum_4 = vdupq_n_u16(0); + + do { + /* + * Load 32 input bytes. + */ + const uint8x16_t bytes1 = vld1q_u8((uint8_t*)(buf)); + const uint8x16_t bytes2 = vld1q_u8((uint8_t*)(buf + 16)); + + /* + * Add previous block byte sum to v_s2. + */ + v_s2 = vaddq_u32(v_s2, v_s1); + + /* + * Horizontally add the bytes for s1. + */ + v_s1 = vpadalq_u16(v_s1, vpadalq_u8(vpaddlq_u8(bytes1), bytes2)); + + /* + * Vertically add the bytes for s2. + */ + v_column_sum_1 = vaddw_u8(v_column_sum_1, vget_low_u8 (bytes1)); + v_column_sum_2 = vaddw_u8(v_column_sum_2, vget_high_u8(bytes1)); + v_column_sum_3 = vaddw_u8(v_column_sum_3, vget_low_u8 (bytes2)); + v_column_sum_4 = vaddw_u8(v_column_sum_4, vget_high_u8(bytes2)); + + buf += BLOCK_SIZE; + + } while (--n); + + v_s2 = vshlq_n_u32(v_s2, 5); + + /* + * Multiply-add bytes by [ 32, 31, 30, ... ] for s2. + */ + v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_column_sum_1), + (uint16x4_t) { 32, 31, 30, 29 }); + v_s2 = vmlal_u16(v_s2, vget_high_u16(v_column_sum_1), + (uint16x4_t) { 28, 27, 26, 25 }); + v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_column_sum_2), + (uint16x4_t) { 24, 23, 22, 21 }); + v_s2 = vmlal_u16(v_s2, vget_high_u16(v_column_sum_2), + (uint16x4_t) { 20, 19, 18, 17 }); + v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_column_sum_3), + (uint16x4_t) { 16, 15, 14, 13 }); + v_s2 = vmlal_u16(v_s2, vget_high_u16(v_column_sum_3), + (uint16x4_t) { 12, 11, 10, 9 }); + v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_column_sum_4), + (uint16x4_t) { 8, 7, 6, 5 }); + v_s2 = vmlal_u16(v_s2, vget_high_u16(v_column_sum_4), + (uint16x4_t) { 4, 3, 2, 1 }); + + /* + * Sum epi32 ints v_s1(s2) and accumulate in s1(s2). + */ + uint32x2_t sum1 = vpadd_u32(vget_low_u32(v_s1), vget_high_u32(v_s1)); + uint32x2_t sum2 = vpadd_u32(vget_low_u32(v_s2), vget_high_u32(v_s2)); + uint32x2_t s1s2 = vpadd_u32(sum1, sum2); + + s1 += vget_lane_u32(s1s2, 0); + s2 += vget_lane_u32(s1s2, 1); + + /* + * Reduce. + */ + s1 %= BASE; + s2 %= BASE; + } + + /* + * Handle leftover data. + */ + if (len) { + if (len >= 16) { + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + len -= 16; + } + + while (len--) { + s2 += (s1 += *buf++); + } + + if (s1 >= BASE) + s1 -= BASE; + s2 %= BASE; + } + + /* + * Return the recombined sums. + */ + return s1 | (s2 << 16); +} + +#endif /* ADLER32_SIMD_SSSE3 */ diff --git a/third_party/zlib/adler32_simd.h b/third_party/zlib/adler32_simd.h new file mode 100644 index 0000000000..52bb14d167 --- /dev/null +++ b/third_party/zlib/adler32_simd.h @@ -0,0 +1,16 @@ +/* adler32_simd.h + * + * Copyright 2017 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#include + +#include "zconf.h" +#include "zutil.h" + +uint32_t ZLIB_INTERNAL adler32_simd_( + uint32_t adler, + const unsigned char *buf, + z_size_t len); diff --git a/third_party/zlib/chromeconf.h b/third_party/zlib/chromeconf.h new file mode 100644 index 0000000000..5ecf29edbf --- /dev/null +++ b/third_party/zlib/chromeconf.h @@ -0,0 +1,199 @@ +/* Copyright 2017 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. */ + +#ifndef THIRD_PARTY_ZLIB_CHROMECONF_H_ +#define THIRD_PARTY_ZLIB_CHROMECONF_H_ + +#if defined(COMPONENT_BUILD) +#if defined(WIN32) +#if defined(ZLIB_IMPLEMENTATION) +#define ZEXTERN __declspec(dllexport) +#else +#define ZEXTERN __declspec(dllimport) +#endif +#elif defined(ZLIB_IMPLEMENTATION) +#define ZEXTERN __attribute__((visibility("default"))) +#endif +#endif + +/* Rename all zlib names with a Cr_z_ prefix. This is based on the Z_PREFIX + * option from zconf.h, but with a custom prefix. Where zconf.h would rename + * both a macro and its underscore-suffixed internal implementation (such as + * deflateInit2 and deflateInit2_), only the implementation is renamed here. + * The Byte type is also omitted. + * + * To generate this list, run + * sed -rn -e 's/^# *define +([^ ]+) +(z_[^ ]+)$/#define \1 Cr_\2/p' zconf.h + * (use -E instead of -r on macOS). + * + * gzread is also addressed by modifications in gzread.c and zlib.h. */ + +#define Z_CR_PREFIX_SET + +#define _dist_code Cr_z__dist_code +#define _length_code Cr_z__length_code +#define _tr_align Cr_z__tr_align +#define _tr_flush_bits Cr_z__tr_flush_bits +#define _tr_flush_block Cr_z__tr_flush_block +#define _tr_init Cr_z__tr_init +#define _tr_stored_block Cr_z__tr_stored_block +#define _tr_tally Cr_z__tr_tally +#define adler32 Cr_z_adler32 +#define adler32_combine Cr_z_adler32_combine +#define adler32_combine64 Cr_z_adler32_combine64 +#define adler32_z Cr_z_adler32_z +#define compress Cr_z_compress +#define compress2 Cr_z_compress2 +#define compressBound Cr_z_compressBound +#define crc32 Cr_z_crc32 +#define crc32_combine Cr_z_crc32_combine +#define crc32_combine64 Cr_z_crc32_combine64 +#define crc32_z Cr_z_crc32_z +#define deflate Cr_z_deflate +#define deflateBound Cr_z_deflateBound +#define deflateCopy Cr_z_deflateCopy +#define deflateEnd Cr_z_deflateEnd +#define deflateGetDictionary Cr_z_deflateGetDictionary +/* #undef deflateInit */ +/* #undef deflateInit2 */ +#define deflateInit2_ Cr_z_deflateInit2_ +#define deflateInit_ Cr_z_deflateInit_ +#define deflateParams Cr_z_deflateParams +#define deflatePending Cr_z_deflatePending +#define deflatePrime Cr_z_deflatePrime +#define deflateReset Cr_z_deflateReset +#define deflateResetKeep Cr_z_deflateResetKeep +#define deflateSetDictionary Cr_z_deflateSetDictionary +#define deflateSetHeader Cr_z_deflateSetHeader +#define deflateTune Cr_z_deflateTune +#define deflate_copyright Cr_z_deflate_copyright +#define get_crc_table Cr_z_get_crc_table +#define gz_error Cr_z_gz_error +#define gz_intmax Cr_z_gz_intmax +#define gz_strwinerror Cr_z_gz_strwinerror +#define gzbuffer Cr_z_gzbuffer +#define gzclearerr Cr_z_gzclearerr +#define gzclose Cr_z_gzclose +#define gzclose_r Cr_z_gzclose_r +#define gzclose_w Cr_z_gzclose_w +#define gzdirect Cr_z_gzdirect +#define gzdopen Cr_z_gzdopen +#define gzeof Cr_z_gzeof +#define gzerror Cr_z_gzerror +#define gzflush Cr_z_gzflush +#define gzfread Cr_z_gzfread +#define gzfwrite Cr_z_gzfwrite +#define gzgetc Cr_z_gzgetc +#define gzgetc_ Cr_z_gzgetc_ +#define gzgets Cr_z_gzgets +#define gzoffset Cr_z_gzoffset +#define gzoffset64 Cr_z_gzoffset64 +#define gzopen Cr_z_gzopen +#define gzopen64 Cr_z_gzopen64 +#define gzopen_w Cr_z_gzopen_w +#define gzprintf Cr_z_gzprintf +#define gzputc Cr_z_gzputc +#define gzputs Cr_z_gzputs +#define gzread Cr_z_gzread +#define gzrewind Cr_z_gzrewind +#define gzseek Cr_z_gzseek +#define gzseek64 Cr_z_gzseek64 +#define gzsetparams Cr_z_gzsetparams +#define gztell Cr_z_gztell +#define gztell64 Cr_z_gztell64 +#define gzungetc Cr_z_gzungetc +#define gzvprintf Cr_z_gzvprintf +#define gzwrite Cr_z_gzwrite +#define inflate Cr_z_inflate +#define inflateBack Cr_z_inflateBack +#define inflateBackEnd Cr_z_inflateBackEnd +/* #undef inflateBackInit */ +#define inflateBackInit_ Cr_z_inflateBackInit_ +#define inflateCodesUsed Cr_z_inflateCodesUsed +#define inflateCopy Cr_z_inflateCopy +#define inflateEnd Cr_z_inflateEnd +#define inflateGetDictionary Cr_z_inflateGetDictionary +#define inflateGetHeader Cr_z_inflateGetHeader +/* #undef inflateInit */ +/* #undef inflateInit2 */ +#define inflateInit2_ Cr_z_inflateInit2_ +#define inflateInit_ Cr_z_inflateInit_ +#define inflateMark Cr_z_inflateMark +#define inflatePrime Cr_z_inflatePrime +#define inflateReset Cr_z_inflateReset +#define inflateReset2 Cr_z_inflateReset2 +#define inflateResetKeep Cr_z_inflateResetKeep +#define inflateSetDictionary Cr_z_inflateSetDictionary +#define inflateSync Cr_z_inflateSync +#define inflateSyncPoint Cr_z_inflateSyncPoint +#define inflateUndermine Cr_z_inflateUndermine +#define inflateValidate Cr_z_inflateValidate +#define inflate_copyright Cr_z_inflate_copyright +#define inflate_fast Cr_z_inflate_fast +#define inflate_table Cr_z_inflate_table +#define uncompress Cr_z_uncompress +#define uncompress2 Cr_z_uncompress2 +#define zError Cr_z_zError +#define zcalloc Cr_z_zcalloc +#define zcfree Cr_z_zcfree +#define zlibCompileFlags Cr_z_zlibCompileFlags +#define zlibVersion Cr_z_zlibVersion +/* #undef Byte */ +#define Bytef Cr_z_Bytef +#define alloc_func Cr_z_alloc_func +#define charf Cr_z_charf +#define free_func Cr_z_free_func +#define gzFile Cr_z_gzFile +#define gz_header Cr_z_gz_header +#define gz_headerp Cr_z_gz_headerp +#define in_func Cr_z_in_func +#define intf Cr_z_intf +#define out_func Cr_z_out_func +#define uInt Cr_z_uInt +#define uIntf Cr_z_uIntf +#define uLong Cr_z_uLong +#define uLongf Cr_z_uLongf +#define voidp Cr_z_voidp +#define voidpc Cr_z_voidpc +#define voidpf Cr_z_voidpf +#define gz_header_s Cr_z_gz_header_s +/* #undef internal_state */ +/* #undef z_off64_t */ + +/* An exported symbol that isn't handled by Z_PREFIX in zconf.h */ +#define z_errmsg Cr_z_z_errmsg + +/* Symbols added in simd.patch */ +#define copy_with_crc Cr_z_copy_with_crc +#define crc_finalize Cr_z_crc_finalize +#define crc_fold_512to32 Cr_z_crc_fold_512to32 +#define crc_fold_copy Cr_z_crc_fold_copy +#define crc_fold_init Cr_z_crc_fold_init +#define crc_reset Cr_z_crc_reset +#define fill_window_sse Cr_z_fill_window_sse +#define deflate_read_buf Cr_z_deflate_read_buf +#define x86_check_features Cr_z_x86_check_features +#define x86_cpu_enable_simd Cr_z_x86_cpu_enable_simd + +/* Symbols added by adler_simd.c */ +#define adler32_simd_ Cr_z_adler32_simd_ +#define x86_cpu_enable_ssse3 Cr_z_x86_cpu_enable_ssse3 + +/* Symbols added by contrib/optimizations/inffast_chunk */ +#define inflate_fast_chunk_ Cr_z_inflate_fast_chunk_ + +/* Symbols added by crc32_simd.c */ +#define crc32_sse42_simd_ Cr_z_crc32_sse42_simd_ + +/* Symbols added by armv8_crc32 */ +#define arm_cpu_enable_crc32 Cr_z_arm_cpu_enable_crc32 +#define arm_cpu_enable_pmull Cr_z_arm_cpu_enable_pmull +#define arm_check_features Cr_z_arm_check_features +#define armv8_crc32_little Cr_z_armv8_crc32_little + +/* Symbols added by cpu_features.c */ +#define cpu_check_features Cr_z_cpu_check_features +#define x86_cpu_enable_sse2 Cr_z_x86_cpu_enable_sse2 + +#endif /* THIRD_PARTY_ZLIB_CHROMECONF_H_ */ diff --git a/third_party/zlib/compress.c b/third_party/zlib/compress.c index ea4dfbe9d7..6e8517335d 100644 --- a/third_party/zlib/compress.c +++ b/third_party/zlib/compress.c @@ -1,5 +1,5 @@ /* compress.c -- compress a memory buffer - * Copyright (C) 1995-2005 Jean-loup Gailly. + * Copyright (C) 1995-2005, 2014, 2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -28,16 +28,11 @@ int ZEXPORT compress2 (dest, destLen, source, sourceLen, level) { z_stream stream; int err; + const uInt max = (uInt)-1; + uLong left; - stream.next_in = (Bytef*)source; - stream.avail_in = (uInt)sourceLen; -#ifdef MAXSEG_64K - /* Check for source > 64K on 16-bit machine: */ - if ((uLong)stream.avail_in != sourceLen) return Z_BUF_ERROR; -#endif - stream.next_out = dest; - stream.avail_out = (uInt)*destLen; - if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR; + left = *destLen; + *destLen = 0; stream.zalloc = (alloc_func)0; stream.zfree = (free_func)0; @@ -46,15 +41,26 @@ int ZEXPORT compress2 (dest, destLen, source, sourceLen, level) err = deflateInit(&stream, level); if (err != Z_OK) return err; - err = deflate(&stream, Z_FINISH); - if (err != Z_STREAM_END) { - deflateEnd(&stream); - return err == Z_OK ? Z_BUF_ERROR : err; - } - *destLen = stream.total_out; + stream.next_out = dest; + stream.avail_out = 0; + stream.next_in = (z_const Bytef *)source; + stream.avail_in = 0; - err = deflateEnd(&stream); - return err; + do { + if (stream.avail_out == 0) { + stream.avail_out = left > (uLong)max ? max : (uInt)left; + left -= stream.avail_out; + } + if (stream.avail_in == 0) { + stream.avail_in = sourceLen > (uLong)max ? max : (uInt)sourceLen; + sourceLen -= stream.avail_in; + } + err = deflate(&stream, sourceLen ? Z_NO_FLUSH : Z_FINISH); + } while (err == Z_OK); + + *destLen = stream.total_out; + deflateEnd(&stream); + return err == Z_STREAM_END ? Z_OK : err; } /* =========================================================================== @@ -75,6 +81,16 @@ int ZEXPORT compress (dest, destLen, source, sourceLen) uLong ZEXPORT compressBound (sourceLen) uLong sourceLen; { - return sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + - (sourceLen >> 25) + 13; + sourceLen = sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + + (sourceLen >> 25) + 13; + /* FIXME(cavalcantii): usage of CRC32 Castagnoli as a hash function + * for the hash table of symbols used for compression has a side effect + * where for compression level [4, 5] it will increase the output buffer size + * by 0.1% (i.e. less than 1%) for a high entropy input (i.e. random data). + * To avoid a scenario where client code would fail, for safety we increase + * the expected output size by 0.8% (i.e. 8x more than the worst scenario). + * See: http://crbug.com/990489 + */ + sourceLen += sourceLen >> 7; // Equivalent to 1.0078125 + return sourceLen; } diff --git a/third_party/zlib/contrib/bench/zlib_bench.cc b/third_party/zlib/contrib/bench/zlib_bench.cc new file mode 100644 index 0000000000..bc2f741a25 --- /dev/null +++ b/third_party/zlib/contrib/bench/zlib_bench.cc @@ -0,0 +1,347 @@ +/* + * Copyright 2018 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + * + * A benchmark test harness for measuring decoding performance of gzip or zlib + * (deflate) encoded compressed data. Given a file containing any data, encode + * (compress) it into gzip or zlib format and then decode (uncompress). Output + * the median and maximum encoding and decoding rates in MB/s. + * + * Raw deflate (no gzip or zlib stream wrapper) mode is also supported. Select + * it with the [raw] argument. Use the [gzip] [zlib] arguments to select those + * stream wrappers. + * + * Note this code can be compiled outside of the Chromium build system against + * the system zlib (-lz) with g++ or clang++ as follows: + * + * g++|clang++ -O3 -Wall -std=c++11 -lstdc++ -lz zlib_bench.cc + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "zlib.h" + +void error_exit(const char* error, int code) { + fprintf(stderr, "%s (%d)\n", error, code); + exit(code); +} + +inline char* string_data(std::string* s) { + return s->empty() ? nullptr : &*s->begin(); +} + +struct Data { + Data(size_t s) { data.reset(new (std::nothrow) char[size = s]); } + std::unique_ptr data; + size_t size; +}; + +Data read_file_data_or_exit(const char* name) { + std::ifstream file(name, std::ios::in | std::ios::binary); + if (!file) { + perror(name); + exit(1); + } + + file.seekg(0, std::ios::end); + Data data(file.tellg()); + file.seekg(0, std::ios::beg); + + if (file && data.data) + file.read(data.data.get(), data.size); + + if (!file || !data.data || !data.size) { + perror((std::string("failed: reading ") + name).c_str()); + exit(1); + } + + return data; +} + +size_t zlib_estimate_compressed_size(size_t input_size) { + return compressBound(input_size); +} + +enum zlib_wrapper { + kWrapperNONE, + kWrapperZLIB, + kWrapperGZIP, + kWrapperZRAW, +}; + +inline int zlib_stream_wrapper_type(zlib_wrapper type) { + if (type == kWrapperZLIB) // zlib DEFLATE stream wrapper + return MAX_WBITS; + if (type == kWrapperGZIP) // gzip DEFLATE stream wrapper + return MAX_WBITS + 16; + if (type == kWrapperZRAW) // no wrapper, use raw DEFLATE + return -MAX_WBITS; + error_exit("bad wrapper type", int(type)); + return 0; +} + +const char* zlib_wrapper_name(zlib_wrapper type) { + if (type == kWrapperZLIB) + return "ZLIB"; + if (type == kWrapperGZIP) + return "GZIP"; + if (type == kWrapperZRAW) + return "RAW"; + error_exit("bad wrapper type", int(type)); + return nullptr; +} + +static int zlib_strategy = Z_DEFAULT_STRATEGY; + +const char* zlib_level_strategy_name(int compression_level) { + if (compression_level == 0) + return ""; // strategy is meaningless at level 0 + if (zlib_strategy == Z_HUFFMAN_ONLY) + return "huffman "; + if (zlib_strategy == Z_RLE) + return "rle "; + if (zlib_strategy == Z_DEFAULT_STRATEGY) + return ""; + error_exit("bad strategy", zlib_strategy); + return nullptr; +} + +static int zlib_compression_level = Z_DEFAULT_COMPRESSION; + +void zlib_compress( + const zlib_wrapper type, + const char* input, + const size_t input_size, + std::string* output, + bool resize_output = false) +{ + if (resize_output) + output->resize(zlib_estimate_compressed_size(input_size)); + size_t output_size = output->size(); + + z_stream stream; + memset(&stream, 0, sizeof(stream)); + + int result = deflateInit2(&stream, zlib_compression_level, Z_DEFLATED, + zlib_stream_wrapper_type(type), MAX_MEM_LEVEL, zlib_strategy); + if (result != Z_OK) + error_exit("deflateInit2 failed", result); + + stream.next_out = (Bytef*)string_data(output); + stream.avail_out = (uInt)output_size; + stream.next_in = (z_const Bytef*)input; + stream.avail_in = (uInt)input_size; + + result = deflate(&stream, Z_FINISH); + if (result == Z_STREAM_END) + output_size = stream.total_out; + result |= deflateEnd(&stream); + if (result != Z_STREAM_END) + error_exit("compress failed", result); + + if (resize_output) + output->resize(output_size); +} + +void zlib_uncompress( + const zlib_wrapper type, + const std::string& input, + const size_t output_size, + std::string* output) +{ + z_stream stream; + memset(&stream, 0, sizeof(stream)); + + int result = inflateInit2(&stream, zlib_stream_wrapper_type(type)); + if (result != Z_OK) + error_exit("inflateInit2 failed", result); + + stream.next_out = (Bytef*)string_data(output); + stream.avail_out = (uInt)output->size(); + stream.next_in = (z_const Bytef*)input.data(); + stream.avail_in = (uInt)input.size(); + + result = inflate(&stream, Z_FINISH); + if (stream.total_out != output_size) + result = Z_DATA_ERROR; + result |= inflateEnd(&stream); + if (result == Z_STREAM_END) + return; + + std::string error("uncompress failed: "); + if (stream.msg) + error.append(stream.msg); + error_exit(error.c_str(), result); +} + +void verify_equal(const char* input, size_t size, std::string* output) { + const char* data = string_data(output); + if (output->size() == size && !memcmp(data, input, size)) + return; + fprintf(stderr, "uncompressed data does not match the input data\n"); + exit(3); +} + +void zlib_file(const char* name, const zlib_wrapper type) { + /* + * Read the file data. + */ + const auto file = read_file_data_or_exit(name); + const int length = static_cast(file.size); + const char* data = file.data.get(); + + /* + * Report compression strategy and file name. + */ + const char* strategy = zlib_level_strategy_name(zlib_compression_level); + printf("%s%-40s :\n", strategy, name); + + /* + * Chop the data into blocks. + */ + const int block_size = 1 << 20; + const int blocks = (length + block_size - 1) / block_size; + + std::vector input(blocks); + std::vector input_length(blocks); + std::vector compressed(blocks); + std::vector output(blocks); + + for (int b = 0; b < blocks; ++b) { + int input_start = b * block_size; + int input_limit = std::min((b + 1) * block_size, length); + input[b] = data + input_start; + input_length[b] = input_limit - input_start; + } + + /* + * Run the zlib compress/uncompress loop a few times with |repeats| to + * process about 10MB of data if the length is small relative to 10MB. + * If length is large relative to 10MB, process the data once. + */ + const int mega_byte = 1024 * 1024; + const int repeats = (10 * mega_byte + length) / (length + 1); + const int runs = 5; + double ctime[runs]; + double utime[runs]; + + for (int run = 0; run < runs; ++run) { + const auto now = [] { return std::chrono::steady_clock::now(); }; + + // Pre-grow the output buffer so we don't measure string resize time. + for (int b = 0; b < blocks; ++b) + compressed[b].resize(zlib_estimate_compressed_size(block_size)); + + auto start = now(); + for (int b = 0; b < blocks; ++b) + for (int r = 0; r < repeats; ++r) + zlib_compress(type, input[b], input_length[b], &compressed[b]); + ctime[run] = std::chrono::duration(now() - start).count(); + + // Compress again, resizing compressed, so we don't leave junk at the + // end of the compressed string that could confuse zlib_uncompress(). + for (int b = 0; b < blocks; ++b) + zlib_compress(type, input[b], input_length[b], &compressed[b], true); + + for (int b = 0; b < blocks; ++b) + output[b].resize(input_length[b]); + + start = now(); + for (int r = 0; r < repeats; ++r) + for (int b = 0; b < blocks; ++b) + zlib_uncompress(type, compressed[b], input_length[b], &output[b]); + utime[run] = std::chrono::duration(now() - start).count(); + + for (int b = 0; b < blocks; ++b) + verify_equal(input[b], input_length[b], &output[b]); + } + + /* + * Output the median/maximum compress/uncompress rates in MB/s. + */ + size_t output_length = 0; + for (size_t i = 0; i < compressed.size(); ++i) + output_length += compressed[i].size(); + + std::sort(ctime, ctime + runs); + std::sort(utime, utime + runs); + + double deflate_rate_med = length * repeats / mega_byte / ctime[runs / 2]; + double inflate_rate_med = length * repeats / mega_byte / utime[runs / 2]; + double deflate_rate_max = length * repeats / mega_byte / ctime[0]; + double inflate_rate_max = length * repeats / mega_byte / utime[0]; + + // type, block size, compression ratio, etc + printf("%s: [b %dM] bytes %6d -> %6u %4.1f%%", + zlib_wrapper_name(type), block_size / (1 << 20), length, + static_cast(output_length), output_length * 100.0 / length); + + // compress / uncompress median (max) rates + printf(" comp %5.1f (%5.1f) MB/s uncomp %5.1f (%5.1f) MB/s\n", + deflate_rate_med, deflate_rate_max, inflate_rate_med, inflate_rate_max); +} + +static int argn = 1; + +char* get_option(int argc, char* argv[], const char* option) { + if (argn < argc) + return !strcmp(argv[argn], option) ? argv[argn++] : nullptr; + return nullptr; +} + +bool get_compression(int argc, char* argv[], int* value) { + if (argn < argc) + *value = isdigit(argv[argn][0]) ? atoi(argv[argn++]) : -1; + return *value >= 0 && *value <= 9; +} + +void usage_exit(const char* program) { + printf( + "usage: %s gzip|zlib|raw [--compression 0:9] [--huffman|--rle] " + "files...\n", + program); + exit(1); +} + +int main(int argc, char* argv[]) { + zlib_wrapper type; + if (get_option(argc, argv, "zlib")) + type = kWrapperZLIB; + else if (get_option(argc, argv, "gzip")) + type = kWrapperGZIP; + else if (get_option(argc, argv, "raw")) + type = kWrapperZRAW; + else + usage_exit(argv[0]); + + while (argn < argc && argv[argn][0] == '-') { + if (get_option(argc, argv, "--compression")) { + if (!get_compression(argc, argv, &zlib_compression_level)) + usage_exit(argv[0]); + } else if (get_option(argc, argv, "--huffman")) { + zlib_strategy = Z_HUFFMAN_ONLY; + } else if (get_option(argc, argv, "--rle")) { + zlib_strategy = Z_RLE; + } else { + usage_exit(argv[0]); + } + } + + if (argn >= argc) + usage_exit(argv[0]); + while (argn < argc) + zlib_file(argv[argn++], type); + + return 0; +} diff --git a/third_party/zlib/contrib/minizip/crypt.h b/third_party/zlib/contrib/minizip/crypt.h index a01d08d932..1e9e8200b2 100644 --- a/third_party/zlib/contrib/minizip/crypt.h +++ b/third_party/zlib/contrib/minizip/crypt.h @@ -32,7 +32,7 @@ /*********************************************************************** * Return the next byte in the pseudo-random sequence */ -static int decrypt_byte(unsigned long* pkeys, const unsigned long* pcrc_32_tab) +static int decrypt_byte(unsigned long* pkeys, const z_crc_t* pcrc_32_tab) { unsigned temp; /* POTENTIAL BUG: temp*(temp^1) may overflow in an * unpredictable manner on 16-bit systems; not a problem @@ -45,7 +45,7 @@ static int decrypt_byte(unsigned long* pkeys, const unsigned long* pcrc_32_tab) /*********************************************************************** * Update the encryption keys with the next byte of plain text */ -static int update_keys(unsigned long* pkeys,const unsigned long* pcrc_32_tab,int c) +static int update_keys(unsigned long* pkeys,const z_crc_t* pcrc_32_tab,int c) { (*(pkeys+0)) = CRC32((*(pkeys+0)), c); (*(pkeys+1)) += (*(pkeys+0)) & 0xff; @@ -62,7 +62,7 @@ static int update_keys(unsigned long* pkeys,const unsigned long* pcrc_32_tab,int * Initialize the encryption keys and the random header according to * the given password. */ -static void init_keys(const char* passwd,unsigned long* pkeys,const unsigned long* pcrc_32_tab) +static void init_keys(const char* passwd,unsigned long* pkeys,const z_crc_t* pcrc_32_tab) { *(pkeys+0) = 305419896L; *(pkeys+1) = 591751049L; @@ -91,7 +91,7 @@ static int crypthead(const char* passwd, /* password string */ unsigned char* buf, /* where to write header */ int bufSize, unsigned long* pkeys, - const unsigned long* pcrc_32_tab, + const z_crc_t* pcrc_32_tab, unsigned long crcForCrypting) { int n; /* index in random header */ diff --git a/third_party/zlib/contrib/minizip/ioapi.c b/third_party/zlib/contrib/minizip/ioapi.c index 49958f61ff..543910b5e1 100644 --- a/third_party/zlib/contrib/minizip/ioapi.c +++ b/third_party/zlib/contrib/minizip/ioapi.c @@ -10,10 +10,22 @@ */ -#if (defined(_WIN32)) +#if defined(_WIN32) && (!(defined(_CRT_SECURE_NO_WARNINGS))) #define _CRT_SECURE_NO_WARNINGS #endif +#if defined(__APPLE__) || defined(__Fuchsia__) || defined(IOAPI_NO_64) +// In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions +#define FOPEN_FUNC(filename, mode) fopen(filename, mode) +#define FTELLO_FUNC(stream) ftello(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko(stream, offset, origin) +#else +#define FOPEN_FUNC(filename, mode) fopen64(filename, mode) +#define FTELLO_FUNC(stream) ftello64(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko64(stream, offset, origin) +#endif + + #include "ioapi.h" voidpf call_zopen64 (const zlib_filefunc64_32_def* pfilefunc,const void*filename,int mode) @@ -47,7 +59,7 @@ ZPOS64_T call_ztell64 (const zlib_filefunc64_32_def* pfilefunc,voidpf filestream else { uLong tell_uLong = (*(pfilefunc->ztell32_file))(pfilefunc->zfile_func64.opaque,filestream); - if ((tell_uLong) == ((uLong)-1)) + if ((tell_uLong) == MAXU32) return (ZPOS64_T)-1; else return tell_uLong; @@ -112,7 +124,7 @@ static voidpf ZCALLBACK fopen64_file_func (voidpf opaque, const void* filename, mode_fopen = "wb"; if ((filename!=NULL) && (mode_fopen != NULL)) - file = fopen64((const char*)filename, mode_fopen); + file = FOPEN_FUNC((const char*)filename, mode_fopen); return file; } @@ -142,7 +154,7 @@ static long ZCALLBACK ftell_file_func (voidpf opaque, voidpf stream) static ZPOS64_T ZCALLBACK ftell64_file_func (voidpf opaque, voidpf stream) { ZPOS64_T ret; - ret = ftello64((FILE *)stream); + ret = FTELLO_FUNC((FILE *)stream); return ret; } @@ -188,7 +200,7 @@ static long ZCALLBACK fseek64_file_func (voidpf opaque, voidpf stream, ZPOS64_T } ret = 0; - if(fseeko64((FILE *)stream, offset, fseek_origin) != 0) + if(FSEEKO_FUNC((FILE *)stream, offset, fseek_origin) != 0) ret = -1; return ret; diff --git a/third_party/zlib/contrib/minizip/ioapi.h b/third_party/zlib/contrib/minizip/ioapi.h index dfa3b6ca3a..c1b7a54847 100644 --- a/third_party/zlib/contrib/minizip/ioapi.h +++ b/third_party/zlib/contrib/minizip/ioapi.h @@ -21,7 +21,7 @@ #ifndef _ZLIBIOAPI64_H #define _ZLIBIOAPI64_H -#if (!defined(_WIN32)) && (!defined(WIN32)) +#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) // Linux needs this to support file operation on files larger then 4+GB // But might need better if/def to select just the platforms that needs them. @@ -38,6 +38,7 @@ #ifndef _FILE_OFFSET_BIT #define _FILE_OFFSET_BIT 64 #endif + #endif #include @@ -49,6 +50,11 @@ #define ftello64 ftell #define fseeko64 fseek #else +#ifdef __FreeBSD__ +#define fopen64 fopen +#define ftello64 ftello +#define fseeko64 fseeko +#endif #ifdef _MSC_VER #define fopen64 fopen #if (_MSC_VER >= 1400) && (!(defined(NO_MSCVER_FILE64_FUNC))) @@ -85,6 +91,8 @@ typedef 64BIT_INT_CUSTOM_TYPE ZPOS64_T; typedef uint64_t ZPOS64_T; #else +/* Maximum unsigned 32-bit value used as placeholder for zip64 */ +#define MAXU32 0xffffffff #if defined(_MSC_VER) || defined(__BORLANDC__) typedef unsigned __int64 ZPOS64_T; diff --git a/third_party/zlib/contrib/minizip/iowin32.c b/third_party/zlib/contrib/minizip/iowin32.c index 6a2a883be7..c6bc314b3c 100644 --- a/third_party/zlib/contrib/minizip/iowin32.c +++ b/third_party/zlib/contrib/minizip/iowin32.c @@ -25,6 +25,19 @@ #define INVALID_SET_FILE_POINTER ((DWORD)-1) #endif + +#ifdef _WIN32_WINNT +#undef _WIN32_WINNT +#define _WIN32_WINNT 0x601 +#endif + +#if !defined(IOWIN32_USING_WINRT_API) +#if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_APP) +// Windows Store or Universal Windows Platform +#define IOWIN32_USING_WINRT_API 1 +#endif +#endif + voidpf ZCALLBACK win32_open_file_func OF((voidpf opaque, const char* filename, int mode)); uLong ZCALLBACK win32_read_file_func OF((voidpf opaque, voidpf stream, void* buf, uLong size)); uLong ZCALLBACK win32_write_file_func OF((voidpf opaque, voidpf stream, const void* buf, uLong size)); @@ -93,8 +106,22 @@ voidpf ZCALLBACK win32_open64_file_func (voidpf opaque,const void* filename,int win32_translate_open_mode(mode,&dwDesiredAccess,&dwCreationDisposition,&dwShareMode,&dwFlagsAndAttributes); +#ifdef IOWIN32_USING_WINRT_API +#ifdef UNICODE + if ((filename!=NULL) && (dwDesiredAccess != 0)) + hFile = CreateFile2((LPCTSTR)filename, dwDesiredAccess, dwShareMode, dwCreationDisposition, NULL); +#else + if ((filename!=NULL) && (dwDesiredAccess != 0)) + { + WCHAR filenameW[FILENAME_MAX + 0x200 + 1]; + MultiByteToWideChar(CP_ACP,0,(const char*)filename,-1,filenameW,FILENAME_MAX + 0x200); + hFile = CreateFile2(filenameW, dwDesiredAccess, dwShareMode, dwCreationDisposition, NULL); + } +#endif +#else if ((filename!=NULL) && (dwDesiredAccess != 0)) hFile = CreateFile((LPCTSTR)filename, dwDesiredAccess, dwShareMode, NULL, dwCreationDisposition, dwFlagsAndAttributes, NULL); +#endif return win32_build_iowin(hFile); } @@ -108,8 +135,17 @@ voidpf ZCALLBACK win32_open64_file_funcA (voidpf opaque,const void* filename,int win32_translate_open_mode(mode,&dwDesiredAccess,&dwCreationDisposition,&dwShareMode,&dwFlagsAndAttributes); +#ifdef IOWIN32_USING_WINRT_API + if ((filename!=NULL) && (dwDesiredAccess != 0)) + { + WCHAR filenameW[FILENAME_MAX + 0x200 + 1]; + MultiByteToWideChar(CP_ACP,0,(const char*)filename,-1,filenameW,FILENAME_MAX + 0x200); + hFile = CreateFile2(filenameW, dwDesiredAccess, dwShareMode, dwCreationDisposition, NULL); + } +#else if ((filename!=NULL) && (dwDesiredAccess != 0)) hFile = CreateFileA((LPCSTR)filename, dwDesiredAccess, dwShareMode, NULL, dwCreationDisposition, dwFlagsAndAttributes, NULL); +#endif return win32_build_iowin(hFile); } @@ -123,8 +159,13 @@ voidpf ZCALLBACK win32_open64_file_funcW (voidpf opaque,const void* filename,int win32_translate_open_mode(mode,&dwDesiredAccess,&dwCreationDisposition,&dwShareMode,&dwFlagsAndAttributes); +#ifdef IOWIN32_USING_WINRT_API + if ((filename!=NULL) && (dwDesiredAccess != 0)) + hFile = CreateFile2((LPCWSTR)filename, dwDesiredAccess, dwShareMode, dwCreationDisposition,NULL); +#else if ((filename!=NULL) && (dwDesiredAccess != 0)) hFile = CreateFileW((LPCWSTR)filename, dwDesiredAccess, dwShareMode, NULL, dwCreationDisposition, dwFlagsAndAttributes, NULL); +#endif return win32_build_iowin(hFile); } @@ -138,8 +179,22 @@ voidpf ZCALLBACK win32_open_file_func (voidpf opaque,const char* filename,int mo win32_translate_open_mode(mode,&dwDesiredAccess,&dwCreationDisposition,&dwShareMode,&dwFlagsAndAttributes); +#ifdef IOWIN32_USING_WINRT_API +#ifdef UNICODE + if ((filename!=NULL) && (dwDesiredAccess != 0)) + hFile = CreateFile2((LPCTSTR)filename, dwDesiredAccess, dwShareMode, dwCreationDisposition, NULL); +#else + if ((filename!=NULL) && (dwDesiredAccess != 0)) + { + WCHAR filenameW[FILENAME_MAX + 0x200 + 1]; + MultiByteToWideChar(CP_ACP,0,(const char*)filename,-1,filenameW,FILENAME_MAX + 0x200); + hFile = CreateFile2(filenameW, dwDesiredAccess, dwShareMode, dwCreationDisposition, NULL); + } +#endif +#else if ((filename!=NULL) && (dwDesiredAccess != 0)) hFile = CreateFile((LPCTSTR)filename, dwDesiredAccess, dwShareMode, NULL, dwCreationDisposition, dwFlagsAndAttributes, NULL); +#endif return win32_build_iowin(hFile); } @@ -188,6 +243,26 @@ uLong ZCALLBACK win32_write_file_func (voidpf opaque,voidpf stream,const void* b return ret; } +static BOOL MySetFilePointerEx(HANDLE hFile, LARGE_INTEGER pos, LARGE_INTEGER *newPos, DWORD dwMoveMethod) +{ +#ifdef IOWIN32_USING_WINRT_API + return SetFilePointerEx(hFile, pos, newPos, dwMoveMethod); +#else + LONG lHigh = pos.HighPart; + DWORD dwNewPos = SetFilePointer(hFile, pos.LowPart, &lHigh, dwMoveMethod); + BOOL fOk = TRUE; + if (dwNewPos == 0xFFFFFFFF) + if (GetLastError() != NO_ERROR) + fOk = FALSE; + if ((newPos != NULL) && (fOk)) + { + newPos->LowPart = dwNewPos; + newPos->HighPart = lHigh; + } + return fOk; +#endif +} + long ZCALLBACK win32_tell_file_func (voidpf opaque,voidpf stream) { long ret=-1; @@ -196,15 +271,17 @@ long ZCALLBACK win32_tell_file_func (voidpf opaque,voidpf stream) hFile = ((WIN32FILE_IOWIN*)stream) -> hf; if (hFile != NULL) { - DWORD dwSet = SetFilePointer(hFile, 0, NULL, FILE_CURRENT); - if (dwSet == INVALID_SET_FILE_POINTER) + LARGE_INTEGER pos; + pos.QuadPart = 0; + + if (!MySetFilePointerEx(hFile, pos, &pos, FILE_CURRENT)) { DWORD dwErr = GetLastError(); ((WIN32FILE_IOWIN*)stream) -> error=(int)dwErr; ret = -1; } else - ret=(long)dwSet; + ret=(long)pos.LowPart; } return ret; } @@ -218,17 +295,17 @@ ZPOS64_T ZCALLBACK win32_tell64_file_func (voidpf opaque, voidpf stream) if (hFile) { - LARGE_INTEGER li; - li.QuadPart = 0; - li.u.LowPart = SetFilePointer(hFile, li.u.LowPart, &li.u.HighPart, FILE_CURRENT); - if ( (li.LowPart == 0xFFFFFFFF) && (GetLastError() != NO_ERROR)) + LARGE_INTEGER pos; + pos.QuadPart = 0; + + if (!MySetFilePointerEx(hFile, pos, &pos, FILE_CURRENT)) { DWORD dwErr = GetLastError(); ((WIN32FILE_IOWIN*)stream) -> error=(int)dwErr; ret = (ZPOS64_T)-1; } else - ret=li.QuadPart; + ret=pos.QuadPart; } return ret; } @@ -258,8 +335,9 @@ long ZCALLBACK win32_seek_file_func (voidpf opaque,voidpf stream,uLong offset,in if (hFile != NULL) { - DWORD dwSet = SetFilePointer(hFile, offset, NULL, dwMoveMethod); - if (dwSet == INVALID_SET_FILE_POINTER) + LARGE_INTEGER pos; + pos.QuadPart = offset; + if (!MySetFilePointerEx(hFile, pos, NULL, dwMoveMethod)) { DWORD dwErr = GetLastError(); ((WIN32FILE_IOWIN*)stream) -> error=(int)dwErr; @@ -296,9 +374,9 @@ long ZCALLBACK win32_seek64_file_func (voidpf opaque, voidpf stream,ZPOS64_T off if (hFile) { - LARGE_INTEGER* li = (LARGE_INTEGER*)&offset; - DWORD dwSet = SetFilePointer(hFile, li->u.LowPart, &li->u.HighPart, dwMoveMethod); - if (dwSet == INVALID_SET_FILE_POINTER) + LARGE_INTEGER pos; + pos.QuadPart = offset; + if (!MySetFilePointerEx(hFile, pos, NULL, dwMoveMethod)) { DWORD dwErr = GetLastError(); ((WIN32FILE_IOWIN*)stream) -> error=(int)dwErr; diff --git a/third_party/zlib/contrib/minizip/miniunz.c b/third_party/zlib/contrib/minizip/miniunz.c index 9ed009fbd9..3d65401be5 100644 --- a/third_party/zlib/contrib/minizip/miniunz.c +++ b/third_party/zlib/contrib/minizip/miniunz.c @@ -12,7 +12,7 @@ Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) */ -#ifndef _WIN32 +#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) #ifndef __USE_FILE_OFFSET64 #define __USE_FILE_OFFSET64 #endif @@ -27,6 +27,18 @@ #endif #endif +#ifdef __APPLE__ +// In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions +#define FOPEN_FUNC(filename, mode) fopen(filename, mode) +#define FTELLO_FUNC(stream) ftello(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko(stream, offset, origin) +#else +#define FOPEN_FUNC(filename, mode) fopen64(filename, mode) +#define FTELLO_FUNC(stream) ftello64(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko64(stream, offset, origin) +#endif + + #include #include #include @@ -34,14 +46,15 @@ #include #include -#ifdef unix -# include -# include -#else +#ifdef _WIN32 # include # include +#else +# include +# include #endif + #include "unzip.h" #define CASESENSITIVITY (0) @@ -84,7 +97,7 @@ void change_file_date(filename,dosdate,tmu_date) SetFileTime(hFile,&ftm,&ftLastAcc,&ftm); CloseHandle(hFile); #else -#ifdef unix +#ifdef unix || __APPLE__ struct utimbuf ut; struct tm newdate; newdate.tm_sec = tmu_date.tm_sec; @@ -114,10 +127,10 @@ int mymkdir(dirname) int ret=0; #ifdef _WIN32 ret = _mkdir(dirname); -#else -#ifdef unix +#elif unix + ret = mkdir (dirname,0775); +#elif __APPLE__ ret = mkdir (dirname,0775); -#endif #endif return ret; } @@ -364,7 +377,7 @@ int do_extract_currentfile(uf,popt_extract_without_path,popt_overwrite,password) { char rep=0; FILE* ftestexist; - ftestexist = fopen64(write_filename,"rb"); + ftestexist = FOPEN_FUNC(write_filename,"rb"); if (ftestexist!=NULL) { fclose(ftestexist); @@ -395,8 +408,7 @@ int do_extract_currentfile(uf,popt_extract_without_path,popt_overwrite,password) if ((skip==0) && (err==UNZ_OK)) { - fout=fopen64(write_filename,"wb"); - + fout=FOPEN_FUNC(write_filename,"wb"); /* some zipfile don't contain directory alone before file */ if ((fout==NULL) && ((*popt_extract_without_path)==0) && (filename_withoutpath!=(char*)filename_inzip)) @@ -405,7 +417,7 @@ int do_extract_currentfile(uf,popt_extract_without_path,popt_overwrite,password) *(filename_withoutpath-1)='\0'; makedir(write_filename); *(filename_withoutpath-1)=c; - fout=fopen64(write_filename,"wb"); + fout=FOPEN_FUNC(write_filename,"wb"); } if (fout==NULL) diff --git a/third_party/zlib/contrib/minizip/minizip.c b/third_party/zlib/contrib/minizip/minizip.c index 7a4fa5a643..4288962ece 100644 --- a/third_party/zlib/contrib/minizip/minizip.c +++ b/third_party/zlib/contrib/minizip/minizip.c @@ -13,7 +13,7 @@ */ -#ifndef _WIN32 +#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) #ifndef __USE_FILE_OFFSET64 #define __USE_FILE_OFFSET64 #endif @@ -28,6 +28,19 @@ #endif #endif +#ifdef __APPLE__ +// In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions +#define FOPEN_FUNC(filename, mode) fopen(filename, mode) +#define FTELLO_FUNC(stream) ftello(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko(stream, offset, origin) +#else +#define FOPEN_FUNC(filename, mode) fopen64(filename, mode) +#define FTELLO_FUNC(stream) ftello64(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko64(stream, offset, origin) +#endif + + + #include #include #include @@ -35,14 +48,14 @@ #include #include -#ifdef unix +#ifdef _WIN32 +# include +# include +#else # include # include # include # include -#else -# include -# include #endif #include "zip.h" @@ -81,7 +94,7 @@ uLong filetime(f, tmzip, dt) return ret; } #else -#ifdef unix +#ifdef unix || __APPLE__ uLong filetime(f, tmzip, dt) char *f; /* name of file to get info on */ tm_zip *tmzip; /* return value: access, modific. and creation times */ @@ -142,7 +155,7 @@ int check_exist_file(filename) { FILE* ftestexist; int ret = 1; - ftestexist = fopen64(filename,"rb"); + ftestexist = FOPEN_FUNC(filename,"rb"); if (ftestexist==NULL) ret = 0; else @@ -173,7 +186,8 @@ int getFileCrc(const char* filenameinzip,void*buf,unsigned long size_buf,unsigne { unsigned long calculate_crc=0; int err=ZIP_OK; - FILE * fin = fopen64(filenameinzip,"rb"); + FILE * fin = FOPEN_FUNC(filenameinzip,"rb"); + unsigned long size_read = 0; unsigned long total_read = 0; if (fin==NULL) @@ -211,13 +225,12 @@ int isLargeFile(const char* filename) { int largeFile = 0; ZPOS64_T pos = 0; - FILE* pFile = fopen64(filename, "rb"); + FILE* pFile = FOPEN_FUNC(filename, "rb"); if(pFile != NULL) { - int n = fseeko64(pFile, 0, SEEK_END); - - pos = ftello64(pFile); + int n = FSEEKO_FUNC(pFile, 0, SEEK_END); + pos = FTELLO_FUNC(pFile); printf("File : %s is %lld bytes\n", filename, pos); @@ -447,7 +460,7 @@ int main(argc,argv) printf("error in opening %s in zipfile\n",filenameinzip); else { - fin = fopen64(filenameinzip,"rb"); + fin = FOPEN_FUNC(filenameinzip,"rb"); if (fin==NULL) { err=ZIP_ERRNO; diff --git a/third_party/zlib/contrib/minizip/mztools.c b/third_party/zlib/contrib/minizip/mztools.c index 026bf2820e..8bf9cca326 100644 --- a/third_party/zlib/contrib/minizip/mztools.c +++ b/third_party/zlib/contrib/minizip/mztools.c @@ -42,7 +42,7 @@ uLong* bytesRecovered; int entries = 0; uLong totalBytes = 0; char header[30]; - char filename[256]; + char filename[1024]; char extra[1024]; int offset = 0; int offsetCD = 0; @@ -73,9 +73,14 @@ uLong* bytesRecovered; /* Filename */ if (fnsize > 0) { - if (fread(filename, 1, fnsize, fpZip) == fnsize) { - if (fwrite(filename, 1, fnsize, fpOut) == fnsize) { - offset += fnsize; + if (fnsize < sizeof(filename)) { + if (fread(filename, 1, fnsize, fpZip) == fnsize) { + if (fwrite(filename, 1, fnsize, fpOut) == fnsize) { + offset += fnsize; + } else { + err = Z_ERRNO; + break; + } } else { err = Z_ERRNO; break; @@ -91,9 +96,14 @@ uLong* bytesRecovered; /* Extra field */ if (extsize > 0) { - if (fread(extra, 1, extsize, fpZip) == extsize) { - if (fwrite(extra, 1, extsize, fpOut) == extsize) { - offset += extsize; + if (extsize < sizeof(extra)) { + if (fread(extra, 1, extsize, fpZip) == extsize) { + if (fwrite(extra, 1, extsize, fpOut) == extsize) { + offset += extsize; + } else { + err = Z_ERRNO; + break; + } } else { err = Z_ERRNO; break; diff --git a/third_party/zlib/contrib/minizip/mztools.h b/third_party/zlib/contrib/minizip/mztools.h index 90c834d3ab..f295ffeda6 100644 --- a/third_party/zlib/contrib/minizip/mztools.h +++ b/third_party/zlib/contrib/minizip/mztools.h @@ -28,4 +28,10 @@ extern int ZEXPORT unzRepair(const char* file, uLong* nRecovered, uLong* bytesRecovered); + +#ifdef __cplusplus +} +#endif + + #endif diff --git a/third_party/zlib/contrib/minizip/unzip.c b/third_party/zlib/contrib/minizip/unzip.c index af59b80294..e8b2bc5c76 100644 --- a/third_party/zlib/contrib/minizip/unzip.c +++ b/third_party/zlib/contrib/minizip/unzip.c @@ -68,10 +68,6 @@ #include #include -#ifndef NOUNCRYPT - #define NOUNCRYPT -#endif - #include "third_party/zlib/zlib.h" #include "unzip.h" @@ -188,7 +184,7 @@ typedef struct # ifndef NOUNCRYPT unsigned long keys[3]; /* keys defining the pseudo-random sequence */ - const unsigned long* pcrc_32_tab; + const z_crc_t* pcrc_32_tab; # endif } unz64_s; @@ -200,7 +196,7 @@ typedef struct /* =========================================================================== Read a byte from a gz_stream; update next_in and avail_in. Return EOF for end of file. - IN assertion: the stream s has been sucessfully opened for reading. + IN assertion: the stream s has been successfully opened for reading. */ @@ -801,9 +797,9 @@ extern unzFile ZEXPORT unzOpen64 (const void *path) } /* - Close a ZipFile opened with unzipOpen. - If there is files inside the .Zip opened with unzipOpenCurrentFile (see later), - these files MUST be closed with unzipCloseCurrentFile before call unzipClose. + Close a ZipFile opened with unzOpen. + If there is files inside the .Zip opened with unzOpenCurrentFile (see later), + these files MUST be closed with unzCloseCurrentFile before call unzClose. return UNZ_OK if there is no problem. */ extern int ZEXPORT unzClose (unzFile file) { @@ -1040,26 +1036,26 @@ local int unz64local_GetCurrentFileInfoInternal (unzFile file, { uLong uL; - if(file_info.uncompressed_size == (ZPOS64_T)(unsigned long)-1) + if(file_info.uncompressed_size == MAXU32) { if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info.uncompressed_size) != UNZ_OK) err=UNZ_ERRNO; } - if(file_info.compressed_size == (ZPOS64_T)(unsigned long)-1) + if(file_info.compressed_size == MAXU32) { if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info.compressed_size) != UNZ_OK) err=UNZ_ERRNO; } - if(file_info_internal.offset_curfile == (ZPOS64_T)(unsigned long)-1) + if(file_info_internal.offset_curfile == MAXU32) { /* Relative Header offset */ if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info_internal.offset_curfile) != UNZ_OK) err=UNZ_ERRNO; } - if(file_info.disk_num_start == (unsigned long)-1) + if(file_info.disk_num_start == MAXU32) { /* Disk Start Number */ if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) @@ -1145,7 +1141,7 @@ extern int ZEXPORT unzGetCurrentFileInfo (unzFile file, szFileName,fileNameBufferSize, extraField,extraFieldBufferSize, szComment,commentBufferSize); - if (err==UNZ_OK) + if ((err==UNZ_OK) && (pfile_info != NULL)) { pfile_info->version = file_info64.version; pfile_info->version_needed = file_info64.version_needed; @@ -1223,7 +1219,7 @@ extern int ZEXPORT unzGoToNextFile (unzFile file) /* Try locate the file szFileName in the zipfile. - For the iCaseSensitivity signification, see unzipStringFileNameCompare + For the iCaseSensitivity signification, see unzStringFileNameCompare return value : UNZ_OK if the file is found. It becomes the current file. @@ -1630,6 +1626,7 @@ extern int ZEXPORT unzOpenCurrentFile3 (unzFile file, int* method, zdecode(s->keys,s->pcrc_32_tab,source[i]); s->pfile_in_zip_read->pos_in_zipfile+=12; + s->pfile_in_zip_read->rest_read_compressed-=12; s->encrypted=1; } # endif @@ -1696,7 +1693,7 @@ extern int ZEXPORT unzReadCurrentFile (unzFile file, voidp buf, unsigned len) return UNZ_PARAMERROR; - if ((pfile_in_zip_read_info->read_buffer == NULL)) + if (pfile_in_zip_read_info->read_buffer == NULL) return UNZ_END_OF_LIST_OF_FILE; if (len==0) return 0; @@ -1993,7 +1990,7 @@ extern int ZEXPORT unzGetLocalExtrafield (unzFile file, voidp buf, unsigned len) } /* - Close the file in zip opened with unzipOpenCurrentFile + Close the file in zip opened with unzOpenCurrentFile Return UNZ_CRCERROR if all the file was read but the CRC is not good */ extern int ZEXPORT unzCloseCurrentFile (unzFile file) diff --git a/third_party/zlib/contrib/minizip/unzip.h b/third_party/zlib/contrib/minizip/unzip.h index 80eb36e55a..3c0143529b 100644 --- a/third_party/zlib/contrib/minizip/unzip.h +++ b/third_party/zlib/contrib/minizip/unzip.h @@ -197,9 +197,9 @@ extern unzFile ZEXPORT unzOpen2_64 OF((const void *path, extern int ZEXPORT unzClose OF((unzFile file)); /* - Close a ZipFile opened with unzipOpen. + Close a ZipFile opened with unzOpen. If there is files inside the .Zip opened with unzOpenCurrentFile (see later), - these files MUST be closed with unzipCloseCurrentFile before call unzipClose. + these files MUST be closed with unzCloseCurrentFile before call unzClose. return UNZ_OK if there is no problem. */ extern int ZEXPORT unzGetGlobalInfo OF((unzFile file, diff --git a/third_party/zlib/contrib/minizip/zip.c b/third_party/zlib/contrib/minizip/zip.c index 5a828851f4..65c0c72518 100644 --- a/third_party/zlib/contrib/minizip/zip.c +++ b/third_party/zlib/contrib/minizip/zip.c @@ -15,7 +15,7 @@ Oct-2009 - Mathias Svensson - Did some code cleanup and refactoring to get better overview of some functions. Oct-2009 - Mathias Svensson - Added zipRemoveExtraInfoBlock to strip extra field data from its ZIP64 data It is used when recreting zip archive with RAW when deleting items from a zip. - ZIP64 data is automaticly added to items that needs it, and existing ZIP64 data need to be removed. + ZIP64 data is automatically added to items that needs it, and existing ZIP64 data need to be removed. Oct-2009 - Mathias Svensson - Added support for BZIP2 as compression mode (bzip2 lib is required) Jan-2010 - back to unzip and minizip 1.0 name scheme, with compatibility layer @@ -116,7 +116,7 @@ typedef struct linkedlist_datablock_internal_s struct linkedlist_datablock_internal_s* next_datablock; uLong avail_in_this_block; uLong filled_in_this_block; - uLong unused; /* for future use and alignement */ + uLong unused; /* for future use and alignment */ unsigned char data[SIZEDATA_INDATABLOCK]; } linkedlist_datablock_internal; @@ -157,7 +157,7 @@ typedef struct ZPOS64_T totalUncompressedData; #ifndef NOCRYPT unsigned long keys[3]; /* keys defining the pseudo-random sequence */ - const unsigned long* pcrc_32_tab; + const z_crc_t* pcrc_32_tab; int crypt_header_size; #endif } curfile64_info; @@ -171,7 +171,7 @@ typedef struct curfile64_info ci; /* info on the file curretly writing */ ZPOS64_T begin_pos; /* position of the beginning of the zipfile */ - ZPOS64_T add_position_when_writting_offset; + ZPOS64_T add_position_when_writing_offset; ZPOS64_T number_entry; #ifndef NO_ADDFILEINEXISTINGZIP @@ -807,7 +807,7 @@ int LoadCentralDirectoryRecord(zip64_internal* pziinit) } byte_before_the_zipfile = central_pos - (offset_central_dir+size_central_dir); - pziinit->add_position_when_writting_offset = byte_before_the_zipfile; + pziinit->add_position_when_writing_offset = byte_before_the_zipfile; { ZPOS64_T size_central_dir_to_read = size_central_dir; @@ -875,7 +875,7 @@ extern zipFile ZEXPORT zipOpen3 (const void *pathname, int append, zipcharpc* gl ziinit.in_opened_file_inzip = 0; ziinit.ci.stream_initialised = 0; ziinit.number_entry = 0; - ziinit.add_position_when_writting_offset = 0; + ziinit.add_position_when_writing_offset = 0; init_linkedlist(&(ziinit.central_dir)); @@ -1067,6 +1067,7 @@ extern int ZEXPORT zipOpenNewFileInZip4_64 (zipFile file, const char* filename, int err = ZIP_OK; # ifdef NOCRYPT + (crcForCrypting); if (password != NULL) return ZIP_PARAMERROR; # endif @@ -1114,9 +1115,9 @@ extern int ZEXPORT zipOpenNewFileInZip4_64 (zipFile file, const char* filename, zi->ci.flag = flagBase; if ((level==8) || (level==9)) zi->ci.flag |= 2; - if ((level==2)) + if (level==2) zi->ci.flag |= 4; - if ((level==1)) + if (level==1) zi->ci.flag |= 6; if (password != NULL) zi->ci.flag |= 1; @@ -1163,7 +1164,7 @@ extern int ZEXPORT zipOpenNewFileInZip4_64 (zipFile file, const char* filename, if(zi->ci.pos_local_header >= 0xffffffff) zip64local_putValue_inmemory(zi->ci.central_header+42,(uLong)0xffffffff,4); else - zip64local_putValue_inmemory(zi->ci.central_header+42,(uLong)zi->ci.pos_local_header - zi->add_position_when_writting_offset,4); + zip64local_putValue_inmemory(zi->ci.central_header+42,(uLong)zi->ci.pos_local_header - zi->add_position_when_writing_offset,4); for (i=0;ici.central_header+SIZECENTRALHEADER+i) = *(filename+i); @@ -1710,7 +1711,7 @@ extern int ZEXPORT zipCloseFileInZipRaw64 (zipFile file, ZPOS64_T uncompressed_s if (err==ZIP_OK) err = zip64local_putValue(&zi->z_filefunc,zi->filestream,crc32,4); /* crc 32, unknown */ - if(uncompressed_size >= 0xffffffff) + if(uncompressed_size >= 0xffffffff || compressed_size >= 0xffffffff ) { if(zi->ci.pos_zip64extrainfo > 0) { @@ -1724,6 +1725,8 @@ extern int ZEXPORT zipCloseFileInZipRaw64 (zipFile file, ZPOS64_T uncompressed_s if (err==ZIP_OK) /* uncompressed size, unknown */ err = zip64local_putValue(&zi->z_filefunc, zi->filestream, compressed_size, 8); } + else + err = ZIP_BADZIPFILE; // Caller passed zip64 = 0, so no room for zip64 info -> fatal } else { @@ -1752,7 +1755,7 @@ extern int ZEXPORT zipCloseFileInZip (zipFile file) int Write_Zip64EndOfCentralDirectoryLocator(zip64_internal* zi, ZPOS64_T zip64eocd_pos_inzip) { int err = ZIP_OK; - ZPOS64_T pos = zip64eocd_pos_inzip - zi->add_position_when_writting_offset; + ZPOS64_T pos = zip64eocd_pos_inzip - zi->add_position_when_writing_offset; err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)ZIP64ENDLOCHEADERMAGIC,4); @@ -1805,7 +1808,7 @@ int Write_Zip64EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centra if (err==ZIP_OK) /* offset of start of central directory with respect to the starting disk number */ { - ZPOS64_T pos = centraldir_pos_inzip - zi->add_position_when_writting_offset; + ZPOS64_T pos = centraldir_pos_inzip - zi->add_position_when_writing_offset; err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (ZPOS64_T)pos,8); } return err; @@ -1846,13 +1849,13 @@ int Write_EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, if (err==ZIP_OK) /* offset of start of central directory with respect to the starting disk number */ { - ZPOS64_T pos = centraldir_pos_inzip - zi->add_position_when_writting_offset; + ZPOS64_T pos = centraldir_pos_inzip - zi->add_position_when_writing_offset; if(pos >= 0xffffffff) { err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (uLong)0xffffffff,4); } else - err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (uLong)(centraldir_pos_inzip - zi->add_position_when_writting_offset),4); + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (uLong)(centraldir_pos_inzip - zi->add_position_when_writing_offset),4); } return err; @@ -1918,8 +1921,8 @@ extern int ZEXPORT zipClose (zipFile file, const char* global_comment) } free_linkedlist(&(zi->central_dir)); - pos = centraldir_pos_inzip - zi->add_position_when_writting_offset; - if(pos >= 0xffffffff) + pos = centraldir_pos_inzip - zi->add_position_when_writing_offset; + if(pos >= 0xffffffff || zi->number_entry > 0xFFFF) { ZPOS64_T Zip64EOCDpos = ZTELL64(zi->z_filefunc,zi->filestream); Write_Zip64EndOfCentralDirectoryRecord(zi, size_centraldir, centraldir_pos_inzip); diff --git a/third_party/zlib/contrib/optimizations/chunkcopy.h b/third_party/zlib/contrib/optimizations/chunkcopy.h new file mode 100644 index 0000000000..38ba0ed8a2 --- /dev/null +++ b/third_party/zlib/contrib/optimizations/chunkcopy.h @@ -0,0 +1,444 @@ +/* chunkcopy.h -- fast chunk copy and set operations + * Copyright (C) 2017 ARM, Inc. + * Copyright 2017 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#ifndef CHUNKCOPY_H +#define CHUNKCOPY_H + +#include +#include "zutil.h" + +#define Z_STATIC_ASSERT(name, assert) typedef char name[(assert) ? 1 : -1] + +#if __STDC_VERSION__ >= 199901L +#define Z_RESTRICT restrict +#else +#define Z_RESTRICT +#endif + +#if defined(__clang__) || defined(__GNUC__) || defined(__llvm__) +#define Z_BUILTIN_MEMCPY __builtin_memcpy +#else +#define Z_BUILTIN_MEMCPY zmemcpy +#endif + +#if defined(INFLATE_CHUNK_SIMD_NEON) +#include +typedef uint8x16_t z_vec128i_t; +#elif defined(INFLATE_CHUNK_SIMD_SSE2) +#include +typedef __m128i z_vec128i_t; +#else +#error chunkcopy.h inflate chunk SIMD is not defined for your build target +#endif + +/* + * chunk copy type: the z_vec128i_t type size should be exactly 128-bits + * and equal to CHUNKCOPY_CHUNK_SIZE. + */ +#define CHUNKCOPY_CHUNK_SIZE sizeof(z_vec128i_t) + +Z_STATIC_ASSERT(vector_128_bits_wide, + CHUNKCOPY_CHUNK_SIZE == sizeof(int8_t) * 16); + +/* + * Ask the compiler to perform a wide, unaligned load with a machine + * instruction appropriate for the z_vec128i_t type. + */ +static inline z_vec128i_t loadchunk( + const unsigned char FAR* s) { + z_vec128i_t v; + Z_BUILTIN_MEMCPY(&v, s, sizeof(v)); + return v; +} + +/* + * Ask the compiler to perform a wide, unaligned store with a machine + * instruction appropriate for the z_vec128i_t type. + */ +static inline void storechunk( + unsigned char FAR* d, + const z_vec128i_t v) { + Z_BUILTIN_MEMCPY(d, &v, sizeof(v)); +} + +/* + * Perform a memcpy-like operation, assuming that length is non-zero and that + * it's OK to overwrite at least CHUNKCOPY_CHUNK_SIZE bytes of output even if + * the length is shorter than this. + * + * It also guarantees that it will properly unroll the data if the distance + * between `out` and `from` is at least CHUNKCOPY_CHUNK_SIZE, which we rely on + * in chunkcopy_relaxed(). + * + * Aside from better memory bus utilisation, this means that short copies + * (CHUNKCOPY_CHUNK_SIZE bytes or fewer) will fall straight through the loop + * without iteration, which will hopefully make the branch prediction more + * reliable. + */ +static inline unsigned char FAR* chunkcopy_core( + unsigned char FAR* out, + const unsigned char FAR* from, + unsigned len) { + const int bump = (--len % CHUNKCOPY_CHUNK_SIZE) + 1; + storechunk(out, loadchunk(from)); + out += bump; + from += bump; + len /= CHUNKCOPY_CHUNK_SIZE; + while (len-- > 0) { + storechunk(out, loadchunk(from)); + out += CHUNKCOPY_CHUNK_SIZE; + from += CHUNKCOPY_CHUNK_SIZE; + } + return out; +} + +/* + * Like chunkcopy_core(), but avoid writing beyond of legal output. + * + * Accepts an additional pointer to the end of safe output. A generic safe + * copy would use (out + len), but it's normally the case that the end of the + * output buffer is beyond the end of the current copy, and this can still be + * exploited. + */ +static inline unsigned char FAR* chunkcopy_core_safe( + unsigned char FAR* out, + const unsigned char FAR* from, + unsigned len, + unsigned char FAR* limit) { + Assert(out + len <= limit, "chunk copy exceeds safety limit"); + if ((limit - out) < (ptrdiff_t)CHUNKCOPY_CHUNK_SIZE) { + const unsigned char FAR* Z_RESTRICT rfrom = from; + if (len & 8) { + Z_BUILTIN_MEMCPY(out, rfrom, 8); + out += 8; + rfrom += 8; + } + if (len & 4) { + Z_BUILTIN_MEMCPY(out, rfrom, 4); + out += 4; + rfrom += 4; + } + if (len & 2) { + Z_BUILTIN_MEMCPY(out, rfrom, 2); + out += 2; + rfrom += 2; + } + if (len & 1) { + *out++ = *rfrom++; + } + return out; + } + return chunkcopy_core(out, from, len); +} + +/* + * Perform short copies until distance can be rewritten as being at least + * CHUNKCOPY_CHUNK_SIZE. + * + * Assumes it's OK to overwrite at least the first 2*CHUNKCOPY_CHUNK_SIZE + * bytes of output even if the copy is shorter than this. This assumption + * holds within zlib inflate_fast(), which starts every iteration with at + * least 258 bytes of output space available (258 being the maximum length + * output from a single token; see inffast.c). + */ +static inline unsigned char FAR* chunkunroll_relaxed( + unsigned char FAR* out, + unsigned FAR* dist, + unsigned FAR* len) { + const unsigned char FAR* from = out - *dist; + while (*dist < *len && *dist < CHUNKCOPY_CHUNK_SIZE) { + storechunk(out, loadchunk(from)); + out += *dist; + *len -= *dist; + *dist += *dist; + } + return out; +} + +#if defined(INFLATE_CHUNK_SIMD_NEON) +/* + * v_load64_dup(): load *src as an unaligned 64-bit int and duplicate it in + * every 64-bit component of the 128-bit result (64-bit int splat). + */ +static inline z_vec128i_t v_load64_dup(const void* src) { + return vcombine_u8(vld1_u8(src), vld1_u8(src)); +} + +/* + * v_load32_dup(): load *src as an unaligned 32-bit int and duplicate it in + * every 32-bit component of the 128-bit result (32-bit int splat). + */ +static inline z_vec128i_t v_load32_dup(const void* src) { + int32_t i32; + Z_BUILTIN_MEMCPY(&i32, src, sizeof(i32)); + return vreinterpretq_u8_s32(vdupq_n_s32(i32)); +} + +/* + * v_load16_dup(): load *src as an unaligned 16-bit int and duplicate it in + * every 16-bit component of the 128-bit result (16-bit int splat). + */ +static inline z_vec128i_t v_load16_dup(const void* src) { + int16_t i16; + Z_BUILTIN_MEMCPY(&i16, src, sizeof(i16)); + return vreinterpretq_u8_s16(vdupq_n_s16(i16)); +} + +/* + * v_load8_dup(): load the 8-bit int *src and duplicate it in every 8-bit + * component of the 128-bit result (8-bit int splat). + */ +static inline z_vec128i_t v_load8_dup(const void* src) { + return vld1q_dup_u8((const uint8_t*)src); +} + +/* + * v_store_128(): store the 128-bit vec in a memory destination (that might + * not be 16-byte aligned) void* out. + */ +static inline void v_store_128(void* out, const z_vec128i_t vec) { + vst1q_u8(out, vec); +} + +#elif defined(INFLATE_CHUNK_SIMD_SSE2) +/* + * v_load64_dup(): load *src as an unaligned 64-bit int and duplicate it in + * every 64-bit component of the 128-bit result (64-bit int splat). + */ +static inline z_vec128i_t v_load64_dup(const void* src) { + int64_t i64; + Z_BUILTIN_MEMCPY(&i64, src, sizeof(i64)); + return _mm_set1_epi64x(i64); +} + +/* + * v_load32_dup(): load *src as an unaligned 32-bit int and duplicate it in + * every 32-bit component of the 128-bit result (32-bit int splat). + */ +static inline z_vec128i_t v_load32_dup(const void* src) { + int32_t i32; + Z_BUILTIN_MEMCPY(&i32, src, sizeof(i32)); + return _mm_set1_epi32(i32); +} + +/* + * v_load16_dup(): load *src as an unaligned 16-bit int and duplicate it in + * every 16-bit component of the 128-bit result (16-bit int splat). + */ +static inline z_vec128i_t v_load16_dup(const void* src) { + int16_t i16; + Z_BUILTIN_MEMCPY(&i16, src, sizeof(i16)); + return _mm_set1_epi16(i16); +} + +/* + * v_load8_dup(): load the 8-bit int *src and duplicate it in every 8-bit + * component of the 128-bit result (8-bit int splat). + */ +static inline z_vec128i_t v_load8_dup(const void* src) { + return _mm_set1_epi8(*(const char*)src); +} + +/* + * v_store_128(): store the 128-bit vec in a memory destination (that might + * not be 16-byte aligned) void* out. + */ +static inline void v_store_128(void* out, const z_vec128i_t vec) { + _mm_storeu_si128((__m128i*)out, vec); +} +#endif + +/* + * Perform an overlapping copy which behaves as a memset() operation, but + * supporting periods other than one, and assume that length is non-zero and + * that it's OK to overwrite at least CHUNKCOPY_CHUNK_SIZE*3 bytes of output + * even if the length is shorter than this. + */ +static inline unsigned char FAR* chunkset_core( + unsigned char FAR* out, + unsigned period, + unsigned len) { + z_vec128i_t v; + const int bump = ((len - 1) % sizeof(v)) + 1; + + switch (period) { + case 1: + v = v_load8_dup(out - 1); + v_store_128(out, v); + out += bump; + len -= bump; + while (len > 0) { + v_store_128(out, v); + out += sizeof(v); + len -= sizeof(v); + } + return out; + case 2: + v = v_load16_dup(out - 2); + v_store_128(out, v); + out += bump; + len -= bump; + if (len > 0) { + v = v_load16_dup(out - 2); + do { + v_store_128(out, v); + out += sizeof(v); + len -= sizeof(v); + } while (len > 0); + } + return out; + case 4: + v = v_load32_dup(out - 4); + v_store_128(out, v); + out += bump; + len -= bump; + if (len > 0) { + v = v_load32_dup(out - 4); + do { + v_store_128(out, v); + out += sizeof(v); + len -= sizeof(v); + } while (len > 0); + } + return out; + case 8: + v = v_load64_dup(out - 8); + v_store_128(out, v); + out += bump; + len -= bump; + if (len > 0) { + v = v_load64_dup(out - 8); + do { + v_store_128(out, v); + out += sizeof(v); + len -= sizeof(v); + } while (len > 0); + } + return out; + } + out = chunkunroll_relaxed(out, &period, &len); + return chunkcopy_core(out, out - period, len); +} + +/* + * Perform a memcpy-like operation, but assume that length is non-zero and that + * it's OK to overwrite at least CHUNKCOPY_CHUNK_SIZE bytes of output even if + * the length is shorter than this. + * + * Unlike chunkcopy_core() above, no guarantee is made regarding the behaviour + * of overlapping buffers, regardless of the distance between the pointers. + * This is reflected in the `restrict`-qualified pointers, allowing the + * compiler to re-order loads and stores. + */ +static inline unsigned char FAR* chunkcopy_relaxed( + unsigned char FAR* Z_RESTRICT out, + const unsigned char FAR* Z_RESTRICT from, + unsigned len) { + return chunkcopy_core(out, from, len); +} + +/* + * Like chunkcopy_relaxed(), but avoid writing beyond of legal output. + * + * Unlike chunkcopy_core_safe() above, no guarantee is made regarding the + * behaviour of overlapping buffers, regardless of the distance between the + * pointers. This is reflected in the `restrict`-qualified pointers, allowing + * the compiler to re-order loads and stores. + * + * Accepts an additional pointer to the end of safe output. A generic safe + * copy would use (out + len), but it's normally the case that the end of the + * output buffer is beyond the end of the current copy, and this can still be + * exploited. + */ +static inline unsigned char FAR* chunkcopy_safe( + unsigned char FAR* out, + const unsigned char FAR* Z_RESTRICT from, + unsigned len, + unsigned char FAR* limit) { + Assert(out + len <= limit, "chunk copy exceeds safety limit"); + return chunkcopy_core_safe(out, from, len, limit); +} + +/* + * Perform chunky copy within the same buffer, where the source and destination + * may potentially overlap. + * + * Assumes that len > 0 on entry, and that it's safe to write at least + * CHUNKCOPY_CHUNK_SIZE*3 bytes to the output. + */ +static inline unsigned char FAR* chunkcopy_lapped_relaxed( + unsigned char FAR* out, + unsigned dist, + unsigned len) { + if (dist < len && dist < CHUNKCOPY_CHUNK_SIZE) { + return chunkset_core(out, dist, len); + } + return chunkcopy_core(out, out - dist, len); +} + +/* + * Behave like chunkcopy_lapped_relaxed(), but avoid writing beyond of legal + * output. + * + * Accepts an additional pointer to the end of safe output. A generic safe + * copy would use (out + len), but it's normally the case that the end of the + * output buffer is beyond the end of the current copy, and this can still be + * exploited. + */ +static inline unsigned char FAR* chunkcopy_lapped_safe( + unsigned char FAR* out, + unsigned dist, + unsigned len, + unsigned char FAR* limit) { + Assert(out + len <= limit, "chunk copy exceeds safety limit"); + if ((limit - out) < (ptrdiff_t)(3 * CHUNKCOPY_CHUNK_SIZE)) { + /* TODO(cavalcantii): try harder to optimise this */ + while (len-- > 0) { + *out = *(out - dist); + out++; + } + return out; + } + return chunkcopy_lapped_relaxed(out, dist, len); +} + +/* + * The chunk-copy code above deals with writing the decoded DEFLATE data to + * the output with SIMD methods to increase decode speed. Reading the input + * to the DEFLATE decoder with a wide, SIMD method can also increase decode + * speed. This option is supported on little endian machines, and reads the + * input data in 64-bit (8 byte) chunks. + */ + +#ifdef INFLATE_CHUNK_READ_64LE +/* + * Buffer the input in a uint64_t (8 bytes) in the wide input reading case. + */ +typedef uint64_t inflate_holder_t; + +/* + * Ask the compiler to perform a wide, unaligned load of a uint64_t using a + * machine instruction appropriate for the uint64_t type. + */ +static inline inflate_holder_t read64le(const unsigned char FAR *in) { + inflate_holder_t input; + Z_BUILTIN_MEMCPY(&input, in, sizeof(input)); + return input; +} +#else +/* + * Otherwise, buffer the input bits using zlib's default input buffer type. + */ +typedef unsigned long inflate_holder_t; + +#endif /* INFLATE_CHUNK_READ_64LE */ + +#undef Z_STATIC_ASSERT +#undef Z_RESTRICT +#undef Z_BUILTIN_MEMCPY + +#endif /* CHUNKCOPY_H */ diff --git a/third_party/zlib/contrib/optimizations/inffast_chunk.c b/third_party/zlib/contrib/optimizations/inffast_chunk.c new file mode 100644 index 0000000000..4099edf396 --- /dev/null +++ b/third_party/zlib/contrib/optimizations/inffast_chunk.c @@ -0,0 +1,359 @@ +/* inffast_chunk.c -- fast decoding + * Copyright (C) 1995-2017 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "zutil.h" +#include "inftrees.h" +#include "inflate.h" +#include "contrib/optimizations/inffast_chunk.h" +#include "contrib/optimizations/chunkcopy.h" + +#ifdef ASMINF +# pragma message("Assembler code may have bugs -- use at your own risk") +#else + +/* + Decode literal, length, and distance codes and write out the resulting + literal and match bytes until either not enough input or output is + available, an end-of-block is encountered, or a data error is encountered. + When large enough input and output buffers are supplied to inflate(), for + example, a 16K input buffer and a 64K output buffer, more than 95% of the + inflate() execution time is spent in this routine. + + Entry assumptions: + + state->mode == LEN + strm->avail_in >= INFLATE_FAST_MIN_INPUT (6 or 8 bytes) + strm->avail_out >= INFLATE_FAST_MIN_OUTPUT (258 bytes) + start >= strm->avail_out + state->bits < 8 + (state->hold >> state->bits) == 0 + strm->next_out[0..strm->avail_out] does not overlap with + strm->next_in[0..strm->avail_in] + strm->state->window is allocated with an additional + CHUNKCOPY_CHUNK_SIZE-1 bytes of padding beyond strm->state->wsize + + On return, state->mode is one of: + + LEN -- ran out of enough output space or enough available input + TYPE -- reached end of block code, inflate() to interpret next block + BAD -- error in block data + + Notes: + + INFLATE_FAST_MIN_INPUT: 6 or 8 bytes + + - The maximum input bits used by a length/distance pair is 15 bits for the + length code, 5 bits for the length extra, 15 bits for the distance code, + and 13 bits for the distance extra. This totals 48 bits, or six bytes. + Therefore if strm->avail_in >= 6, then there is enough input to avoid + checking for available input while decoding. + + - The wide input data reading option reads 64 input bits at a time. Thus, + if strm->avail_in >= 8, then there is enough input to avoid checking for + available input while decoding. Reading consumes the input with: + + hold |= read64le(in) << bits; + in += 6; + bits += 48; + + reporting 6 bytes of new input because |bits| is 0..15 (2 bytes rounded + up, worst case) and 6 bytes is enough to decode as noted above. At exit, + hold &= (1U << bits) - 1 drops excess input to keep the invariant: + + (state->hold >> state->bits) == 0 + + INFLATE_FAST_MIN_OUTPUT: 258 bytes + + - The maximum bytes that a single length/distance pair can output is 258 + bytes, which is the maximum length that can be coded. inflate_fast() + requires strm->avail_out >= 258 for each loop to avoid checking for + available output space while decoding. + */ +void ZLIB_INTERNAL inflate_fast_chunk_(strm, start) +z_streamp strm; +unsigned start; /* inflate()'s starting value for strm->avail_out */ +{ + struct inflate_state FAR *state; + z_const unsigned char FAR *in; /* local strm->next_in */ + z_const unsigned char FAR *last; /* have enough input while in < last */ + unsigned char FAR *out; /* local strm->next_out */ + unsigned char FAR *beg; /* inflate()'s initial strm->next_out */ + unsigned char FAR *end; /* while out < end, enough space available */ + unsigned char FAR *limit; /* safety limit for chunky copies */ +#ifdef INFLATE_STRICT + unsigned dmax; /* maximum distance from zlib header */ +#endif + unsigned wsize; /* window size or zero if not using window */ + unsigned whave; /* valid bytes in the window */ + unsigned wnext; /* window write index */ + unsigned char FAR *window; /* allocated sliding window, if wsize != 0 */ + inflate_holder_t hold; /* local strm->hold */ + unsigned bits; /* local strm->bits */ + code const FAR *lcode; /* local strm->lencode */ + code const FAR *dcode; /* local strm->distcode */ + unsigned lmask; /* mask for first level of length codes */ + unsigned dmask; /* mask for first level of distance codes */ + code here; /* retrieved table entry */ + unsigned op; /* code bits, operation, extra bits, or */ + /* window position, window bytes to copy */ + unsigned len; /* match length, unused bytes */ + unsigned dist; /* match distance */ + unsigned char FAR *from; /* where to copy match from */ + + /* copy state to local variables */ + state = (struct inflate_state FAR *)strm->state; + in = strm->next_in; + last = in + (strm->avail_in - (INFLATE_FAST_MIN_INPUT - 1)); + out = strm->next_out; + beg = out - (start - strm->avail_out); + end = out + (strm->avail_out - (INFLATE_FAST_MIN_OUTPUT - 1)); + limit = out + strm->avail_out; +#ifdef INFLATE_STRICT + dmax = state->dmax; +#endif + wsize = state->wsize; + whave = state->whave; + wnext = (state->wnext == 0 && whave >= wsize) ? wsize : state->wnext; + window = state->window; + hold = state->hold; + bits = state->bits; + lcode = state->lencode; + dcode = state->distcode; + lmask = (1U << state->lenbits) - 1; + dmask = (1U << state->distbits) - 1; + + /* decode literals and length/distances until end-of-block or not enough + input data or output space */ + do { + if (bits < 15) { +#ifdef INFLATE_CHUNK_READ_64LE + hold |= read64le(in) << bits; + in += 6; + bits += 48; +#else + hold += (unsigned long)(*in++) << bits; + bits += 8; + hold += (unsigned long)(*in++) << bits; + bits += 8; +#endif + } + here = lcode[hold & lmask]; + dolen: + op = (unsigned)(here.bits); + hold >>= op; + bits -= op; + op = (unsigned)(here.op); + if (op == 0) { /* literal */ + Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? + "inflate: literal '%c'\n" : + "inflate: literal 0x%02x\n", here.val)); + *out++ = (unsigned char)(here.val); + } + else if (op & 16) { /* length base */ + len = (unsigned)(here.val); + op &= 15; /* number of extra bits */ + if (op) { + if (bits < op) { +#ifdef INFLATE_CHUNK_READ_64LE + hold |= read64le(in) << bits; + in += 6; + bits += 48; +#else + hold += (unsigned long)(*in++) << bits; + bits += 8; +#endif + } + len += (unsigned)hold & ((1U << op) - 1); + hold >>= op; + bits -= op; + } + Tracevv((stderr, "inflate: length %u\n", len)); + if (bits < 15) { +#ifdef INFLATE_CHUNK_READ_64LE + hold |= read64le(in) << bits; + in += 6; + bits += 48; +#else + hold += (unsigned long)(*in++) << bits; + bits += 8; + hold += (unsigned long)(*in++) << bits; + bits += 8; +#endif + } + here = dcode[hold & dmask]; + dodist: + op = (unsigned)(here.bits); + hold >>= op; + bits -= op; + op = (unsigned)(here.op); + if (op & 16) { /* distance base */ + dist = (unsigned)(here.val); + op &= 15; /* number of extra bits */ + if (bits < op) { +#ifdef INFLATE_CHUNK_READ_64LE + hold |= read64le(in) << bits; + in += 6; + bits += 48; +#else + hold += (unsigned long)(*in++) << bits; + bits += 8; + if (bits < op) { + hold += (unsigned long)(*in++) << bits; + bits += 8; + } +#endif + } + dist += (unsigned)hold & ((1U << op) - 1); +#ifdef INFLATE_STRICT + if (dist > dmax) { + strm->msg = (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#endif + hold >>= op; + bits -= op; + Tracevv((stderr, "inflate: distance %u\n", dist)); + op = (unsigned)(out - beg); /* max distance in output */ + if (dist > op) { /* see if copy from window */ + op = dist - op; /* distance back in window */ + if (op > whave) { + if (state->sane) { + strm->msg = + (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + if (len <= op - whave) { + do { + *out++ = 0; + } while (--len); + continue; + } + len -= op - whave; + do { + *out++ = 0; + } while (--op > whave); + if (op == 0) { + from = out - dist; + do { + *out++ = *from++; + } while (--len); + continue; + } +#endif + } + from = window; + if (wnext >= op) { /* contiguous in window */ + from += wnext - op; + } + else { /* wrap around window */ + op -= wnext; + from += wsize - op; + if (op < len) { /* some from end of window */ + len -= op; + out = chunkcopy_safe(out, from, op, limit); + from = window; /* more from start of window */ + op = wnext; + /* This (rare) case can create a situation where + the first chunkcopy below must be checked. + */ + } + } + if (op < len) { /* still need some from output */ + out = chunkcopy_safe(out, from, op, limit); + len -= op; + /* When dist is small the amount of data that can be + copied from the window is also small, and progress + towards the dangerous end of the output buffer is + also small. This means that for trivial memsets and + for chunkunroll_relaxed() a safety check is + unnecessary. However, these conditions may not be + entered at all, and in that case it's possible that + the main copy is near the end. + */ + out = chunkunroll_relaxed(out, &dist, &len); + out = chunkcopy_safe(out, out - dist, len, limit); + } else { + /* from points to window, so there is no risk of + overlapping pointers requiring memset-like behaviour + */ + out = chunkcopy_safe(out, from, len, limit); + } + } + else { + /* Whole reference is in range of current output. No + range checks are necessary because we start with room + for at least 258 bytes of output, so unroll and roundoff + operations can write beyond `out+len` so long as they + stay within 258 bytes of `out`. + */ + out = chunkcopy_lapped_relaxed(out, dist, len); + } + } + else if ((op & 64) == 0) { /* 2nd level distance code */ + here = dcode[here.val + (hold & ((1U << op) - 1))]; + goto dodist; + } + else { + strm->msg = (char *)"invalid distance code"; + state->mode = BAD; + break; + } + } + else if ((op & 64) == 0) { /* 2nd level length code */ + here = lcode[here.val + (hold & ((1U << op) - 1))]; + goto dolen; + } + else if (op & 32) { /* end-of-block */ + Tracevv((stderr, "inflate: end of block\n")); + state->mode = TYPE; + break; + } + else { + strm->msg = (char *)"invalid literal/length code"; + state->mode = BAD; + break; + } + } while (in < last && out < end); + + /* return unused bytes (on entry, bits < 8, so in won't go too far back) */ + len = bits >> 3; + in -= len; + bits -= len << 3; + hold &= (1U << bits) - 1; + + /* update state and return */ + strm->next_in = in; + strm->next_out = out; + strm->avail_in = (unsigned)(in < last ? + (INFLATE_FAST_MIN_INPUT - 1) + (last - in) : + (INFLATE_FAST_MIN_INPUT - 1) - (in - last)); + strm->avail_out = (unsigned)(out < end ? + (INFLATE_FAST_MIN_OUTPUT - 1) + (end - out) : + (INFLATE_FAST_MIN_OUTPUT - 1) - (out - end)); + state->hold = hold; + state->bits = bits; + + Assert((state->hold >> state->bits) == 0, "invalid input data state"); + return; +} + +/* + inflate_fast() speedups that turned out slower (on a PowerPC G3 750CXe): + - Using bit fields for code structure + - Different op definition to avoid & for extra bits (do & for table bits) + - Three separate decoding do-loops for direct, window, and wnext == 0 + - Special case for distance > 1 copies to do overlapped load and store copy + - Explicit branch predictions (based on measured branch probabilities) + - Deferring match copy and interspersed it with decoding subsequent codes + - Swapping literal/length else + - Swapping window/direct else + - Larger unrolled copy loops (three is about right) + - Moving len -= 3 statement into middle of loop + */ + +#endif /* !ASMINF */ diff --git a/third_party/zlib/contrib/optimizations/inffast_chunk.h b/third_party/zlib/contrib/optimizations/inffast_chunk.h new file mode 100644 index 0000000000..39c771b863 --- /dev/null +++ b/third_party/zlib/contrib/optimizations/inffast_chunk.h @@ -0,0 +1,26 @@ +/* inffast_chunk.h -- header to use inffast_chunk.c + * Copyright (C) 1995-2003, 2010 Mark Adler + * Copyright (C) 2017 ARM, Inc. + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* WARNING: this file should *not* be used by applications. It is + part of the implementation of the compression library and is + subject to change. Applications should only use zlib.h. + */ + +#include "inffast.h" + +/* INFLATE_FAST_MIN_INPUT: the minimum number of input bytes needed so that + we can safely call inflate_fast() with only one up-front bounds check. One + length/distance code pair (15 bits for the length code, 5 bits for length + extra, 15 bits for the distance code, 13 bits for distance extra) requires + reading up to 48 input bits (6 bytes). The wide input data reading option + requires a little endian machine, and reads 64 input bits (8 bytes). +*/ +#ifdef INFLATE_CHUNK_READ_64LE +#undef INFLATE_FAST_MIN_INPUT +#define INFLATE_FAST_MIN_INPUT 8 +#endif + +void ZLIB_INTERNAL inflate_fast_chunk_ OF((z_streamp strm, unsigned start)); diff --git a/third_party/zlib/contrib/optimizations/inflate.c b/third_party/zlib/contrib/optimizations/inflate.c new file mode 100644 index 0000000000..81d558bd6e --- /dev/null +++ b/third_party/zlib/contrib/optimizations/inflate.c @@ -0,0 +1,1583 @@ +/* inflate.c -- zlib decompression + * Copyright (C) 1995-2016 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* + * Change history: + * + * 1.2.beta0 24 Nov 2002 + * - First version -- complete rewrite of inflate to simplify code, avoid + * creation of window when not needed, minimize use of window when it is + * needed, make inffast.c even faster, implement gzip decoding, and to + * improve code readability and style over the previous zlib inflate code + * + * 1.2.beta1 25 Nov 2002 + * - Use pointers for available input and output checking in inffast.c + * - Remove input and output counters in inffast.c + * - Change inffast.c entry and loop from avail_in >= 7 to >= 6 + * - Remove unnecessary second byte pull from length extra in inffast.c + * - Unroll direct copy to three copies per loop in inffast.c + * + * 1.2.beta2 4 Dec 2002 + * - Change external routine names to reduce potential conflicts + * - Correct filename to inffixed.h for fixed tables in inflate.c + * - Make hbuf[] unsigned char to match parameter type in inflate.c + * - Change strm->next_out[-state->offset] to *(strm->next_out - state->offset) + * to avoid negation problem on Alphas (64 bit) in inflate.c + * + * 1.2.beta3 22 Dec 2002 + * - Add comments on state->bits assertion in inffast.c + * - Add comments on op field in inftrees.h + * - Fix bug in reuse of allocated window after inflateReset() + * - Remove bit fields--back to byte structure for speed + * - Remove distance extra == 0 check in inflate_fast()--only helps for lengths + * - Change post-increments to pre-increments in inflate_fast(), PPC biased? + * - Add compile time option, POSTINC, to use post-increments instead (Intel?) + * - Make MATCH copy in inflate() much faster for when inflate_fast() not used + * - Use local copies of stream next and avail values, as well as local bit + * buffer and bit count in inflate()--for speed when inflate_fast() not used + * + * 1.2.beta4 1 Jan 2003 + * - Split ptr - 257 statements in inflate_table() to avoid compiler warnings + * - Move a comment on output buffer sizes from inffast.c to inflate.c + * - Add comments in inffast.c to introduce the inflate_fast() routine + * - Rearrange window copies in inflate_fast() for speed and simplification + * - Unroll last copy for window match in inflate_fast() + * - Use local copies of window variables in inflate_fast() for speed + * - Pull out common wnext == 0 case for speed in inflate_fast() + * - Make op and len in inflate_fast() unsigned for consistency + * - Add FAR to lcode and dcode declarations in inflate_fast() + * - Simplified bad distance check in inflate_fast() + * - Added inflateBackInit(), inflateBack(), and inflateBackEnd() in new + * source file infback.c to provide a call-back interface to inflate for + * programs like gzip and unzip -- uses window as output buffer to avoid + * window copying + * + * 1.2.beta5 1 Jan 2003 + * - Improved inflateBack() interface to allow the caller to provide initial + * input in strm. + * - Fixed stored blocks bug in inflateBack() + * + * 1.2.beta6 4 Jan 2003 + * - Added comments in inffast.c on effectiveness of POSTINC + * - Typecasting all around to reduce compiler warnings + * - Changed loops from while (1) or do {} while (1) to for (;;), again to + * make compilers happy + * - Changed type of window in inflateBackInit() to unsigned char * + * + * 1.2.beta7 27 Jan 2003 + * - Changed many types to unsigned or unsigned short to avoid warnings + * - Added inflateCopy() function + * + * 1.2.0 9 Mar 2003 + * - Changed inflateBack() interface to provide separate opaque descriptors + * for the in() and out() functions + * - Changed inflateBack() argument and in_func typedef to swap the length + * and buffer address return values for the input function + * - Check next_in and next_out for Z_NULL on entry to inflate() + * + * The history for versions after 1.2.0 are in ChangeLog in zlib distribution. + */ + +#include "zutil.h" +#include "inftrees.h" +#include "inflate.h" +#include "contrib/optimizations/inffast_chunk.h" +#include "contrib/optimizations/chunkcopy.h" + +#ifdef MAKEFIXED +# ifndef BUILDFIXED +# define BUILDFIXED +# endif +#endif + +/* function prototypes */ +local int inflateStateCheck OF((z_streamp strm)); +local void fixedtables OF((struct inflate_state FAR *state)); +local int updatewindow OF((z_streamp strm, const unsigned char FAR *end, + unsigned copy)); +#ifdef BUILDFIXED + void makefixed OF((void)); +#endif +local unsigned syncsearch OF((unsigned FAR *have, const unsigned char FAR *buf, + unsigned len)); + +local int inflateStateCheck(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (strm == Z_NULL || + strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) + return 1; + state = (struct inflate_state FAR *)strm->state; + if (state == Z_NULL || state->strm != strm || + state->mode < HEAD || state->mode > SYNC) + return 1; + return 0; +} + +int ZEXPORT inflateResetKeep(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + strm->total_in = strm->total_out = state->total = 0; + strm->msg = Z_NULL; + if (state->wrap) /* to support ill-conceived Java test suite */ + strm->adler = state->wrap & 1; + state->mode = HEAD; + state->last = 0; + state->havedict = 0; + state->dmax = 32768U; + state->head = Z_NULL; + state->hold = 0; + state->bits = 0; + state->lencode = state->distcode = state->next = state->codes; + state->sane = 1; + state->back = -1; + Tracev((stderr, "inflate: reset\n")); + return Z_OK; +} + +int ZEXPORT inflateReset(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + state->wsize = 0; + state->whave = 0; + state->wnext = 0; + return inflateResetKeep(strm); +} + +int ZEXPORT inflateReset2(strm, windowBits) +z_streamp strm; +int windowBits; +{ + int wrap; + struct inflate_state FAR *state; + + /* get the state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + + /* extract wrap request from windowBits parameter */ + if (windowBits < 0) { + wrap = 0; + windowBits = -windowBits; + } + else { + wrap = (windowBits >> 4) + 5; +#ifdef GUNZIP + if (windowBits < 48) + windowBits &= 15; +#endif + } + + /* set number of window bits, free window if different */ + if (windowBits && (windowBits < 8 || windowBits > 15)) + return Z_STREAM_ERROR; + if (state->window != Z_NULL && state->wbits != (unsigned)windowBits) { + ZFREE(strm, state->window); + state->window = Z_NULL; + } + + /* update state and reset the rest of it */ + state->wrap = wrap; + state->wbits = (unsigned)windowBits; + return inflateReset(strm); +} + +int ZEXPORT inflateInit2_(strm, windowBits, version, stream_size) +z_streamp strm; +int windowBits; +const char *version; +int stream_size; +{ + int ret; + struct inflate_state FAR *state; + + if (version == Z_NULL || version[0] != ZLIB_VERSION[0] || + stream_size != (int)(sizeof(z_stream))) + return Z_VERSION_ERROR; + if (strm == Z_NULL) return Z_STREAM_ERROR; + strm->msg = Z_NULL; /* in case we return an error */ + if (strm->zalloc == (alloc_func)0) { +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zalloc = zcalloc; + strm->opaque = (voidpf)0; +#endif + } + if (strm->zfree == (free_func)0) +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zfree = zcfree; +#endif + state = (struct inflate_state FAR *) + ZALLOC(strm, 1, sizeof(struct inflate_state)); + if (state == Z_NULL) return Z_MEM_ERROR; + Tracev((stderr, "inflate: allocated\n")); + strm->state = (struct internal_state FAR *)state; + state->strm = strm; + state->window = Z_NULL; + state->mode = HEAD; /* to pass state test in inflateReset2() */ + state->check = 1L; /* 1L is the result of adler32() zero length data */ + ret = inflateReset2(strm, windowBits); + if (ret != Z_OK) { + ZFREE(strm, state); + strm->state = Z_NULL; + } + return ret; +} + +int ZEXPORT inflateInit_(strm, version, stream_size) +z_streamp strm; +const char *version; +int stream_size; +{ + return inflateInit2_(strm, DEF_WBITS, version, stream_size); +} + +int ZEXPORT inflatePrime(strm, bits, value) +z_streamp strm; +int bits; +int value; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (bits < 0) { + state->hold = 0; + state->bits = 0; + return Z_OK; + } + if (bits > 16 || state->bits + (uInt)bits > 32) return Z_STREAM_ERROR; + value &= (1L << bits) - 1; + state->hold += (unsigned)value << state->bits; + state->bits += (uInt)bits; + return Z_OK; +} + +/* + Return state with length and distance decoding tables and index sizes set to + fixed code decoding. Normally this returns fixed tables from inffixed.h. + If BUILDFIXED is defined, then instead this routine builds the tables the + first time it's called, and returns those tables the first time and + thereafter. This reduces the size of the code by about 2K bytes, in + exchange for a little execution time. However, BUILDFIXED should not be + used for threaded applications, since the rewriting of the tables and virgin + may not be thread-safe. + */ +local void fixedtables(state) +struct inflate_state FAR *state; +{ +#ifdef BUILDFIXED + static int virgin = 1; + static code *lenfix, *distfix; + static code fixed[544]; + + /* build fixed huffman tables if first call (may not be thread safe) */ + if (virgin) { + unsigned sym, bits; + static code *next; + + /* literal/length table */ + sym = 0; + while (sym < 144) state->lens[sym++] = 8; + while (sym < 256) state->lens[sym++] = 9; + while (sym < 280) state->lens[sym++] = 7; + while (sym < 288) state->lens[sym++] = 8; + next = fixed; + lenfix = next; + bits = 9; + inflate_table(LENS, state->lens, 288, &(next), &(bits), state->work); + + /* distance table */ + sym = 0; + while (sym < 32) state->lens[sym++] = 5; + distfix = next; + bits = 5; + inflate_table(DISTS, state->lens, 32, &(next), &(bits), state->work); + + /* do this just once */ + virgin = 0; + } +#else /* !BUILDFIXED */ +# include "inffixed.h" +#endif /* BUILDFIXED */ + state->lencode = lenfix; + state->lenbits = 9; + state->distcode = distfix; + state->distbits = 5; +} + +#ifdef MAKEFIXED +#include + +/* + Write out the inffixed.h that is #include'd above. Defining MAKEFIXED also + defines BUILDFIXED, so the tables are built on the fly. makefixed() writes + those tables to stdout, which would be piped to inffixed.h. A small program + can simply call makefixed to do this: + + void makefixed(void); + + int main(void) + { + makefixed(); + return 0; + } + + Then that can be linked with zlib built with MAKEFIXED defined and run: + + a.out > inffixed.h + */ +void makefixed() +{ + unsigned low, size; + struct inflate_state state; + + fixedtables(&state); + puts(" /* inffixed.h -- table for decoding fixed codes"); + puts(" * Generated automatically by makefixed()."); + puts(" */"); + puts(""); + puts(" /* WARNING: this file should *not* be used by applications."); + puts(" It is part of the implementation of this library and is"); + puts(" subject to change. Applications should only use zlib.h."); + puts(" */"); + puts(""); + size = 1U << 9; + printf(" static const code lenfix[%u] = {", size); + low = 0; + for (;;) { + if ((low % 7) == 0) printf("\n "); + printf("{%u,%u,%d}", (low & 127) == 99 ? 64 : state.lencode[low].op, + state.lencode[low].bits, state.lencode[low].val); + if (++low == size) break; + putchar(','); + } + puts("\n };"); + size = 1U << 5; + printf("\n static const code distfix[%u] = {", size); + low = 0; + for (;;) { + if ((low % 6) == 0) printf("\n "); + printf("{%u,%u,%d}", state.distcode[low].op, state.distcode[low].bits, + state.distcode[low].val); + if (++low == size) break; + putchar(','); + } + puts("\n };"); +} +#endif /* MAKEFIXED */ + +/* + Update the window with the last wsize (normally 32K) bytes written before + returning. If window does not exist yet, create it. This is only called + when a window is already in use, or when output has been written during this + inflate call, but the end of the deflate stream has not been reached yet. + It is also called to create a window for dictionary data when a dictionary + is loaded. + + Providing output buffers larger than 32K to inflate() should provide a speed + advantage, since only the last 32K of output is copied to the sliding window + upon return from inflate(), and since all distances after the first 32K of + output will fall in the output data, making match copies simpler and faster. + The advantage may be dependent on the size of the processor's data caches. + */ +local int updatewindow(strm, end, copy) +z_streamp strm; +const Bytef *end; +unsigned copy; +{ + struct inflate_state FAR *state; + unsigned dist; + + state = (struct inflate_state FAR *)strm->state; + + /* if it hasn't been done already, allocate space for the window */ + if (state->window == Z_NULL) { + unsigned wsize = 1U << state->wbits; + state->window = (unsigned char FAR *) + ZALLOC(strm, wsize + CHUNKCOPY_CHUNK_SIZE, + sizeof(unsigned char)); + if (state->window == Z_NULL) return 1; +#ifdef INFLATE_CLEAR_UNUSED_UNDEFINED + /* Copies from the overflow portion of this buffer are undefined and + may cause analysis tools to raise a warning if we don't initialize + it. However, this undefined data overwrites other undefined data + and is subsequently either overwritten or left deliberately + undefined at the end of decode; so there's really no point. + */ + zmemzero(state->window + wsize, CHUNKCOPY_CHUNK_SIZE); +#endif + } + + /* if window not in use yet, initialize */ + if (state->wsize == 0) { + state->wsize = 1U << state->wbits; + state->wnext = 0; + state->whave = 0; + } + + /* copy state->wsize or less output bytes into the circular window */ + if (copy >= state->wsize) { + zmemcpy(state->window, end - state->wsize, state->wsize); + state->wnext = 0; + state->whave = state->wsize; + } + else { + dist = state->wsize - state->wnext; + if (dist > copy) dist = copy; + zmemcpy(state->window + state->wnext, end - copy, dist); + copy -= dist; + if (copy) { + zmemcpy(state->window, end - copy, copy); + state->wnext = copy; + state->whave = state->wsize; + } + else { + state->wnext += dist; + if (state->wnext == state->wsize) state->wnext = 0; + if (state->whave < state->wsize) state->whave += dist; + } + } + return 0; +} + +/* Macros for inflate(): */ + +/* check function to use adler32() for zlib or crc32() for gzip */ +#ifdef GUNZIP +# define UPDATE(check, buf, len) \ + (state->flags ? crc32(check, buf, len) : adler32(check, buf, len)) +#else +# define UPDATE(check, buf, len) adler32(check, buf, len) +#endif + +/* check macros for header crc */ +#ifdef GUNZIP +# define CRC2(check, word) \ + do { \ + hbuf[0] = (unsigned char)(word); \ + hbuf[1] = (unsigned char)((word) >> 8); \ + check = crc32(check, hbuf, 2); \ + } while (0) + +# define CRC4(check, word) \ + do { \ + hbuf[0] = (unsigned char)(word); \ + hbuf[1] = (unsigned char)((word) >> 8); \ + hbuf[2] = (unsigned char)((word) >> 16); \ + hbuf[3] = (unsigned char)((word) >> 24); \ + check = crc32(check, hbuf, 4); \ + } while (0) +#endif + +/* Load registers with state in inflate() for speed */ +#define LOAD() \ + do { \ + put = strm->next_out; \ + left = strm->avail_out; \ + next = strm->next_in; \ + have = strm->avail_in; \ + hold = state->hold; \ + bits = state->bits; \ + } while (0) + +/* Restore state from registers in inflate() */ +#define RESTORE() \ + do { \ + strm->next_out = put; \ + strm->avail_out = left; \ + strm->next_in = next; \ + strm->avail_in = have; \ + state->hold = hold; \ + state->bits = bits; \ + } while (0) + +/* Clear the input bit accumulator */ +#define INITBITS() \ + do { \ + hold = 0; \ + bits = 0; \ + } while (0) + +/* Get a byte of input into the bit accumulator, or return from inflate() + if there is no input available. */ +#define PULLBYTE() \ + do { \ + if (have == 0) goto inf_leave; \ + have--; \ + hold += (unsigned long)(*next++) << bits; \ + bits += 8; \ + } while (0) + +/* Assure that there are at least n bits in the bit accumulator. If there is + not enough available input to do that, then return from inflate(). */ +#define NEEDBITS(n) \ + do { \ + while (bits < (unsigned)(n)) \ + PULLBYTE(); \ + } while (0) + +/* Return the low n bits of the bit accumulator (n < 16) */ +#define BITS(n) \ + ((unsigned)hold & ((1U << (n)) - 1)) + +/* Remove n bits from the bit accumulator */ +#define DROPBITS(n) \ + do { \ + hold >>= (n); \ + bits -= (unsigned)(n); \ + } while (0) + +/* Remove zero to seven bits as needed to go to a byte boundary */ +#define BYTEBITS() \ + do { \ + hold >>= bits & 7; \ + bits -= bits & 7; \ + } while (0) + +/* + inflate() uses a state machine to process as much input data and generate as + much output data as possible before returning. The state machine is + structured roughly as follows: + + for (;;) switch (state) { + ... + case STATEn: + if (not enough input data or output space to make progress) + return; + ... make progress ... + state = STATEm; + break; + ... + } + + so when inflate() is called again, the same case is attempted again, and + if the appropriate resources are provided, the machine proceeds to the + next state. The NEEDBITS() macro is usually the way the state evaluates + whether it can proceed or should return. NEEDBITS() does the return if + the requested bits are not available. The typical use of the BITS macros + is: + + NEEDBITS(n); + ... do something with BITS(n) ... + DROPBITS(n); + + where NEEDBITS(n) either returns from inflate() if there isn't enough + input left to load n bits into the accumulator, or it continues. BITS(n) + gives the low n bits in the accumulator. When done, DROPBITS(n) drops + the low n bits off the accumulator. INITBITS() clears the accumulator + and sets the number of available bits to zero. BYTEBITS() discards just + enough bits to put the accumulator on a byte boundary. After BYTEBITS() + and a NEEDBITS(8), then BITS(8) would return the next byte in the stream. + + NEEDBITS(n) uses PULLBYTE() to get an available byte of input, or to return + if there is no input available. The decoding of variable length codes uses + PULLBYTE() directly in order to pull just enough bytes to decode the next + code, and no more. + + Some states loop until they get enough input, making sure that enough + state information is maintained to continue the loop where it left off + if NEEDBITS() returns in the loop. For example, want, need, and keep + would all have to actually be part of the saved state in case NEEDBITS() + returns: + + case STATEw: + while (want < need) { + NEEDBITS(n); + keep[want++] = BITS(n); + DROPBITS(n); + } + state = STATEx; + case STATEx: + + As shown above, if the next state is also the next case, then the break + is omitted. + + A state may also return if there is not enough output space available to + complete that state. Those states are copying stored data, writing a + literal byte, and copying a matching string. + + When returning, a "goto inf_leave" is used to update the total counters, + update the check value, and determine whether any progress has been made + during that inflate() call in order to return the proper return code. + Progress is defined as a change in either strm->avail_in or strm->avail_out. + When there is a window, goto inf_leave will update the window with the last + output written. If a goto inf_leave occurs in the middle of decompression + and there is no window currently, goto inf_leave will create one and copy + output to the window for the next call of inflate(). + + In this implementation, the flush parameter of inflate() only affects the + return code (per zlib.h). inflate() always writes as much as possible to + strm->next_out, given the space available and the provided input--the effect + documented in zlib.h of Z_SYNC_FLUSH. Furthermore, inflate() always defers + the allocation of and copying into a sliding window until necessary, which + provides the effect documented in zlib.h for Z_FINISH when the entire input + stream available. So the only thing the flush parameter actually does is: + when flush is set to Z_FINISH, inflate() cannot return Z_OK. Instead it + will return Z_BUF_ERROR if it has not reached the end of the stream. + */ + +int ZEXPORT inflate(strm, flush) +z_streamp strm; +int flush; +{ + struct inflate_state FAR *state; + z_const unsigned char FAR *next; /* next input */ + unsigned char FAR *put; /* next output */ + unsigned have, left; /* available input and output */ + unsigned long hold; /* bit buffer */ + unsigned bits; /* bits in bit buffer */ + unsigned in, out; /* save starting available input and output */ + unsigned copy; /* number of stored or match bytes to copy */ + unsigned char FAR *from; /* where to copy match bytes from */ + code here; /* current decoding table entry */ + code last; /* parent table entry */ + unsigned len; /* length to copy for repeats, bits to drop */ + int ret; /* return code */ +#ifdef GUNZIP + unsigned char hbuf[4]; /* buffer for gzip header crc calculation */ +#endif + static const unsigned short order[19] = /* permutation of code lengths */ + {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; + + if (inflateStateCheck(strm) || strm->next_out == Z_NULL || + (strm->next_in == Z_NULL && strm->avail_in != 0)) + return Z_STREAM_ERROR; + + state = (struct inflate_state FAR *)strm->state; + if (state->mode == TYPE) state->mode = TYPEDO; /* skip check */ + LOAD(); + in = have; + out = left; + ret = Z_OK; + for (;;) + switch (state->mode) { + case HEAD: + if (state->wrap == 0) { + state->mode = TYPEDO; + break; + } + NEEDBITS(16); +#ifdef GUNZIP + if ((state->wrap & 2) && hold == 0x8b1f) { /* gzip header */ + if (state->wbits == 0) + state->wbits = 15; + state->check = crc32(0L, Z_NULL, 0); + CRC2(state->check, hold); + INITBITS(); + state->mode = FLAGS; + break; + } + state->flags = 0; /* expect zlib header */ + if (state->head != Z_NULL) + state->head->done = -1; + if (!(state->wrap & 1) || /* check if zlib header allowed */ +#else + if ( +#endif + ((BITS(8) << 8) + (hold >> 8)) % 31) { + strm->msg = (char *)"incorrect header check"; + state->mode = BAD; + break; + } + if (BITS(4) != Z_DEFLATED) { + strm->msg = (char *)"unknown compression method"; + state->mode = BAD; + break; + } + DROPBITS(4); + len = BITS(4) + 8; + if (state->wbits == 0) + state->wbits = len; + if (len > 15 || len > state->wbits) { + strm->msg = (char *)"invalid window size"; + state->mode = BAD; + break; + } + state->dmax = 1U << len; + Tracev((stderr, "inflate: zlib header ok\n")); + strm->adler = state->check = adler32(0L, Z_NULL, 0); + state->mode = hold & 0x200 ? DICTID : TYPE; + INITBITS(); + break; +#ifdef GUNZIP + case FLAGS: + NEEDBITS(16); + state->flags = (int)(hold); + if ((state->flags & 0xff) != Z_DEFLATED) { + strm->msg = (char *)"unknown compression method"; + state->mode = BAD; + break; + } + if (state->flags & 0xe000) { + strm->msg = (char *)"unknown header flags set"; + state->mode = BAD; + break; + } + if (state->head != Z_NULL) + state->head->text = (int)((hold >> 8) & 1); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); + INITBITS(); + state->mode = TIME; + case TIME: + NEEDBITS(32); + if (state->head != Z_NULL) + state->head->time = hold; + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC4(state->check, hold); + INITBITS(); + state->mode = OS; + case OS: + NEEDBITS(16); + if (state->head != Z_NULL) { + state->head->xflags = (int)(hold & 0xff); + state->head->os = (int)(hold >> 8); + } + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); + INITBITS(); + state->mode = EXLEN; + case EXLEN: + if (state->flags & 0x0400) { + NEEDBITS(16); + state->length = (unsigned)(hold); + if (state->head != Z_NULL) + state->head->extra_len = (unsigned)hold; + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); + INITBITS(); + } + else if (state->head != Z_NULL) + state->head->extra = Z_NULL; + state->mode = EXTRA; + case EXTRA: + if (state->flags & 0x0400) { + copy = state->length; + if (copy > have) copy = have; + if (copy) { + if (state->head != Z_NULL && + state->head->extra != Z_NULL) { + len = state->head->extra_len - state->length; + zmemcpy(state->head->extra + len, next, + len + copy > state->head->extra_max ? + state->head->extra_max - len : copy); + } + if ((state->flags & 0x0200) && (state->wrap & 4)) + state->check = crc32(state->check, next, copy); + have -= copy; + next += copy; + state->length -= copy; + } + if (state->length) goto inf_leave; + } + state->length = 0; + state->mode = NAME; + case NAME: + if (state->flags & 0x0800) { + if (have == 0) goto inf_leave; + copy = 0; + do { + len = (unsigned)(next[copy++]); + if (state->head != Z_NULL && + state->head->name != Z_NULL && + state->length < state->head->name_max) + state->head->name[state->length++] = (Bytef)len; + } while (len && copy < have); + if ((state->flags & 0x0200) && (state->wrap & 4)) + state->check = crc32(state->check, next, copy); + have -= copy; + next += copy; + if (len) goto inf_leave; + } + else if (state->head != Z_NULL) + state->head->name = Z_NULL; + state->length = 0; + state->mode = COMMENT; + case COMMENT: + if (state->flags & 0x1000) { + if (have == 0) goto inf_leave; + copy = 0; + do { + len = (unsigned)(next[copy++]); + if (state->head != Z_NULL && + state->head->comment != Z_NULL && + state->length < state->head->comm_max) + state->head->comment[state->length++] = (Bytef)len; + } while (len && copy < have); + if ((state->flags & 0x0200) && (state->wrap & 4)) + state->check = crc32(state->check, next, copy); + have -= copy; + next += copy; + if (len) goto inf_leave; + } + else if (state->head != Z_NULL) + state->head->comment = Z_NULL; + state->mode = HCRC; + case HCRC: + if (state->flags & 0x0200) { + NEEDBITS(16); + if ((state->wrap & 4) && hold != (state->check & 0xffff)) { + strm->msg = (char *)"header crc mismatch"; + state->mode = BAD; + break; + } + INITBITS(); + } + if (state->head != Z_NULL) { + state->head->hcrc = (int)((state->flags >> 9) & 1); + state->head->done = 1; + } + strm->adler = state->check = crc32(0L, Z_NULL, 0); + state->mode = TYPE; + break; +#endif + case DICTID: + NEEDBITS(32); + strm->adler = state->check = ZSWAP32(hold); + INITBITS(); + state->mode = DICT; + case DICT: + if (state->havedict == 0) { + RESTORE(); + return Z_NEED_DICT; + } + strm->adler = state->check = adler32(0L, Z_NULL, 0); + state->mode = TYPE; + case TYPE: + if (flush == Z_BLOCK || flush == Z_TREES) goto inf_leave; + case TYPEDO: + if (state->last) { + BYTEBITS(); + state->mode = CHECK; + break; + } + NEEDBITS(3); + state->last = BITS(1); + DROPBITS(1); + switch (BITS(2)) { + case 0: /* stored block */ + Tracev((stderr, "inflate: stored block%s\n", + state->last ? " (last)" : "")); + state->mode = STORED; + break; + case 1: /* fixed block */ + fixedtables(state); + Tracev((stderr, "inflate: fixed codes block%s\n", + state->last ? " (last)" : "")); + state->mode = LEN_; /* decode codes */ + if (flush == Z_TREES) { + DROPBITS(2); + goto inf_leave; + } + break; + case 2: /* dynamic block */ + Tracev((stderr, "inflate: dynamic codes block%s\n", + state->last ? " (last)" : "")); + state->mode = TABLE; + break; + case 3: + strm->msg = (char *)"invalid block type"; + state->mode = BAD; + } + DROPBITS(2); + break; + case STORED: + BYTEBITS(); /* go to byte boundary */ + NEEDBITS(32); + if ((hold & 0xffff) != ((hold >> 16) ^ 0xffff)) { + strm->msg = (char *)"invalid stored block lengths"; + state->mode = BAD; + break; + } + state->length = (unsigned)hold & 0xffff; + Tracev((stderr, "inflate: stored length %u\n", + state->length)); + INITBITS(); + state->mode = COPY_; + if (flush == Z_TREES) goto inf_leave; + case COPY_: + state->mode = COPY; + case COPY: + copy = state->length; + if (copy) { + if (copy > have) copy = have; + if (copy > left) copy = left; + if (copy == 0) goto inf_leave; + zmemcpy(put, next, copy); + have -= copy; + next += copy; + left -= copy; + put += copy; + state->length -= copy; + break; + } + Tracev((stderr, "inflate: stored end\n")); + state->mode = TYPE; + break; + case TABLE: + NEEDBITS(14); + state->nlen = BITS(5) + 257; + DROPBITS(5); + state->ndist = BITS(5) + 1; + DROPBITS(5); + state->ncode = BITS(4) + 4; + DROPBITS(4); +#ifndef PKZIP_BUG_WORKAROUND + if (state->nlen > 286 || state->ndist > 30) { + strm->msg = (char *)"too many length or distance symbols"; + state->mode = BAD; + break; + } +#endif + Tracev((stderr, "inflate: table sizes ok\n")); + state->have = 0; + state->mode = LENLENS; + case LENLENS: + while (state->have < state->ncode) { + NEEDBITS(3); + state->lens[order[state->have++]] = (unsigned short)BITS(3); + DROPBITS(3); + } + while (state->have < 19) + state->lens[order[state->have++]] = 0; + state->next = state->codes; + state->lencode = (const code FAR *)(state->next); + state->lenbits = 7; + ret = inflate_table(CODES, state->lens, 19, &(state->next), + &(state->lenbits), state->work); + if (ret) { + strm->msg = (char *)"invalid code lengths set"; + state->mode = BAD; + break; + } + Tracev((stderr, "inflate: code lengths ok\n")); + state->have = 0; + state->mode = CODELENS; + case CODELENS: + while (state->have < state->nlen + state->ndist) { + for (;;) { + here = state->lencode[BITS(state->lenbits)]; + if ((unsigned)(here.bits) <= bits) break; + PULLBYTE(); + } + if (here.val < 16) { + DROPBITS(here.bits); + state->lens[state->have++] = here.val; + } + else { + if (here.val == 16) { + NEEDBITS(here.bits + 2); + DROPBITS(here.bits); + if (state->have == 0) { + strm->msg = (char *)"invalid bit length repeat"; + state->mode = BAD; + break; + } + len = state->lens[state->have - 1]; + copy = 3 + BITS(2); + DROPBITS(2); + } + else if (here.val == 17) { + NEEDBITS(here.bits + 3); + DROPBITS(here.bits); + len = 0; + copy = 3 + BITS(3); + DROPBITS(3); + } + else { + NEEDBITS(here.bits + 7); + DROPBITS(here.bits); + len = 0; + copy = 11 + BITS(7); + DROPBITS(7); + } + if (state->have + copy > state->nlen + state->ndist) { + strm->msg = (char *)"invalid bit length repeat"; + state->mode = BAD; + break; + } + while (copy--) + state->lens[state->have++] = (unsigned short)len; + } + } + + /* handle error breaks in while */ + if (state->mode == BAD) break; + + /* check for end-of-block code (better have one) */ + if (state->lens[256] == 0) { + strm->msg = (char *)"invalid code -- missing end-of-block"; + state->mode = BAD; + break; + } + + /* build code tables -- note: do not change the lenbits or distbits + values here (9 and 6) without reading the comments in inftrees.h + concerning the ENOUGH constants, which depend on those values */ + state->next = state->codes; + state->lencode = (const code FAR *)(state->next); + state->lenbits = 9; + ret = inflate_table(LENS, state->lens, state->nlen, &(state->next), + &(state->lenbits), state->work); + if (ret) { + strm->msg = (char *)"invalid literal/lengths set"; + state->mode = BAD; + break; + } + state->distcode = (const code FAR *)(state->next); + state->distbits = 6; + ret = inflate_table(DISTS, state->lens + state->nlen, state->ndist, + &(state->next), &(state->distbits), state->work); + if (ret) { + strm->msg = (char *)"invalid distances set"; + state->mode = BAD; + break; + } + Tracev((stderr, "inflate: codes ok\n")); + state->mode = LEN_; + if (flush == Z_TREES) goto inf_leave; + case LEN_: + state->mode = LEN; + case LEN: + if (have >= INFLATE_FAST_MIN_INPUT && + left >= INFLATE_FAST_MIN_OUTPUT) { + RESTORE(); + inflate_fast_chunk_(strm, out); + LOAD(); + if (state->mode == TYPE) + state->back = -1; + break; + } + state->back = 0; + for (;;) { + here = state->lencode[BITS(state->lenbits)]; + if ((unsigned)(here.bits) <= bits) break; + PULLBYTE(); + } + if (here.op && (here.op & 0xf0) == 0) { + last = here; + for (;;) { + here = state->lencode[last.val + + (BITS(last.bits + last.op) >> last.bits)]; + if ((unsigned)(last.bits + here.bits) <= bits) break; + PULLBYTE(); + } + DROPBITS(last.bits); + state->back += last.bits; + } + DROPBITS(here.bits); + state->back += here.bits; + state->length = (unsigned)here.val; + if ((int)(here.op) == 0) { + Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? + "inflate: literal '%c'\n" : + "inflate: literal 0x%02x\n", here.val)); + state->mode = LIT; + break; + } + if (here.op & 32) { + Tracevv((stderr, "inflate: end of block\n")); + state->back = -1; + state->mode = TYPE; + break; + } + if (here.op & 64) { + strm->msg = (char *)"invalid literal/length code"; + state->mode = BAD; + break; + } + state->extra = (unsigned)(here.op) & 15; + state->mode = LENEXT; + case LENEXT: + if (state->extra) { + NEEDBITS(state->extra); + state->length += BITS(state->extra); + DROPBITS(state->extra); + state->back += state->extra; + } + Tracevv((stderr, "inflate: length %u\n", state->length)); + state->was = state->length; + state->mode = DIST; + case DIST: + for (;;) { + here = state->distcode[BITS(state->distbits)]; + if ((unsigned)(here.bits) <= bits) break; + PULLBYTE(); + } + if ((here.op & 0xf0) == 0) { + last = here; + for (;;) { + here = state->distcode[last.val + + (BITS(last.bits + last.op) >> last.bits)]; + if ((unsigned)(last.bits + here.bits) <= bits) break; + PULLBYTE(); + } + DROPBITS(last.bits); + state->back += last.bits; + } + DROPBITS(here.bits); + state->back += here.bits; + if (here.op & 64) { + strm->msg = (char *)"invalid distance code"; + state->mode = BAD; + break; + } + state->offset = (unsigned)here.val; + state->extra = (unsigned)(here.op) & 15; + state->mode = DISTEXT; + case DISTEXT: + if (state->extra) { + NEEDBITS(state->extra); + state->offset += BITS(state->extra); + DROPBITS(state->extra); + state->back += state->extra; + } +#ifdef INFLATE_STRICT + if (state->offset > state->dmax) { + strm->msg = (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#endif + Tracevv((stderr, "inflate: distance %u\n", state->offset)); + state->mode = MATCH; + case MATCH: + if (left == 0) goto inf_leave; + copy = out - left; + if (state->offset > copy) { /* copy from window */ + copy = state->offset - copy; + if (copy > state->whave) { + if (state->sane) { + strm->msg = (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + Trace((stderr, "inflate.c too far\n")); + copy -= state->whave; + if (copy > state->length) copy = state->length; + if (copy > left) copy = left; + left -= copy; + state->length -= copy; + do { + *put++ = 0; + } while (--copy); + if (state->length == 0) state->mode = LEN; + break; +#endif + } + if (copy > state->wnext) { + copy -= state->wnext; + from = state->window + (state->wsize - copy); + } + else + from = state->window + (state->wnext - copy); + if (copy > state->length) copy = state->length; + if (copy > left) copy = left; + put = chunkcopy_safe(put, from, copy, put + left); + } + else { /* copy from output */ + copy = state->length; + if (copy > left) copy = left; + put = chunkcopy_lapped_safe(put, state->offset, copy, put + left); + } + left -= copy; + state->length -= copy; + if (state->length == 0) state->mode = LEN; + break; + case LIT: + if (left == 0) goto inf_leave; + *put++ = (unsigned char)(state->length); + left--; + state->mode = LEN; + break; + case CHECK: + if (state->wrap) { + NEEDBITS(32); + out -= left; + strm->total_out += out; + state->total += out; + if ((state->wrap & 4) && out) + strm->adler = state->check = + UPDATE(state->check, put - out, out); + out = left; + if ((state->wrap & 4) && ( +#ifdef GUNZIP + state->flags ? hold : +#endif + ZSWAP32(hold)) != state->check) { + strm->msg = (char *)"incorrect data check"; + state->mode = BAD; + break; + } + INITBITS(); + Tracev((stderr, "inflate: check matches trailer\n")); + } +#ifdef GUNZIP + state->mode = LENGTH; + case LENGTH: + if (state->wrap && state->flags) { + NEEDBITS(32); + if (hold != (state->total & 0xffffffffUL)) { + strm->msg = (char *)"incorrect length check"; + state->mode = BAD; + break; + } + INITBITS(); + Tracev((stderr, "inflate: length matches trailer\n")); + } +#endif + state->mode = DONE; + case DONE: + ret = Z_STREAM_END; + goto inf_leave; + case BAD: + ret = Z_DATA_ERROR; + goto inf_leave; + case MEM: + return Z_MEM_ERROR; + case SYNC: + default: + return Z_STREAM_ERROR; + } + + /* + Return from inflate(), updating the total counts and the check value. + If there was no progress during the inflate() call, return a buffer + error. Call updatewindow() to create and/or update the window state. + Note: a memory error from inflate() is non-recoverable. + */ + inf_leave: + /* We write a defined value in the unused space to help mark + * where the stream has ended. We don't use zeros as that can + * mislead clients relying on undefined behavior (i.e. assuming + * that the data is over when the buffer has a zero/null value). + */ + if (left >= CHUNKCOPY_CHUNK_SIZE) + memset(put, 0x55, CHUNKCOPY_CHUNK_SIZE); + else + memset(put, 0x55, left); + + RESTORE(); + if (state->wsize || (out != strm->avail_out && state->mode < BAD && + (state->mode < CHECK || flush != Z_FINISH))) + if (updatewindow(strm, strm->next_out, out - strm->avail_out)) { + state->mode = MEM; + return Z_MEM_ERROR; + } + in -= strm->avail_in; + out -= strm->avail_out; + strm->total_in += in; + strm->total_out += out; + state->total += out; + if ((state->wrap & 4) && out) + strm->adler = state->check = + UPDATE(state->check, strm->next_out - out, out); + strm->data_type = (int)state->bits + (state->last ? 64 : 0) + + (state->mode == TYPE ? 128 : 0) + + (state->mode == LEN_ || state->mode == COPY_ ? 256 : 0); + if (((in == 0 && out == 0) || flush == Z_FINISH) && ret == Z_OK) + ret = Z_BUF_ERROR; + return ret; +} + +int ZEXPORT inflateEnd(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (inflateStateCheck(strm)) + return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (state->window != Z_NULL) ZFREE(strm, state->window); + ZFREE(strm, strm->state); + strm->state = Z_NULL; + Tracev((stderr, "inflate: end\n")); + return Z_OK; +} + +int ZEXPORT inflateGetDictionary(strm, dictionary, dictLength) +z_streamp strm; +Bytef *dictionary; +uInt *dictLength; +{ + struct inflate_state FAR *state; + + /* check state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + + /* copy dictionary */ + if (state->whave && dictionary != Z_NULL) { + zmemcpy(dictionary, state->window + state->wnext, + state->whave - state->wnext); + zmemcpy(dictionary + state->whave - state->wnext, + state->window, state->wnext); + } + if (dictLength != Z_NULL) + *dictLength = state->whave; + return Z_OK; +} + +int ZEXPORT inflateSetDictionary(strm, dictionary, dictLength) +z_streamp strm; +const Bytef *dictionary; +uInt dictLength; +{ + struct inflate_state FAR *state; + unsigned long dictid; + int ret; + + /* check state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (state->wrap != 0 && state->mode != DICT) + return Z_STREAM_ERROR; + + /* check for correct dictionary identifier */ + if (state->mode == DICT) { + dictid = adler32(0L, Z_NULL, 0); + dictid = adler32(dictid, dictionary, dictLength); + if (dictid != state->check) + return Z_DATA_ERROR; + } + + /* copy dictionary to window using updatewindow(), which will amend the + existing dictionary if appropriate */ + ret = updatewindow(strm, dictionary + dictLength, dictLength); + if (ret) { + state->mode = MEM; + return Z_MEM_ERROR; + } + state->havedict = 1; + Tracev((stderr, "inflate: dictionary set\n")); + return Z_OK; +} + +int ZEXPORT inflateGetHeader(strm, head) +z_streamp strm; +gz_headerp head; +{ + struct inflate_state FAR *state; + + /* check state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if ((state->wrap & 2) == 0) return Z_STREAM_ERROR; + + /* save header structure */ + state->head = head; + head->done = 0; + return Z_OK; +} + +/* + Search buf[0..len-1] for the pattern: 0, 0, 0xff, 0xff. Return when found + or when out of input. When called, *have is the number of pattern bytes + found in order so far, in 0..3. On return *have is updated to the new + state. If on return *have equals four, then the pattern was found and the + return value is how many bytes were read including the last byte of the + pattern. If *have is less than four, then the pattern has not been found + yet and the return value is len. In the latter case, syncsearch() can be + called again with more data and the *have state. *have is initialized to + zero for the first call. + */ +local unsigned syncsearch(have, buf, len) +unsigned FAR *have; +const unsigned char FAR *buf; +unsigned len; +{ + unsigned got; + unsigned next; + + got = *have; + next = 0; + while (next < len && got < 4) { + if ((int)(buf[next]) == (got < 2 ? 0 : 0xff)) + got++; + else if (buf[next]) + got = 0; + else + got = 4 - got; + next++; + } + *have = got; + return next; +} + +int ZEXPORT inflateSync(strm) +z_streamp strm; +{ + unsigned len; /* number of bytes to look at or looked at */ + unsigned long in, out; /* temporary to save total_in and total_out */ + unsigned char buf[4]; /* to restore bit buffer to byte string */ + struct inflate_state FAR *state; + + /* check parameters */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (strm->avail_in == 0 && state->bits < 8) return Z_BUF_ERROR; + + /* if first time, start search in bit buffer */ + if (state->mode != SYNC) { + state->mode = SYNC; + state->hold <<= state->bits & 7; + state->bits -= state->bits & 7; + len = 0; + while (state->bits >= 8) { + buf[len++] = (unsigned char)(state->hold); + state->hold >>= 8; + state->bits -= 8; + } + state->have = 0; + syncsearch(&(state->have), buf, len); + } + + /* search available input */ + len = syncsearch(&(state->have), strm->next_in, strm->avail_in); + strm->avail_in -= len; + strm->next_in += len; + strm->total_in += len; + + /* return no joy or set up to restart inflate() on a new block */ + if (state->have != 4) return Z_DATA_ERROR; + in = strm->total_in; out = strm->total_out; + inflateReset(strm); + strm->total_in = in; strm->total_out = out; + state->mode = TYPE; + return Z_OK; +} + +/* + Returns true if inflate is currently at the end of a block generated by + Z_SYNC_FLUSH or Z_FULL_FLUSH. This function is used by one PPP + implementation to provide an additional safety check. PPP uses + Z_SYNC_FLUSH but removes the length bytes of the resulting empty stored + block. When decompressing, PPP checks that at the end of input packet, + inflate is waiting for these length bytes. + */ +int ZEXPORT inflateSyncPoint(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + return state->mode == STORED && state->bits == 0; +} + +int ZEXPORT inflateCopy(dest, source) +z_streamp dest; +z_streamp source; +{ + struct inflate_state FAR *state; + struct inflate_state FAR *copy; + unsigned char FAR *window; + unsigned wsize; + + /* check input */ + if (inflateStateCheck(source) || dest == Z_NULL) + return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)source->state; + + /* allocate space */ + copy = (struct inflate_state FAR *) + ZALLOC(source, 1, sizeof(struct inflate_state)); + if (copy == Z_NULL) return Z_MEM_ERROR; + window = Z_NULL; + if (state->window != Z_NULL) { + window = (unsigned char FAR *) + ZALLOC(source, 1U << state->wbits, sizeof(unsigned char)); + if (window == Z_NULL) { + ZFREE(source, copy); + return Z_MEM_ERROR; + } + } + + /* copy state */ + zmemcpy((voidpf)dest, (voidpf)source, sizeof(z_stream)); + zmemcpy((voidpf)copy, (voidpf)state, sizeof(struct inflate_state)); + copy->strm = dest; + if (state->lencode >= state->codes && + state->lencode <= state->codes + ENOUGH - 1) { + copy->lencode = copy->codes + (state->lencode - state->codes); + copy->distcode = copy->codes + (state->distcode - state->codes); + } + copy->next = copy->codes + (state->next - state->codes); + if (window != Z_NULL) { + wsize = 1U << state->wbits; + zmemcpy(window, state->window, wsize); + } + copy->window = window; + dest->state = (struct internal_state FAR *)copy; + return Z_OK; +} + +int ZEXPORT inflateUndermine(strm, subvert) +z_streamp strm; +int subvert; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + state->sane = !subvert; + return Z_OK; +#else + (void)subvert; + state->sane = 1; + return Z_DATA_ERROR; +#endif +} + +int ZEXPORT inflateValidate(strm, check) +z_streamp strm; +int check; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (check) + state->wrap |= 4; + else + state->wrap &= ~4; + return Z_OK; +} + +long ZEXPORT inflateMark(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) + return -(1L << 16); + state = (struct inflate_state FAR *)strm->state; + return (long)(((unsigned long)((long)state->back)) << 16) + + (state->mode == COPY ? state->length : + (state->mode == MATCH ? state->was - state->length : 0)); +} + +unsigned long ZEXPORT inflateCodesUsed(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (inflateStateCheck(strm)) return (unsigned long)-1; + state = (struct inflate_state FAR *)strm->state; + return (unsigned long)(state->next - state->codes); +} diff --git a/third_party/zlib/contrib/optimizations/insert_string.h b/third_party/zlib/contrib/optimizations/insert_string.h new file mode 100644 index 0000000000..d3bc33c5ab --- /dev/null +++ b/third_party/zlib/contrib/optimizations/insert_string.h @@ -0,0 +1,123 @@ +/* insert_string.h + * + * Copyright 2019 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#if defined(_MSC_VER) +#define INLINE __inline +#else +#define INLINE inline +#endif + +#include "cpu_features.h" + +// clang-format off +#if defined(CRC32_SIMD_SSE42_PCLMUL) + #include /* Required to make MSVC bot build pass. */ + + #if defined(__clang__) || defined(__GNUC__) + #define TARGET_CPU_WITH_CRC __attribute__((target("sse4.2"))) + #else + #define TARGET_CPU_WITH_CRC + #endif + + #define _cpu_crc32_u32 _mm_crc32_u32 + +#elif defined(CRC32_ARMV8_CRC32) + #if defined(__clang__) + #define __crc32cw __builtin_arm_crc32cw + #endif + + #if defined(__aarch64__) + #define TARGET_CPU_WITH_CRC __attribute__((target("crc"))) + #else // !defined(__aarch64__) + #define TARGET_CPU_WITH_CRC __attribute__((target("armv8-a,crc"))) + #endif // defined(__aarch64__) + + #define _cpu_crc32_u32 __crc32cw + +#endif +// clang-format on + +#if defined(TARGET_CPU_WITH_CRC) + +TARGET_CPU_WITH_CRC +local INLINE Pos insert_string_simd(deflate_state* const s, const Pos str) { + Pos ret; + unsigned *ip, val, h = 0; + + ip = (unsigned*)&s->window[str]; + val = *ip; + + if (s->level >= 6) + val &= 0xFFFFFF; + + /* Unlike the case of data integrity checks for GZIP format where the + * polynomial used is defined (https://tools.ietf.org/html/rfc1952#page-11), + * here it is just a hash function for the hash table used while + * performing compression. + */ + h = _cpu_crc32_u32(h, val); + + ret = s->head[h & s->hash_mask]; + s->head[h & s->hash_mask] = str; + s->prev[str & s->w_mask] = ret; + return ret; +} + +#endif // TARGET_CPU_WITH_CRC + +/* =========================================================================== + * Update a hash value with the given input byte + * IN assertion: all calls to UPDATE_HASH are made with consecutive input + * characters, so that a running hash key can be computed from the previous + * key instead of complete recalculation each time. + */ +#define UPDATE_HASH(s, h, c) (h = (((h) << s->hash_shift) ^ (c)) & s->hash_mask) + +/* =========================================================================== + * Insert string str in the dictionary and set match_head to the previous head + * of the hash chain (the most recent string with same hash key). Return + * the previous length of the hash chain. + * If this file is compiled with -DFASTEST, the compression level is forced + * to 1, and no hash chains are maintained. + * IN assertion: all calls to INSERT_STRING are made with consecutive input + * characters and the first MIN_MATCH bytes of str are valid (except for + * the last MIN_MATCH-1 bytes of the input file). + */ +local INLINE Pos insert_string_c(deflate_state* const s, const Pos str) { + Pos ret; + + UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH - 1)]); +#ifdef FASTEST + ret = s->head[s->ins_h]; +#else + ret = s->prev[str & s->w_mask] = s->head[s->ins_h]; +#endif + s->head[s->ins_h] = str; + + return ret; +} + +local INLINE Pos insert_string(deflate_state* const s, const Pos str) { +/* insert_string_simd string dictionary insertion: this SIMD symbol hashing + * significantly improves data compression speed. + * + * Note: the generated compressed output is a valid DEFLATE stream but will + * differ from vanilla zlib output ... + */ +#if defined(CHROMIUM_ZLIB_NO_CASTAGNOLI) +/* ... so this build-time option can used to disable the SIMD symbol hasher + * if matching vanilla zlib DEFLATE output is required. + */ (;) /* FALLTHOUGH */ +#elif defined(TARGET_CPU_WITH_CRC) && defined(CRC32_SIMD_SSE42_PCLMUL) + if (x86_cpu_enable_simd) + return insert_string_simd(s, str); +#elif defined(TARGET_CPU_WITH_CRC) && defined(CRC32_ARMV8_CRC32) + if (arm_cpu_enable_crc32) + return insert_string_simd(s, str); +#endif + return insert_string_c(s, str); +} diff --git a/third_party/zlib/contrib/optimizations/slide_hash_neon.h b/third_party/zlib/contrib/optimizations/slide_hash_neon.h new file mode 100644 index 0000000000..26995d70f7 --- /dev/null +++ b/third_party/zlib/contrib/optimizations/slide_hash_neon.h @@ -0,0 +1,65 @@ +/* Copyright 2018 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ +#ifndef __SLIDE_HASH__NEON__ +#define __SLIDE_HASH__NEON__ + +#include "deflate.h" +#include + +inline static void ZLIB_INTERNAL neon_slide_hash_update(Posf *hash, + const uInt hash_size, + const ush w_size) +{ + /* NEON 'Q' registers allow to store 128 bits, so we can load 8x16-bits + * values. For further details, check: + * ARM DHT 0002A, section 1.3.2 NEON Registers. + */ + const size_t chunk = sizeof(uint16x8_t) / sizeof(uint16_t); + /* Unrolling the operation yielded a compression performance boost in both + * ARMv7 (from 11.7% to 13.4%) and ARMv8 (from 3.7% to 7.5%) for HTML4 + * content. For full benchmarking data, check: http://crbug.com/863257. + */ + const size_t stride = 2*chunk; + const uint16x8_t v = vdupq_n_u16(w_size); + + for (Posf *end = hash + hash_size; hash != end; hash += stride) { + uint16x8_t m_low = vld1q_u16(hash); + uint16x8_t m_high = vld1q_u16(hash + chunk); + + /* The first 'q' in vqsubq_u16 makes these subtracts saturate to zero, + * replacing the ternary operator expression in the original code: + * (m >= wsize ? m - wsize : NIL). + */ + m_low = vqsubq_u16(m_low, v); + m_high = vqsubq_u16(m_high, v); + + vst1q_u16(hash, m_low); + vst1q_u16(hash + chunk, m_high); + } +} + + +inline static void ZLIB_INTERNAL neon_slide_hash(Posf *head, Posf *prev, + const unsigned short w_size, + const uInt hash_size) +{ + /* + * SIMD implementation for hash table rebase assumes: + * 1. hash chain offset (Pos) is 2 bytes. + * 2. hash table size is multiple of 32 bytes. + * #1 should be true as Pos is defined as "ush" + * #2 should be true as hash_bits are greater than 7 + */ + const size_t size = hash_size * sizeof(head[0]); + Assert(sizeof(Pos) == 2, "Wrong Pos size."); + Assert((size % sizeof(uint16x8_t) * 2) == 0, "Hash table size error."); + + neon_slide_hash_update(head, hash_size, w_size); +#ifndef FASTEST + neon_slide_hash_update(prev, w_size, w_size); +#endif +} + +#endif diff --git a/third_party/zlib/contrib/tests/fuzzers/BUILD.gn b/third_party/zlib/contrib/tests/fuzzers/BUILD.gn new file mode 100644 index 0000000000..34c3b43d1f --- /dev/null +++ b/third_party/zlib/contrib/tests/fuzzers/BUILD.gn @@ -0,0 +1,29 @@ +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//testing/libfuzzer/fuzzer_test.gni") + +# root BUILD depends on this target. Needed for package discovery +group("fuzzers") { +} + +fuzzer_test("zlib_uncompress_fuzzer") { + sources = [ "uncompress_fuzzer.cc" ] + deps = [ "../../../:zlib" ] +} + +fuzzer_test("zlib_inflate_fuzzer") { + sources = [ "inflate_fuzzer.cc" ] + deps = [ "../../../:zlib" ] +} + +fuzzer_test("zlib_deflate_set_dictionary_fuzzer") { + sources = [ "deflate_set_dictionary_fuzzer.cc" ] + deps = [ "../../../:zlib" ] +} + +fuzzer_test("zlib_deflate_fuzzer") { + sources = [ "deflate_fuzzer.cc" ] + deps = [ "../../../:zlib" ] +} diff --git a/third_party/zlib/contrib/tests/fuzzers/deflate_fuzzer.cc b/third_party/zlib/contrib/tests/fuzzers/deflate_fuzzer.cc new file mode 100644 index 0000000000..6098ff162c --- /dev/null +++ b/third_party/zlib/contrib/tests/fuzzers/deflate_fuzzer.cc @@ -0,0 +1,47 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include +#include +#include +#include +#include + +#include "third_party/zlib/zlib.h" + +static Bytef buffer[256 * 1024] = {0}; + +// Entry point for LibFuzzer. +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + // zlib's deflate requires non-zero input sizes + if (!size) + return 0; + + // We need to strip the 'const' for zlib. + std::vector input_buffer{data, data+size}; + + uLongf buffer_length = static_cast(sizeof(buffer)); + + z_stream stream; + stream.next_in = input_buffer.data(); + stream.avail_in = size; + stream.total_in = size; + stream.next_out = buffer; + stream.avail_out = buffer_length; + stream.total_out = buffer_length; + stream.zalloc = Z_NULL; + stream.zfree = Z_NULL; + + if (Z_OK != deflateInit(&stream, Z_DEFAULT_COMPRESSION)) { + deflateEnd(&stream); + assert(false); + } + + auto deflate_result = deflate(&stream, Z_NO_FLUSH); + deflateEnd(&stream); + if (Z_OK != deflate_result) + assert(false); + + return 0; +} diff --git a/third_party/zlib/contrib/tests/fuzzers/deflate_set_dictionary_fuzzer.cc b/third_party/zlib/contrib/tests/fuzzers/deflate_set_dictionary_fuzzer.cc new file mode 100644 index 0000000000..febbfcb86f --- /dev/null +++ b/third_party/zlib/contrib/tests/fuzzers/deflate_set_dictionary_fuzzer.cc @@ -0,0 +1,43 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include +#include +#include +#include + +#include "third_party/zlib/zlib.h" + +static Bytef buffer[256 * 1024] = {0}; + +// Entry point for LibFuzzer. +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + // We need to strip the 'const' for zlib. + std::vector input_buffer{data, data + size}; + + uLongf buffer_length = static_cast(sizeof(buffer)); + + z_stream stream; + stream.next_in = input_buffer.data(); + stream.avail_in = size; + stream.total_in = size; + stream.next_out = buffer; + stream.avail_out = buffer_length; + stream.total_out = buffer_length; + stream.zalloc = Z_NULL; + stream.zfree = Z_NULL; + + if (Z_OK != deflateInit(&stream, Z_DEFAULT_COMPRESSION)) { + deflateEnd(&stream); + assert(false); + } + + auto deflate_set_dictionary_result = + deflateSetDictionary(&stream, data, size); + deflateEnd(&stream); + if (Z_OK != deflate_set_dictionary_result) + assert(false); + + return 0; +} diff --git a/third_party/zlib/contrib/tests/fuzzers/inflate_fuzzer.cc b/third_party/zlib/contrib/tests/fuzzers/inflate_fuzzer.cc new file mode 100644 index 0000000000..44f9c72c18 --- /dev/null +++ b/third_party/zlib/contrib/tests/fuzzers/inflate_fuzzer.cc @@ -0,0 +1,41 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include +#include +#include +#include +#include + +#include "third_party/zlib/zlib.h" + +static Bytef buffer[256 * 1024] = {0}; + +// Entry point for LibFuzzer. +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + // We need to strip the 'const' for zlib + std::vector input_buffer{data, data+size}; + + uLongf buffer_length = static_cast(sizeof(buffer)); + + z_stream stream; + stream.next_in = input_buffer.data(); + stream.avail_in = size; + stream.total_in = size; + stream.next_out = buffer; + stream.avail_out = buffer_length; + stream.total_out = buffer_length; + stream.zalloc = Z_NULL; + stream.zfree = Z_NULL; + + if (Z_OK != inflateInit(&stream)) { + inflateEnd(&stream); + assert(false); + } + + inflate(&stream, Z_NO_FLUSH); + inflateEnd(&stream); + + return 0; +} diff --git a/third_party/zlib/contrib/tests/fuzzers/uncompress_fuzzer.cc b/third_party/zlib/contrib/tests/fuzzers/uncompress_fuzzer.cc new file mode 100644 index 0000000000..bca5244e8b --- /dev/null +++ b/third_party/zlib/contrib/tests/fuzzers/uncompress_fuzzer.cc @@ -0,0 +1,21 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include +#include +#include + +#include "third_party/zlib/zlib.h" + +static Bytef buffer[256 * 1024] = {0}; + +// Entry point for LibFuzzer. +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + uLongf buffer_length = static_cast(sizeof(buffer)); + if (Z_OK != + uncompress(buffer, &buffer_length, data, static_cast(size))) { + return 0; + } + return 0; +} diff --git a/third_party/zlib/contrib/tests/infcover.cc b/third_party/zlib/contrib/tests/infcover.cc new file mode 100644 index 0000000000..c5300a5211 --- /dev/null +++ b/third_party/zlib/contrib/tests/infcover.cc @@ -0,0 +1,684 @@ +/* infcover.c -- test zlib's inflate routines with full code coverage + * Copyright (C) 2011, 2016 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* to use, do: ./configure --cover && make cover */ +// clang-format off +#include "infcover.h" +#include +#include +#include + +#include "zlib.h" + +/* get definition of internal structure so we can mess with it (see pull()), + and so we can call inflate_trees() (see cover5()) */ +#define ZLIB_INTERNAL +#include "inftrees.h" +#include "inflate.h" + +/* XXX: use C++ streams instead of printf/fputs/etc due to portability + * as type sizes can vary between platforms. + */ +#include +#define local static + +/* XXX: hacking C assert and plugging into GTest. */ +#include "gtest.h" +#if defined(assert) +#undef assert +#define assert EXPECT_TRUE +#endif + +/* XXX: handle what is a reserved word in C++. */ +#define try try_f + +/* -- memory tracking routines -- */ + +/* + These memory tracking routines are provided to zlib and track all of zlib's + allocations and deallocations, check for LIFO operations, keep a current + and high water mark of total bytes requested, optionally set a limit on the + total memory that can be allocated, and when done check for memory leaks. + + They are used as follows: + + z_stream strm; + mem_setup(&strm) initializes the memory tracking and sets the + zalloc, zfree, and opaque members of strm to use + memory tracking for all zlib operations on strm + mem_limit(&strm, limit) sets a limit on the total bytes requested -- a + request that exceeds this limit will result in an + allocation failure (returns NULL) -- setting the + limit to zero means no limit, which is the default + after mem_setup() + mem_used(&strm, "msg") prints to stderr "msg" and the total bytes used + mem_high(&strm, "msg") prints to stderr "msg" and the high water mark + mem_done(&strm, "msg") ends memory tracking, releases all allocations + for the tracking as well as leaked zlib blocks, if + any. If there was anything unusual, such as leaked + blocks, non-FIFO frees, or frees of addresses not + allocated, then "msg" and information about the + problem is printed to stderr. If everything is + normal, nothing is printed. mem_done resets the + strm members to Z_NULL to use the default memory + allocation routines on the next zlib initialization + using strm. + */ + +/* these items are strung together in a linked list, one for each allocation */ +struct mem_item { + void *ptr; /* pointer to allocated memory */ + size_t size; /* requested size of allocation */ + struct mem_item *next; /* pointer to next item in list, or NULL */ +}; + +/* this structure is at the root of the linked list, and tracks statistics */ +struct mem_zone { + struct mem_item *first; /* pointer to first item in list, or NULL */ + size_t total, highwater; /* total allocations, and largest total */ + size_t limit; /* memory allocation limit, or 0 if no limit */ + int notlifo, rogue; /* counts of non-LIFO frees and rogue frees */ +}; + +/* memory allocation routine to pass to zlib */ +local void *mem_alloc(void *mem, unsigned count, unsigned size) +{ + void *ptr; + struct mem_item *item; + struct mem_zone *zone = static_cast(mem); + size_t len = count * (size_t)size; + + /* induced allocation failure */ + if (zone == NULL || (zone->limit && zone->total + len > zone->limit)) + return NULL; + + /* perform allocation using the standard library, fill memory with a + non-zero value to make sure that the code isn't depending on zeros */ + ptr = malloc(len); + if (ptr == NULL) + return NULL; + memset(ptr, 0xa5, len); + + /* create a new item for the list */ + item = static_cast(malloc(sizeof(struct mem_item))); + if (item == NULL) { + free(ptr); + return NULL; + } + item->ptr = ptr; + item->size = len; + + /* insert item at the beginning of the list */ + item->next = zone->first; + zone->first = item; + + /* update the statistics */ + zone->total += item->size; + if (zone->total > zone->highwater) + zone->highwater = zone->total; + + /* return the allocated memory */ + return ptr; +} + +/* memory free routine to pass to zlib */ +local void mem_free(void *mem, void *ptr) +{ + struct mem_item *item, *next; + struct mem_zone *zone = static_cast(mem); + + /* if no zone, just do a free */ + if (zone == NULL) { + free(ptr); + return; + } + + /* point next to the item that matches ptr, or NULL if not found -- remove + the item from the linked list if found */ + next = zone->first; + if (next) { + if (next->ptr == ptr) + zone->first = next->next; /* first one is it, remove from list */ + else { + do { /* search the linked list */ + item = next; + next = item->next; + } while (next != NULL && next->ptr != ptr); + if (next) { /* if found, remove from linked list */ + item->next = next->next; + zone->notlifo++; /* not a LIFO free */ + } + + } + } + + /* if found, update the statistics and free the item */ + if (next) { + zone->total -= next->size; + free(next); + } + + /* if not found, update the rogue count */ + else + zone->rogue++; + + /* in any case, do the requested free with the standard library function */ + free(ptr); +} + +/* set up a controlled memory allocation space for monitoring, set the stream + parameters to the controlled routines, with opaque pointing to the space */ +local void mem_setup(z_stream *strm) +{ + struct mem_zone *zone; + + zone = static_cast(malloc(sizeof(struct mem_zone))); + assert(zone != NULL); + zone->first = NULL; + zone->total = 0; + zone->highwater = 0; + zone->limit = 0; + zone->notlifo = 0; + zone->rogue = 0; + strm->opaque = zone; + strm->zalloc = mem_alloc; + strm->zfree = mem_free; +} + +/* set a limit on the total memory allocation, or 0 to remove the limit */ +local void mem_limit(z_stream *strm, size_t limit) +{ + struct mem_zone *zone = static_cast(strm->opaque); + + zone->limit = limit; +} + +/* show the current total requested allocations in bytes */ +local void mem_used(z_stream *strm, const char *prefix) +{ + struct mem_zone *zone = static_cast(strm->opaque); + + std::cout << prefix << ": " << zone->total << " allocated" << std::endl; +} + +/* show the high water allocation in bytes */ +local void mem_high(z_stream *strm, const char *prefix) +{ + struct mem_zone *zone = static_cast(strm->opaque); + + std::cout << prefix << ": " << zone->highwater << " high water mark" << std::endl; +} + +/* release the memory allocation zone -- if there are any surprises, notify */ +local void mem_done(z_stream *strm, const char *prefix) +{ + int count = 0; + struct mem_item *item, *next; + struct mem_zone *zone = static_cast(strm->opaque); + + /* show high water mark */ + mem_high(strm, prefix); + + /* free leftover allocations and item structures, if any */ + item = zone->first; + while (item != NULL) { + free(item->ptr); + next = item->next; + free(item); + item = next; + count++; + } + + /* issue alerts about anything unexpected */ + if (count || zone->total) + std::cout << "** " << prefix << ": " + << zone->total << " bytes in " + << count << " blocks not freed" + << std::endl; + + if (zone->notlifo) + std::cout << "** " << prefix << ": " + << zone->notlifo << " frees not LIFO" + << std::endl; + + if (zone->rogue) + std::cout << "** " << prefix << ": " + << zone->rogue << " frees not recognized" + << std::endl; + + /* free the zone and delete from the stream */ + free(zone); + strm->opaque = Z_NULL; + strm->zalloc = Z_NULL; + strm->zfree = Z_NULL; +} + +/* -- inflate test routines -- */ + +/* Decode a hexadecimal string, set *len to length, in[] to the bytes. This + decodes liberally, in that hex digits can be adjacent, in which case two in + a row writes a byte. Or they can be delimited by any non-hex character, + where the delimiters are ignored except when a single hex digit is followed + by a delimiter, where that single digit writes a byte. The returned data is + allocated and must eventually be freed. NULL is returned if out of memory. + If the length is not needed, then len can be NULL. */ +local unsigned char *h2b(const char *hex, unsigned *len) +{ + unsigned char *in, *re; + unsigned next, val; + + in = static_cast(malloc((strlen(hex) + 1) >> 1)); + if (in == NULL) + return NULL; + next = 0; + val = 1; + do { + if (*hex >= '0' && *hex <= '9') + val = (val << 4) + *hex - '0'; + else if (*hex >= 'A' && *hex <= 'F') + val = (val << 4) + *hex - 'A' + 10; + else if (*hex >= 'a' && *hex <= 'f') + val = (val << 4) + *hex - 'a' + 10; + else if (val != 1 && val < 32) /* one digit followed by delimiter */ + val += 240; /* make it look like two digits */ + if (val > 255) { /* have two digits */ + in[next++] = val & 0xff; /* save the decoded byte */ + val = 1; /* start over */ + } + } while (*hex++); /* go through the loop with the terminating null */ + if (len != NULL) + *len = next; + re = static_cast(realloc(in, next)); + return re == NULL ? in : re; +} + +/* generic inflate() run, where hex is the hexadecimal input data, what is the + text to include in an error message, step is how much input data to feed + inflate() on each call, or zero to feed it all, win is the window bits + parameter to inflateInit2(), len is the size of the output buffer, and err + is the error code expected from the first inflate() call (the second + inflate() call is expected to return Z_STREAM_END). If win is 47, then + header information is collected with inflateGetHeader(). If a zlib stream + is looking for a dictionary, then an empty dictionary is provided. + inflate() is run until all of the input data is consumed. */ +local void inf(const char *hex, const char *what, unsigned step, int win, unsigned len, + int err) +{ + int ret; + unsigned have; + unsigned char *in, *out; + z_stream strm, copy; + gz_header head; + + mem_setup(&strm); + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit2(&strm, win); + if (ret != Z_OK) { + mem_done(&strm, what); + return; + } + out = static_cast(malloc(len)); assert(out != NULL); + if (win == 47) { + head.extra = out; + head.extra_max = len; + head.name = out; + head.name_max = len; + head.comment = out; + head.comm_max = len; + ret = inflateGetHeader(&strm, &head); assert(ret == Z_OK); + } + in = h2b(hex, &have); assert(in != NULL); + if (step == 0 || step > have) + step = have; + strm.avail_in = step; + have -= step; + strm.next_in = in; + do { + strm.avail_out = len; + strm.next_out = out; + ret = inflate(&strm, Z_NO_FLUSH); assert(err == 9 || ret == err); + if (ret != Z_OK && ret != Z_BUF_ERROR && ret != Z_NEED_DICT) + break; + if (ret == Z_NEED_DICT) { + ret = inflateSetDictionary(&strm, in, 1); + assert(ret == Z_DATA_ERROR); + mem_limit(&strm, 1); + ret = inflateSetDictionary(&strm, out, 0); + assert(ret == Z_MEM_ERROR); + mem_limit(&strm, 0); + ((struct inflate_state *)strm.state)->mode = DICT; + ret = inflateSetDictionary(&strm, out, 0); + assert(ret == Z_OK); + ret = inflate(&strm, Z_NO_FLUSH); assert(ret == Z_BUF_ERROR); + } + ret = inflateCopy(©, &strm); assert(ret == Z_OK); + ret = inflateEnd(©); assert(ret == Z_OK); + err = 9; /* don't care next time around */ + have += strm.avail_in; + strm.avail_in = step > have ? have : step; + have -= strm.avail_in; + } while (strm.avail_in); + free(in); + free(out); + ret = inflateReset2(&strm, -8); assert(ret == Z_OK); + ret = inflateEnd(&strm); assert(ret == Z_OK); + mem_done(&strm, what); +} + +/* cover all of the lines in inflate.c up to inflate() */ +void cover_support(void) +{ + int ret; + z_stream strm; + + mem_setup(&strm); + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit(&strm); assert(ret == Z_OK); + mem_used(&strm, "inflate init"); + ret = inflatePrime(&strm, 5, 31); assert(ret == Z_OK); + ret = inflatePrime(&strm, -1, 0); assert(ret == Z_OK); + ret = inflateSetDictionary(&strm, Z_NULL, 0); + assert(ret == Z_STREAM_ERROR); + ret = inflateEnd(&strm); assert(ret == Z_OK); + mem_done(&strm, "prime"); + + inf("63 0", "force window allocation", 0, -15, 1, Z_OK); + inf("63 18 5", "force window replacement", 0, -8, 259, Z_OK); + inf("63 18 68 30 d0 0 0", "force split window update", 4, -8, 259, Z_OK); + inf("3 0", "use fixed blocks", 0, -15, 1, Z_STREAM_END); + inf("", "bad window size", 0, 1, 0, Z_STREAM_ERROR); + + mem_setup(&strm); + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit_(&strm, ZLIB_VERSION - 1, (int)sizeof(z_stream)); + assert(ret == Z_VERSION_ERROR); + mem_done(&strm, "wrong version"); + + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit(&strm); assert(ret == Z_OK); + ret = inflateEnd(&strm); assert(ret == Z_OK); + std::cout << "inflate built-in memory routines" << std::endl;; +} + +/* cover all inflate() header and trailer cases and code after inflate() */ +void cover_wrap(void) +{ + int ret; + z_stream strm, copy; + unsigned char dict[257]; + + ret = inflate(Z_NULL, 0); assert(ret == Z_STREAM_ERROR); + ret = inflateEnd(Z_NULL); assert(ret == Z_STREAM_ERROR); + ret = inflateCopy(Z_NULL, Z_NULL); assert(ret == Z_STREAM_ERROR); + std::cout << "inflate bad parameters" << std::endl; + + inf("1f 8b 0 0", "bad gzip method", 0, 31, 0, Z_DATA_ERROR); + inf("1f 8b 8 80", "bad gzip flags", 0, 31, 0, Z_DATA_ERROR); + inf("77 85", "bad zlib method", 0, 15, 0, Z_DATA_ERROR); + inf("8 99", "set window size from header", 0, 0, 0, Z_OK); + inf("78 9c", "bad zlib window size", 0, 8, 0, Z_DATA_ERROR); + inf("78 9c 63 0 0 0 1 0 1", "check adler32", 0, 15, 1, Z_STREAM_END); + inf("1f 8b 8 1e 0 0 0 0 0 0 1 0 0 0 0 0 0", "bad header crc", 0, 47, 1, + Z_DATA_ERROR); + inf("1f 8b 8 2 0 0 0 0 0 0 1d 26 3 0 0 0 0 0 0 0 0 0", "check gzip length", + 0, 47, 0, Z_STREAM_END); + inf("78 90", "bad zlib header check", 0, 47, 0, Z_DATA_ERROR); + inf("8 b8 0 0 0 1", "need dictionary", 0, 8, 0, Z_NEED_DICT); + inf("78 9c 63 0", "compute adler32", 0, 15, 1, Z_OK); + + mem_setup(&strm); + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit2(&strm, -8); + strm.avail_in = 2; + strm.next_in = (Bytef *)"\x63"; + strm.avail_out = 1; + strm.next_out = (Bytef *)&ret; + mem_limit(&strm, 1); + ret = inflate(&strm, Z_NO_FLUSH); assert(ret == Z_MEM_ERROR); + ret = inflate(&strm, Z_NO_FLUSH); assert(ret == Z_MEM_ERROR); + mem_limit(&strm, 0); + memset(dict, 0, 257); + ret = inflateSetDictionary(&strm, dict, 257); + assert(ret == Z_OK); + mem_limit(&strm, (sizeof(struct inflate_state) << 1) + 256); + ret = inflatePrime(&strm, 16, 0); assert(ret == Z_OK); + strm.avail_in = 2; + strm.next_in = (Bytef *)"\x80"; + ret = inflateSync(&strm); assert(ret == Z_DATA_ERROR); + ret = inflate(&strm, Z_NO_FLUSH); assert(ret == Z_STREAM_ERROR); + strm.avail_in = 4; + strm.next_in = (Bytef *)"\0\0\xff\xff"; + ret = inflateSync(&strm); assert(ret == Z_OK); + (void)inflateSyncPoint(&strm); + ret = inflateCopy(©, &strm); assert(ret == Z_MEM_ERROR); + mem_limit(&strm, 0); + ret = inflateUndermine(&strm, 1); assert(ret == Z_DATA_ERROR); + (void)inflateMark(&strm); + ret = inflateEnd(&strm); assert(ret == Z_OK); + mem_done(&strm, "miscellaneous, force memory errors"); +} + +/* input and output functions for inflateBack() */ +local unsigned pull(void *desc, unsigned char **buf) +{ + static unsigned int next = 0; + static unsigned char dat[] = {0x63, 0, 2, 0}; + struct inflate_state *state; + + if (desc == Z_NULL) { + next = 0; + return 0; /* no input (already provided at next_in) */ + } + state = reinterpret_cast(((z_stream *)desc)->state); + if (state != Z_NULL) + state->mode = SYNC; /* force an otherwise impossible situation */ + return next < sizeof(dat) ? (*buf = dat + next++, 1) : 0; +} + +local int push(void *desc, unsigned char *buf, unsigned len) +{ + buf += len; + return desc != Z_NULL; /* force error if desc not null */ +} + +/* cover inflateBack() up to common deflate data cases and after those */ +void cover_back(void) +{ + int ret; + z_stream strm; + unsigned char win[32768]; + + ret = inflateBackInit_(Z_NULL, 0, win, 0, 0); + assert(ret == Z_VERSION_ERROR); + ret = inflateBackInit(Z_NULL, 0, win); assert(ret == Z_STREAM_ERROR); + ret = inflateBack(Z_NULL, Z_NULL, Z_NULL, Z_NULL, Z_NULL); + assert(ret == Z_STREAM_ERROR); + ret = inflateBackEnd(Z_NULL); assert(ret == Z_STREAM_ERROR); + std::cout << "inflateBack bad parameters" << std::endl;; + + mem_setup(&strm); + ret = inflateBackInit(&strm, 15, win); assert(ret == Z_OK); + strm.avail_in = 2; + strm.next_in = (Bytef *)"\x03"; + ret = inflateBack(&strm, pull, Z_NULL, push, Z_NULL); + assert(ret == Z_STREAM_END); + /* force output error */ + strm.avail_in = 3; + strm.next_in = (Bytef *)"\x63\x00"; + ret = inflateBack(&strm, pull, Z_NULL, push, &strm); + assert(ret == Z_BUF_ERROR); + /* force mode error by mucking with state */ + ret = inflateBack(&strm, pull, &strm, push, Z_NULL); + assert(ret == Z_STREAM_ERROR); + ret = inflateBackEnd(&strm); assert(ret == Z_OK); + mem_done(&strm, "inflateBack bad state"); + + ret = inflateBackInit(&strm, 15, win); assert(ret == Z_OK); + ret = inflateBackEnd(&strm); assert(ret == Z_OK); + std::cout << "inflateBack built-in memory routines" << std::endl;; +} + +/* do a raw inflate of data in hexadecimal with both inflate and inflateBack */ +local int try(const char *hex, const char *id, int err) +{ + int ret; + unsigned len, size; + unsigned char *in, *out, *win; + char *prefix; + z_stream strm; + + /* convert to hex */ + in = h2b(hex, &len); + assert(in != NULL); + + /* allocate work areas */ + size = len << 3; + out = static_cast(malloc(size)); + assert(out != NULL); + win = static_cast(malloc(32768)); + assert(win != NULL); + prefix = static_cast(malloc(strlen(id) + 6)); + assert(prefix != NULL); + + /* first with inflate */ + strcpy(prefix, id); + strcat(prefix, "-late"); + mem_setup(&strm); + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit2(&strm, err < 0 ? 47 : -15); + assert(ret == Z_OK); + strm.avail_in = len; + strm.next_in = in; + do { + strm.avail_out = size; + strm.next_out = out; + ret = inflate(&strm, Z_TREES); + assert(ret != Z_STREAM_ERROR && ret != Z_MEM_ERROR); + if (ret == Z_DATA_ERROR || ret == Z_NEED_DICT) + break; + } while (strm.avail_in || strm.avail_out == 0); + if (err) { + assert(ret == Z_DATA_ERROR); + assert(strcmp(id, strm.msg) == 0); + } + inflateEnd(&strm); + mem_done(&strm, prefix); + + /* then with inflateBack */ + if (err >= 0) { + strcpy(prefix, id); + strcat(prefix, "-back"); + mem_setup(&strm); + ret = inflateBackInit(&strm, 15, win); + assert(ret == Z_OK); + strm.avail_in = len; + strm.next_in = in; + ret = inflateBack(&strm, pull, Z_NULL, push, Z_NULL); + assert(ret != Z_STREAM_ERROR); + if (err) { + assert(ret == Z_DATA_ERROR); + assert(strcmp(id, strm.msg) == 0); + } + inflateBackEnd(&strm); + mem_done(&strm, prefix); + } + + /* clean up */ + free(prefix); + free(win); + free(out); + free(in); + return ret; +} + +/* cover deflate data cases in both inflate() and inflateBack() */ +void cover_inflate(void) +{ + try("0 0 0 0 0", "invalid stored block lengths", 1); + try("3 0", "fixed", 0); + try("6", "invalid block type", 1); + try("1 1 0 fe ff 0", "stored", 0); + try("fc 0 0", "too many length or distance symbols", 1); + try("4 0 fe ff", "invalid code lengths set", 1); + try("4 0 24 49 0", "invalid bit length repeat", 1); + try("4 0 24 e9 ff ff", "invalid bit length repeat", 1); + try("4 0 24 e9 ff 6d", "invalid code -- missing end-of-block", 1); + try("4 80 49 92 24 49 92 24 71 ff ff 93 11 0", + "invalid literal/lengths set", 1); + try("4 80 49 92 24 49 92 24 f b4 ff ff c3 84", "invalid distances set", 1); + try("4 c0 81 8 0 0 0 0 20 7f eb b 0 0", "invalid literal/length code", 1); + try("2 7e ff ff", "invalid distance code", 1); + try("c c0 81 0 0 0 0 0 90 ff 6b 4 0", "invalid distance too far back", 1); + + /* also trailer mismatch just in inflate() */ + try("1f 8b 8 0 0 0 0 0 0 0 3 0 0 0 0 1", "incorrect data check", -1); + try("1f 8b 8 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 1", + "incorrect length check", -1); + try("5 c0 21 d 0 0 0 80 b0 fe 6d 2f 91 6c", "pull 17", 0); + try("5 e0 81 91 24 cb b2 2c 49 e2 f 2e 8b 9a 47 56 9f fb fe ec d2 ff 1f", + "long code", 0); + try("ed c0 1 1 0 0 0 40 20 ff 57 1b 42 2c 4f", "length extra", 0); + try("ed cf c1 b1 2c 47 10 c4 30 fa 6f 35 1d 1 82 59 3d fb be 2e 2a fc f c", + "long distance and extra", 0); + try("ed c0 81 0 0 0 0 80 a0 fd a9 17 a9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 " + "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 6", "window end", 0); + inf("2 8 20 80 0 3 0", "inflate_fast TYPE return", 0, -15, 258, + Z_STREAM_END); + inf("63 18 5 40 c 0", "window wrap", 3, -8, 300, Z_OK); +} + +/* XXX(cavalcantii): fix linking error due inflate_table. */ +/* cover remaining lines in inftrees.c */ +/* void cover_trees(void) */ +/* { */ +/* int ret; */ +/* unsigned bits; */ +/* unsigned short lens[16], work[16]; */ +/* code *next, table[ENOUGH_DISTS]; */ + +/* /\* we need to call inflate_table() directly in order to manifest not- */ +/* enough errors, since zlib insures that enough is always enough *\/ */ +/* for (bits = 0; bits < 15; bits++) */ +/* lens[bits] = (unsigned short)(bits + 1); */ +/* lens[15] = 15; */ +/* next = table; */ +/* bits = 15; */ +/* ret = inflate_table(DISTS, lens, 16, &next, &bits, work); */ +/* assert(ret == 1); */ +/* next = table; */ +/* bits = 1; */ +/* ret = inflate_table(DISTS, lens, 16, &next, &bits, work); */ +/* assert(ret == 1); */ +/* fputs("inflate_table not enough errors\n", stderr); */ +/* } */ + +/* cover remaining inffast.c decoding and window copying */ +void cover_fast(void) +{ + inf("e5 e0 81 ad 6d cb b2 2c c9 01 1e 59 63 ae 7d ee fb 4d fd b5 35 41 68" + " ff 7f 0f 0 0 0", "fast length extra bits", 0, -8, 258, Z_DATA_ERROR); + inf("25 fd 81 b5 6d 59 b6 6a 49 ea af 35 6 34 eb 8c b9 f6 b9 1e ef 67 49" + " 50 fe ff ff 3f 0 0", "fast distance extra bits", 0, -8, 258, + Z_DATA_ERROR); + inf("3 7e 0 0 0 0 0", "fast invalid distance code", 0, -8, 258, + Z_DATA_ERROR); + inf("1b 7 0 0 0 0 0", "fast invalid literal/length code", 0, -8, 258, + Z_DATA_ERROR); + inf("d c7 1 ae eb 38 c 4 41 a0 87 72 de df fb 1f b8 36 b1 38 5d ff ff 0", + "fast 2nd level codes and too far back", 0, -8, 258, Z_DATA_ERROR); + inf("63 18 5 8c 10 8 0 0 0 0", "very common case", 0, -8, 259, Z_OK); + inf("63 60 60 18 c9 0 8 18 18 18 26 c0 28 0 29 0 0 0", + "contiguous and wrap around window", 6, -8, 259, Z_OK); + inf("63 0 3 0 0 0 0 0", "copy direct from output", 0, -8, 259, + Z_STREAM_END); +} + +// clang-format on diff --git a/third_party/zlib/contrib/tests/infcover.h b/third_party/zlib/contrib/tests/infcover.h new file mode 100644 index 0000000000..b3e112f915 --- /dev/null +++ b/third_party/zlib/contrib/tests/infcover.h @@ -0,0 +1,11 @@ +#ifndef __INF_COVER_H__ +#define __INF_COVER_H__ + +void cover_support(void); +void cover_wrap(void); +void cover_back(void); +void cover_inflate(void); +void cover_trees(void); +void cover_fast(void); + +#endif diff --git a/third_party/zlib/contrib/tests/utils_unittest.cc b/third_party/zlib/contrib/tests/utils_unittest.cc new file mode 100644 index 0000000000..ae41f7ba21 --- /dev/null +++ b/third_party/zlib/contrib/tests/utils_unittest.cc @@ -0,0 +1,91 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the Chromium source repository LICENSE file. + +#include "infcover.h" + +#include +#include + +#include "compression_utils_portable.h" +#include "gtest.h" +#include "zlib.h" + +void TestPayloads(size_t input_size, zlib_internal::WrapperType type) { + std::vector input; + input.reserve(input_size); + for (size_t i = 1; i <= input_size; ++i) + input.push_back(i & 0xff); + + // If it is big enough for GZIP, will work for other wrappers. + std::vector compressed( + zlib_internal::GzipExpectedCompressedSize(input.size())); + std::vector decompressed(input.size()); + + // Libcores's java/util/zip/Deflater default settings: ZLIB, + // DEFAULT_COMPRESSION and DEFAULT_STRATEGY. + unsigned long compressed_size = static_cast(compressed.size()); + int result = zlib_internal::CompressHelper( + type, compressed.data(), &compressed_size, input.data(), input.size(), + Z_DEFAULT_COMPRESSION, nullptr, nullptr); + ASSERT_EQ(result, Z_OK); + + unsigned long decompressed_size = + static_cast(decompressed.size()); + result = zlib_internal::UncompressHelper(type, decompressed.data(), + &decompressed_size, + compressed.data(), compressed_size); + ASSERT_EQ(result, Z_OK); + EXPECT_EQ(input, decompressed); +} + +TEST(ZlibTest, ZlibWrapper) { + // Minimal ZLIB wrapped short stream size is about 8 bytes. + for (size_t i = 1; i < 1024; ++i) + TestPayloads(i, zlib_internal::WrapperType::ZLIB); +} + +TEST(ZlibTest, GzipWrapper) { + // GZIP should be 12 bytes bigger than ZLIB wrapper. + for (size_t i = 1; i < 1024; ++i) + TestPayloads(i, zlib_internal::WrapperType::GZIP); +} + +TEST(ZlibTest, RawWrapper) { + // RAW has no wrapper (V8 Blobs is a known user), size + // should be payload_size + 2 for short payloads. + for (size_t i = 1; i < 1024; ++i) + TestPayloads(i, zlib_internal::WrapperType::ZRAW); +} + +TEST(ZlibTest, InflateCover) { + cover_support(); + cover_wrap(); + cover_back(); + cover_inflate(); + // TODO(cavalcantii): enable this last test. + // cover_trees(); + cover_fast(); +} + +TEST(ZlibTest, DeflateStored) { + const int no_compression = 0; + const zlib_internal::WrapperType type = zlib_internal::WrapperType::GZIP; + std::vector input(1 << 10, 42); + std::vector compressed( + zlib_internal::GzipExpectedCompressedSize(input.size())); + std::vector decompressed(input.size()); + unsigned long compressed_size = static_cast(compressed.size()); + int result = zlib_internal::CompressHelper( + type, compressed.data(), &compressed_size, input.data(), input.size(), + no_compression, nullptr, nullptr); + ASSERT_EQ(result, Z_OK); + + unsigned long decompressed_size = + static_cast(decompressed.size()); + result = zlib_internal::UncompressHelper(type, decompressed.data(), + &decompressed_size, + compressed.data(), compressed_size); + ASSERT_EQ(result, Z_OK); + EXPECT_EQ(input, decompressed); +} diff --git a/third_party/zlib/cpu_features.c b/third_party/zlib/cpu_features.c new file mode 100644 index 0000000000..70f01beebe --- /dev/null +++ b/third_party/zlib/cpu_features.c @@ -0,0 +1,167 @@ +/* cpu_features.c -- Processor features detection. + * + * Copyright 2018 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#include "cpu_features.h" +#include "zutil.h" + +#include +#if defined(_MSC_VER) +#include +#elif defined(ADLER32_SIMD_SSSE3) +#include +#endif + +/* TODO(cavalcantii): remove checks for x86_flags on deflate. + */ +#if defined(ARMV8_OS_MACOS) +/* crc32 is a baseline feature in ARMv8.1-A, and macOS running on arm64 is new + * enough that this can be assumed without runtime detection. */ +int ZLIB_INTERNAL arm_cpu_enable_crc32 = 1; +#else +int ZLIB_INTERNAL arm_cpu_enable_crc32 = 0; +#endif +int ZLIB_INTERNAL arm_cpu_enable_pmull = 0; +int ZLIB_INTERNAL x86_cpu_enable_sse2 = 0; +int ZLIB_INTERNAL x86_cpu_enable_ssse3 = 0; +int ZLIB_INTERNAL x86_cpu_enable_simd = 0; + +#ifndef CPU_NO_SIMD + +#if defined(ARMV8_OS_ANDROID) || defined(ARMV8_OS_LINUX) || defined(ARMV8_OS_FUCHSIA) +#include +#endif + +#if defined(ARMV8_OS_ANDROID) +#include +#elif defined(ARMV8_OS_LINUX) +#include +#include +#elif defined(ARMV8_OS_FUCHSIA) +#include +#include +#include +#elif defined(ARMV8_OS_WINDOWS) || defined(X86_WINDOWS) +#include +#elif !defined(_MSC_VER) +#include +#else +#error cpu_features.c CPU feature detection in not defined for your platform +#endif + +#if !defined(CPU_NO_SIMD) && !defined(ARMV8_OS_MACOS) && !defined(ARM_OS_IOS) +static void _cpu_check_features(void); +#endif + +#if defined(ARMV8_OS_ANDROID) || defined(ARMV8_OS_LINUX) || defined(ARMV8_OS_MACOS) || defined(ARMV8_OS_FUCHSIA) || defined(X86_NOT_WINDOWS) +#if !defined(ARMV8_OS_MACOS) +// _cpu_check_features() doesn't need to do anything on mac/arm since all +// features are known at build time, so don't call it. +// Do provide cpu_check_features() (with a no-op implementation) so that we +// don't have to make all callers of it check for mac/arm. +static pthread_once_t cpu_check_inited_once = PTHREAD_ONCE_INIT; +#endif +void ZLIB_INTERNAL cpu_check_features(void) +{ +#if !defined(ARMV8_OS_MACOS) + pthread_once(&cpu_check_inited_once, _cpu_check_features); +#endif +} +#elif defined(ARMV8_OS_WINDOWS) || defined(X86_WINDOWS) +static INIT_ONCE cpu_check_inited_once = INIT_ONCE_STATIC_INIT; +static BOOL CALLBACK _cpu_check_features_forwarder(PINIT_ONCE once, PVOID param, PVOID* context) +{ + _cpu_check_features(); + return TRUE; +} +void ZLIB_INTERNAL cpu_check_features(void) +{ + InitOnceExecuteOnce(&cpu_check_inited_once, _cpu_check_features_forwarder, + NULL, NULL); +} +#endif + +#if (defined(__ARM_NEON__) || defined(__ARM_NEON)) +/* + * iOS@ARM is a special case where we always have NEON but don't check + * for crypto extensions. + */ +#if !defined(ARMV8_OS_MACOS) && !defined(ARM_OS_IOS) +/* + * See http://bit.ly/2CcoEsr for run-time detection of ARM features and also + * crbug.com/931275 for android_getCpuFeatures() use in the Android sandbox. + */ +static void _cpu_check_features(void) +{ +#if defined(ARMV8_OS_ANDROID) && defined(__aarch64__) + uint64_t features = android_getCpuFeatures(); + arm_cpu_enable_crc32 = !!(features & ANDROID_CPU_ARM64_FEATURE_CRC32); + arm_cpu_enable_pmull = !!(features & ANDROID_CPU_ARM64_FEATURE_PMULL); +#elif defined(ARMV8_OS_ANDROID) /* aarch32 */ + uint64_t features = android_getCpuFeatures(); + arm_cpu_enable_crc32 = !!(features & ANDROID_CPU_ARM_FEATURE_CRC32); + arm_cpu_enable_pmull = !!(features & ANDROID_CPU_ARM_FEATURE_PMULL); +#elif defined(ARMV8_OS_LINUX) && defined(__aarch64__) + unsigned long features = getauxval(AT_HWCAP); + arm_cpu_enable_crc32 = !!(features & HWCAP_CRC32); + arm_cpu_enable_pmull = !!(features & HWCAP_PMULL); +#elif defined(ARMV8_OS_LINUX) && (defined(__ARM_NEON) || defined(__ARM_NEON__)) + /* Query HWCAP2 for ARMV8-A SoCs running in aarch32 mode */ + unsigned long features = getauxval(AT_HWCAP2); + arm_cpu_enable_crc32 = !!(features & HWCAP2_CRC32); + arm_cpu_enable_pmull = !!(features & HWCAP2_PMULL); +#elif defined(ARMV8_OS_FUCHSIA) + uint32_t features; + zx_status_t rc = zx_system_get_features(ZX_FEATURE_KIND_CPU, &features); + if (rc != ZX_OK || (features & ZX_ARM64_FEATURE_ISA_ASIMD) == 0) + return; /* Report nothing if ASIMD(NEON) is missing */ + arm_cpu_enable_crc32 = !!(features & ZX_ARM64_FEATURE_ISA_CRC32); + arm_cpu_enable_pmull = !!(features & ZX_ARM64_FEATURE_ISA_PMULL); +#elif defined(ARMV8_OS_WINDOWS) + arm_cpu_enable_crc32 = IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE); + arm_cpu_enable_pmull = IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE); +#endif +} +#endif +#elif defined(X86_NOT_WINDOWS) || defined(X86_WINDOWS) +/* + * iOS@x86 (i.e. emulator) is another special case where we disable + * SIMD optimizations. + */ +#ifndef CPU_NO_SIMD +/* On x86 we simply use a instruction to check the CPU features. + * (i.e. CPUID). + */ +static void _cpu_check_features(void) +{ + int x86_cpu_has_sse2; + int x86_cpu_has_ssse3; + int x86_cpu_has_sse42; + int x86_cpu_has_pclmulqdq; + int abcd[4]; + +#ifdef _MSC_VER + __cpuid(abcd, 1); +#else + __cpuid(1, abcd[0], abcd[1], abcd[2], abcd[3]); +#endif + + x86_cpu_has_sse2 = abcd[3] & 0x4000000; + x86_cpu_has_ssse3 = abcd[2] & 0x000200; + x86_cpu_has_sse42 = abcd[2] & 0x100000; + x86_cpu_has_pclmulqdq = abcd[2] & 0x2; + + x86_cpu_enable_sse2 = x86_cpu_has_sse2; + + x86_cpu_enable_ssse3 = x86_cpu_has_ssse3; + + x86_cpu_enable_simd = x86_cpu_has_sse2 && + x86_cpu_has_sse42 && + x86_cpu_has_pclmulqdq; +} +#endif +#endif +#endif diff --git a/third_party/zlib/cpu_features.h b/third_party/zlib/cpu_features.h new file mode 100644 index 0000000000..c7b15c5597 --- /dev/null +++ b/third_party/zlib/cpu_features.h @@ -0,0 +1,18 @@ +/* cpu_features.h -- Processor features detection. + * + * Copyright 2018 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#include "zlib.h" + +/* TODO(cavalcantii): remove checks for x86_flags on deflate. + */ +extern int arm_cpu_enable_crc32; +extern int arm_cpu_enable_pmull; +extern int x86_cpu_enable_sse2; +extern int x86_cpu_enable_ssse3; +extern int x86_cpu_enable_simd; + +void cpu_check_features(void); diff --git a/third_party/zlib/crc32.c b/third_party/zlib/crc32.c index 75f2290d83..d4c3248d98 100644 --- a/third_party/zlib/crc32.c +++ b/third_party/zlib/crc32.c @@ -1,5 +1,5 @@ /* crc32.c -- compute the CRC-32 of a data stream - * Copyright (C) 1995-2006, 2010 Mark Adler + * Copyright (C) 1995-2006, 2010, 2011, 2012, 2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h * * Thanks to Rodney Brown for his contribution of faster @@ -17,6 +17,8 @@ of the crc tables. Therefore, if you #define DYNAMIC_CRC_TABLE, you should first call get_crc_table() to initialize the tables before allowing more than one thread to use crc32(). + + DYNAMIC_CRC_TABLE and MAKECRCH can be #defined to write out crc32.h. */ #ifdef MAKECRCH @@ -27,40 +29,22 @@ #endif /* MAKECRCH */ #include "deflate.h" -#include "x86.h" +#include "cpu_features.h" #include "zutil.h" /* for STDC and FAR definitions */ -#define local static - -/* Find a four-byte integer type for crc32_little() and crc32_big(). */ -#ifndef NOBYFOUR -# ifdef STDC /* need ANSI C limits.h to determine sizes */ -# include -# define BYFOUR -# if (UINT_MAX == 0xffffffffUL) - typedef unsigned int u4; -# else -# if (ULONG_MAX == 0xffffffffUL) - typedef unsigned long u4; -# else -# if (USHRT_MAX == 0xffffffffUL) - typedef unsigned short u4; -# else -# undef BYFOUR /* can't find a four-byte integer type! */ -# endif -# endif -# endif -# endif /* STDC */ -#endif /* !NOBYFOUR */ +#if defined(CRC32_SIMD_SSE42_PCLMUL) || defined(CRC32_ARMV8_CRC32) +#include "crc32_simd.h" +#endif /* Definitions for doing the crc four data bytes at a time. */ +#if !defined(NOBYFOUR) && defined(Z_U4) +# define BYFOUR +#endif #ifdef BYFOUR -# define REV(w) ((((w)>>24)&0xff)+(((w)>>8)&0xff00)+ \ - (((w)&0xff00)<<8)+(((w)&0xff)<<24)) local unsigned long crc32_little OF((unsigned long, - const unsigned char FAR *, unsigned)); + const unsigned char FAR *, z_size_t)); local unsigned long crc32_big OF((unsigned long, - const unsigned char FAR *, unsigned)); + const unsigned char FAR *, z_size_t)); # define TBLS 8 #else # define TBLS 1 @@ -70,16 +54,16 @@ local unsigned long gf2_matrix_times OF((unsigned long *mat, unsigned long vec)); local void gf2_matrix_square OF((unsigned long *square, unsigned long *mat)); -local uLong crc32_combine_(uLong crc1, uLong crc2, z_off64_t len2); +local uLong crc32_combine_ OF((uLong crc1, uLong crc2, z_off64_t len2)); #ifdef DYNAMIC_CRC_TABLE local volatile int crc_table_empty = 1; -local unsigned long FAR crc_table[TBLS][256]; +local z_crc_t FAR crc_table[TBLS][256]; local void make_crc_table OF((void)); #ifdef MAKECRCH - local void write_table OF((FILE *, const unsigned long FAR *)); + local void write_table OF((FILE *, const z_crc_t FAR *)); #endif /* MAKECRCH */ /* Generate tables for a byte-wise 32-bit CRC calculation on the polynomial: @@ -109,9 +93,9 @@ local void make_crc_table OF((void)); */ local void make_crc_table() { - unsigned long c; + z_crc_t c; int n, k; - unsigned long poly; /* polynomial exclusive-or pattern */ + z_crc_t poly; /* polynomial exclusive-or pattern */ /* terms of polynomial defining this crc (except x^32): */ static volatile int first = 1; /* flag to limit concurrent making */ static const unsigned char p[] = {0,1,2,4,5,7,8,10,11,12,16,22,23,26}; @@ -123,13 +107,13 @@ local void make_crc_table() first = 0; /* make exclusive-or pattern from polynomial (0xedb88320UL) */ - poly = 0UL; - for (n = 0; n < sizeof(p)/sizeof(unsigned char); n++) - poly |= 1UL << (31 - p[n]); + poly = 0; + for (n = 0; n < (int)(sizeof(p)/sizeof(unsigned char)); n++) + poly |= (z_crc_t)1 << (31 - p[n]); /* generate a crc for every 8-bit value */ for (n = 0; n < 256; n++) { - c = (unsigned long)n; + c = (z_crc_t)n; for (k = 0; k < 8; k++) c = c & 1 ? poly ^ (c >> 1) : c >> 1; crc_table[0][n] = c; @@ -140,11 +124,11 @@ local void make_crc_table() and then the byte reversal of those as well as the first table */ for (n = 0; n < 256; n++) { c = crc_table[0][n]; - crc_table[4][n] = REV(c); + crc_table[4][n] = ZSWAP32(c); for (k = 1; k < 4; k++) { c = crc_table[0][c & 0xff] ^ (c >> 8); crc_table[k][n] = c; - crc_table[k + 4][n] = REV(c); + crc_table[k + 4][n] = ZSWAP32(c); } } #endif /* BYFOUR */ @@ -166,7 +150,7 @@ local void make_crc_table() if (out == NULL) return; fprintf(out, "/* crc32.h -- tables for rapid CRC calculation\n"); fprintf(out, " * Generated automatically by crc32.c\n */\n\n"); - fprintf(out, "local const unsigned long FAR "); + fprintf(out, "local const z_crc_t FAR "); fprintf(out, "crc_table[TBLS][256] =\n{\n {\n"); write_table(out, crc_table[0]); # ifdef BYFOUR @@ -186,12 +170,13 @@ local void make_crc_table() #ifdef MAKECRCH local void write_table(out, table) FILE *out; - const unsigned long FAR *table; + const z_crc_t FAR *table; { int n; for (n = 0; n < 256; n++) - fprintf(out, "%s0x%08lxUL%s", n % 5 ? "" : " ", table[n], + fprintf(out, "%s0x%08lxUL%s", n % 5 ? "" : " ", + (unsigned long)(table[n]), n == 255 ? "\n" : (n % 5 == 4 ? ",\n" : ", ")); } #endif /* MAKECRCH */ @@ -206,13 +191,13 @@ local void write_table(out, table) /* ========================================================================= * This function can be used by asm versions of crc32() */ -const unsigned long FAR * ZEXPORT get_crc_table() +const z_crc_t FAR * ZEXPORT get_crc_table() { #ifdef DYNAMIC_CRC_TABLE if (crc_table_empty) make_crc_table(); #endif /* DYNAMIC_CRC_TABLE */ - return (const unsigned long FAR *)crc_table; + return (const z_crc_t FAR *)crc_table; } /* ========================================================================= */ @@ -220,12 +205,44 @@ const unsigned long FAR * ZEXPORT get_crc_table() #define DO8 DO1; DO1; DO1; DO1; DO1; DO1; DO1; DO1 /* ========================================================================= */ -unsigned long ZEXPORT crc32(crc, buf, len) +unsigned long ZEXPORT crc32_z(crc, buf, len) unsigned long crc; const unsigned char FAR *buf; - uInt len; + z_size_t len; { - if (buf == Z_NULL) return 0UL; + /* + * zlib convention is to call crc32(0, NULL, 0); before making + * calls to crc32(). So this is a good, early (and infrequent) + * place to cache CPU features if needed for those later, more + * interesting crc32() calls. + */ +#if defined(CRC32_SIMD_SSE42_PCLMUL) + /* + * Use x86 sse4.2+pclmul SIMD to compute the crc32. Since this + * routine can be freely used, check CPU features here. + */ + if (buf == Z_NULL) { + if (!len) /* Assume user is calling crc32(0, NULL, 0); */ + cpu_check_features(); + return 0UL; + } + + if (x86_cpu_enable_simd && len >= Z_CRC32_SSE42_MINIMUM_LENGTH) { + /* crc32 16-byte chunks */ + z_size_t chunk_size = len & ~Z_CRC32_SSE42_CHUNKSIZE_MASK; + crc = ~crc32_sse42_simd_(buf, chunk_size, ~(uint32_t)crc); + /* check remaining data */ + len -= chunk_size; + if (!len) + return crc; + /* Fall into the default crc32 for the remaining data. */ + buf += chunk_size; + } +#else + if (buf == Z_NULL) { + return 0UL; + } +#endif /* CRC32_SIMD_SSE42_PCLMUL */ #ifdef DYNAMIC_CRC_TABLE if (crc_table_empty) @@ -234,7 +251,7 @@ unsigned long ZEXPORT crc32(crc, buf, len) #ifdef BYFOUR if (sizeof(void *) == sizeof(ptrdiff_t)) { - u4 endian; + z_crc_t endian; endian = 1; if (*((unsigned char *)(&endian))) @@ -254,8 +271,45 @@ unsigned long ZEXPORT crc32(crc, buf, len) return crc ^ 0xffffffffUL; } +/* ========================================================================= */ +unsigned long ZEXPORT crc32(crc, buf, len) + unsigned long crc; + const unsigned char FAR *buf; + uInt len; +{ +#if defined(CRC32_ARMV8_CRC32) + /* We got to verify ARM CPU features, so exploit the common usage pattern + * of calling this function with Z_NULL for an initial valid crc value. + * This allows to cache the result of the feature check and avoid extraneous + * function calls. + * TODO: try to move this to crc32_z if we don't loose performance on ARM. + */ + if (buf == Z_NULL) { + if (!len) /* Assume user is calling crc32(0, NULL, 0); */ + cpu_check_features(); + return 0UL; + } + + if (arm_cpu_enable_crc32) + return armv8_crc32_little(crc, buf, len); +#endif + return crc32_z(crc, buf, len); +} + #ifdef BYFOUR +/* + This BYFOUR code accesses the passed unsigned char * buffer with a 32-bit + integer pointer type. This violates the strict aliasing rule, where a + compiler can assume, for optimization purposes, that two pointers to + fundamentally different types won't ever point to the same memory. This can + manifest as a problem only if one of the pointers is written to. This code + only reads from those pointers. So long as this code remains isolated in + this compilation unit, there won't be a problem. For this reason, this code + should not be copied and pasted into a compilation unit in which other code + writes to the buffer that is passed to these routines. + */ + /* ========================================================================= */ #define DOLIT4 c ^= *buf4++; \ c = crc_table[3][c & 0xff] ^ crc_table[2][(c >> 8) & 0xff] ^ \ @@ -266,19 +320,19 @@ unsigned long ZEXPORT crc32(crc, buf, len) local unsigned long crc32_little(crc, buf, len) unsigned long crc; const unsigned char FAR *buf; - unsigned len; + z_size_t len; { - register u4 c; - register const u4 FAR *buf4; + register z_crc_t c; + register const z_crc_t FAR *buf4; - c = (u4)crc; + c = (z_crc_t)crc; c = ~c; while (len && ((ptrdiff_t)buf & 3)) { c = crc_table[0][(c ^ *buf++) & 0xff] ^ (c >> 8); len--; } - buf4 = (const u4 FAR *)(const void FAR *)buf; + buf4 = (const z_crc_t FAR *)(const void FAR *)buf; while (len >= 32) { DOLIT32; len -= 32; @@ -297,7 +351,7 @@ local unsigned long crc32_little(crc, buf, len) } /* ========================================================================= */ -#define DOBIG4 c ^= *++buf4; \ +#define DOBIG4 c ^= *buf4++; \ c = crc_table[4][c & 0xff] ^ crc_table[5][(c >> 8) & 0xff] ^ \ crc_table[6][(c >> 16) & 0xff] ^ crc_table[7][c >> 24] #define DOBIG32 DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4 @@ -306,20 +360,19 @@ local unsigned long crc32_little(crc, buf, len) local unsigned long crc32_big(crc, buf, len) unsigned long crc; const unsigned char FAR *buf; - unsigned len; + z_size_t len; { - register u4 c; - register const u4 FAR *buf4; + register z_crc_t c; + register const z_crc_t FAR *buf4; - c = REV((u4)crc); + c = ZSWAP32((z_crc_t)crc); c = ~c; while (len && ((ptrdiff_t)buf & 3)) { c = crc_table[4][(c >> 24) ^ *buf++] ^ (c << 8); len--; } - buf4 = (const u4 FAR *)(const void FAR *)buf; - buf4--; + buf4 = (const z_crc_t FAR *)(const void FAR *)buf; while (len >= 32) { DOBIG32; len -= 32; @@ -328,14 +381,13 @@ local unsigned long crc32_big(crc, buf, len) DOBIG4; len -= 4; } - buf4++; buf = (const unsigned char FAR *)buf4; if (len) do { c = crc_table[4][(c >> 24) ^ *buf++] ^ (c << 8); } while (--len); c = ~c; - return (unsigned long)(REV(c)); + return (unsigned long)(ZSWAP32(c)); } #endif /* BYFOUR */ @@ -445,25 +497,31 @@ uLong ZEXPORT crc32_combine64(crc1, crc2, len2) ZLIB_INTERNAL void crc_reset(deflate_state *const s) { +#ifdef CRC32_SIMD_SSE42_PCLMUL if (x86_cpu_enable_simd) { crc_fold_init(s); return; } +#endif s->strm->adler = crc32(0L, Z_NULL, 0); } ZLIB_INTERNAL void crc_finalize(deflate_state *const s) { +#ifdef CRC32_SIMD_SSE42_PCLMUL if (x86_cpu_enable_simd) s->strm->adler = crc_fold_512to32(s); +#endif } ZLIB_INTERNAL void copy_with_crc(z_streamp strm, Bytef *dst, long size) { +#ifdef CRC32_SIMD_SSE42_PCLMUL if (x86_cpu_enable_simd) { crc_fold_copy(strm->state, dst, strm->next_in, size); return; } +#endif zmemcpy(dst, strm->next_in, size); strm->adler = crc32(strm->adler, dst, size); } diff --git a/third_party/zlib/crc32.h b/third_party/zlib/crc32.h index 8053b6117c..9e0c778102 100644 --- a/third_party/zlib/crc32.h +++ b/third_party/zlib/crc32.h @@ -2,7 +2,7 @@ * Generated automatically by crc32.c */ -local const unsigned long FAR crc_table[TBLS][256] = +local const z_crc_t FAR crc_table[TBLS][256] = { { 0x00000000UL, 0x77073096UL, 0xee0e612cUL, 0x990951baUL, 0x076dc419UL, diff --git a/third_party/zlib/crc32_simd.c b/third_party/zlib/crc32_simd.c new file mode 100644 index 0000000000..c8e5592f38 --- /dev/null +++ b/third_party/zlib/crc32_simd.c @@ -0,0 +1,243 @@ +/* crc32_simd.c + * + * Copyright 2017 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#include "crc32_simd.h" + +#if defined(CRC32_SIMD_SSE42_PCLMUL) + +/* + * crc32_sse42_simd_(): compute the crc32 of the buffer, where the buffer + * length must be at least 64, and a multiple of 16. Based on: + * + * "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction" + * V. Gopal, E. Ozturk, et al., 2009, http://intel.ly/2ySEwL0 + */ + +#include +#include +#include + +uint32_t ZLIB_INTERNAL crc32_sse42_simd_( /* SSE4.2+PCLMUL */ + const unsigned char *buf, + z_size_t len, + uint32_t crc) +{ + /* + * Definitions of the bit-reflected domain constants k1,k2,k3, etc and + * the CRC32+Barrett polynomials given at the end of the paper. + */ + static const uint64_t zalign(16) k1k2[] = { 0x0154442bd4, 0x01c6e41596 }; + static const uint64_t zalign(16) k3k4[] = { 0x01751997d0, 0x00ccaa009e }; + static const uint64_t zalign(16) k5k0[] = { 0x0163cd6124, 0x0000000000 }; + static const uint64_t zalign(16) poly[] = { 0x01db710641, 0x01f7011641 }; + + __m128i x0, x1, x2, x3, x4, x5, x6, x7, x8, y5, y6, y7, y8; + + /* + * There's at least one block of 64. + */ + x1 = _mm_loadu_si128((__m128i *)(buf + 0x00)); + x2 = _mm_loadu_si128((__m128i *)(buf + 0x10)); + x3 = _mm_loadu_si128((__m128i *)(buf + 0x20)); + x4 = _mm_loadu_si128((__m128i *)(buf + 0x30)); + + x1 = _mm_xor_si128(x1, _mm_cvtsi32_si128(crc)); + + x0 = _mm_load_si128((__m128i *)k1k2); + + buf += 64; + len -= 64; + + /* + * Parallel fold blocks of 64, if any. + */ + while (len >= 64) + { + x5 = _mm_clmulepi64_si128(x1, x0, 0x00); + x6 = _mm_clmulepi64_si128(x2, x0, 0x00); + x7 = _mm_clmulepi64_si128(x3, x0, 0x00); + x8 = _mm_clmulepi64_si128(x4, x0, 0x00); + + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x2 = _mm_clmulepi64_si128(x2, x0, 0x11); + x3 = _mm_clmulepi64_si128(x3, x0, 0x11); + x4 = _mm_clmulepi64_si128(x4, x0, 0x11); + + y5 = _mm_loadu_si128((__m128i *)(buf + 0x00)); + y6 = _mm_loadu_si128((__m128i *)(buf + 0x10)); + y7 = _mm_loadu_si128((__m128i *)(buf + 0x20)); + y8 = _mm_loadu_si128((__m128i *)(buf + 0x30)); + + x1 = _mm_xor_si128(x1, x5); + x2 = _mm_xor_si128(x2, x6); + x3 = _mm_xor_si128(x3, x7); + x4 = _mm_xor_si128(x4, x8); + + x1 = _mm_xor_si128(x1, y5); + x2 = _mm_xor_si128(x2, y6); + x3 = _mm_xor_si128(x3, y7); + x4 = _mm_xor_si128(x4, y8); + + buf += 64; + len -= 64; + } + + /* + * Fold into 128-bits. + */ + x0 = _mm_load_si128((__m128i *)k3k4); + + x5 = _mm_clmulepi64_si128(x1, x0, 0x00); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x1 = _mm_xor_si128(x1, x2); + x1 = _mm_xor_si128(x1, x5); + + x5 = _mm_clmulepi64_si128(x1, x0, 0x00); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x1 = _mm_xor_si128(x1, x3); + x1 = _mm_xor_si128(x1, x5); + + x5 = _mm_clmulepi64_si128(x1, x0, 0x00); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x1 = _mm_xor_si128(x1, x4); + x1 = _mm_xor_si128(x1, x5); + + /* + * Single fold blocks of 16, if any. + */ + while (len >= 16) + { + x2 = _mm_loadu_si128((__m128i *)buf); + + x5 = _mm_clmulepi64_si128(x1, x0, 0x00); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x1 = _mm_xor_si128(x1, x2); + x1 = _mm_xor_si128(x1, x5); + + buf += 16; + len -= 16; + } + + /* + * Fold 128-bits to 64-bits. + */ + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x3 = _mm_setr_epi32(~0, 0, ~0, 0); + x1 = _mm_srli_si128(x1, 8); + x1 = _mm_xor_si128(x1, x2); + + x0 = _mm_loadl_epi64((__m128i*)k5k0); + + x2 = _mm_srli_si128(x1, 4); + x1 = _mm_and_si128(x1, x3); + x1 = _mm_clmulepi64_si128(x1, x0, 0x00); + x1 = _mm_xor_si128(x1, x2); + + /* + * Barret reduce to 32-bits. + */ + x0 = _mm_load_si128((__m128i*)poly); + + x2 = _mm_and_si128(x1, x3); + x2 = _mm_clmulepi64_si128(x2, x0, 0x10); + x2 = _mm_and_si128(x2, x3); + x2 = _mm_clmulepi64_si128(x2, x0, 0x00); + x1 = _mm_xor_si128(x1, x2); + + /* + * Return the crc32. + */ + return _mm_extract_epi32(x1, 1); +} + +#elif defined(CRC32_ARMV8_CRC32) + +/* CRC32 checksums using ARMv8-a crypto instructions. + * + * TODO: implement a version using the PMULL instruction. + */ + +#if defined(__clang__) +/* CRC32 intrinsics are #ifdef'ed out of arm_acle.h unless we build with an + * armv8 target, which is incompatible with ThinLTO optimizations on Android. + * (Namely, mixing and matching different module-level targets makes ThinLTO + * warn, and Android defaults to armv7-a. This restriction does not apply to + * function-level `target`s, however.) + * + * Since we only need four crc intrinsics, and since clang's implementation of + * those are just wrappers around compiler builtins, it's simplest to #define + * those builtins directly. If this #define list grows too much (or we depend on + * an intrinsic that isn't a trivial wrapper), we may have to find a better way + * to go about this. + * + * NOTE: clang currently complains that "'+soft-float-abi' is not a recognized + * feature for this target (ignoring feature)." This appears to be a harmless + * bug in clang. + */ +#define __crc32b __builtin_arm_crc32b +#define __crc32d __builtin_arm_crc32d +#define __crc32w __builtin_arm_crc32w +#define __crc32cw __builtin_arm_crc32cw + +#if defined(__aarch64__) +#define TARGET_ARMV8_WITH_CRC __attribute__((target("crc"))) +#else // !defined(__aarch64__) +#define TARGET_ARMV8_WITH_CRC __attribute__((target("armv8-a,crc"))) +#endif // defined(__aarch64__) + +#elif defined(__GNUC__) +/* For GCC, we are setting CRC extensions at module level, so ThinLTO is not + * allowed. We can just include arm_acle.h. + */ +#include +#define TARGET_ARMV8_WITH_CRC +#else // !defined(__GNUC__) && !defined(_aarch64__) +#error ARM CRC32 SIMD extensions only supported for Clang and GCC +#endif + +TARGET_ARMV8_WITH_CRC +uint32_t ZLIB_INTERNAL armv8_crc32_little(unsigned long crc, + const unsigned char *buf, + z_size_t len) +{ + uint32_t c = (uint32_t) ~crc; + + while (len && ((uintptr_t)buf & 7)) { + c = __crc32b(c, *buf++); + --len; + } + + const uint64_t *buf8 = (const uint64_t *)buf; + + while (len >= 64) { + c = __crc32d(c, *buf8++); + c = __crc32d(c, *buf8++); + c = __crc32d(c, *buf8++); + c = __crc32d(c, *buf8++); + + c = __crc32d(c, *buf8++); + c = __crc32d(c, *buf8++); + c = __crc32d(c, *buf8++); + c = __crc32d(c, *buf8++); + len -= 64; + } + + while (len >= 8) { + c = __crc32d(c, *buf8++); + len -= 8; + } + + buf = (const unsigned char *)buf8; + + while (len--) { + c = __crc32b(c, *buf++); + } + + return ~c; +} + +#endif diff --git a/third_party/zlib/crc32_simd.h b/third_party/zlib/crc32_simd.h new file mode 100644 index 0000000000..68bc235cbe --- /dev/null +++ b/third_party/zlib/crc32_simd.h @@ -0,0 +1,36 @@ +/* crc32_simd.h + * + * Copyright 2017 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#include + +#include "zconf.h" +#include "zutil.h" +#include "deflate.h" + +/* + * crc32_sse42_simd_(): compute the crc32 of the buffer, where the buffer + * length must be at least 64, and a multiple of 16. + */ +uint32_t ZLIB_INTERNAL crc32_sse42_simd_( + const unsigned char *buf, + z_size_t len, + uint32_t crc); + +/* + * crc32_sse42_simd_ buffer size constraints: see the use in zlib/crc32.c + * for computing the crc32 of an arbitrary length buffer. + */ +#define Z_CRC32_SSE42_MINIMUM_LENGTH 64 +#define Z_CRC32_SSE42_CHUNKSIZE_MASK 15 + +/* + * CRC32 checksums using ARMv8-a crypto instructions. + */ +uint32_t ZLIB_INTERNAL armv8_crc32_little(unsigned long crc, + const unsigned char* buf, + z_size_t len); + diff --git a/third_party/zlib/crc_folding.c b/third_party/zlib/crc_folding.c index 98c559cee3..ee31d4918d 100644 --- a/third_party/zlib/crc_folding.c +++ b/third_party/zlib/crc_folding.c @@ -18,6 +18,8 @@ #include "deflate.h" +#ifdef CRC32_SIMD_SSE42_PCLMUL + #include #include #include @@ -283,7 +285,7 @@ ZLIB_INTERNAL void crc_fold_copy(deflate_state *const s, goto partial; } - algn_diff = 0 - (unsigned long)src & 0xF; + algn_diff = (0 - (uintptr_t)src) & 0xF; if (algn_diff) { xmm_crc_part = _mm_loadu_si128((__m128i *)src); _mm_storeu_si128((__m128i *)dst, xmm_crc_part); @@ -491,3 +493,5 @@ unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state *const s) return ~crc; CRC_SAVE(s) } + +#endif /* CRC32_SIMD_SSE42_PCLMUL */ diff --git a/third_party/zlib/deflate.c b/third_party/zlib/deflate.c index 55ec21570a..1597196b08 100644 --- a/third_party/zlib/deflate.c +++ b/third_party/zlib/deflate.c @@ -1,5 +1,5 @@ /* deflate.c -- compress data using the deflation algorithm - * Copyright (C) 1995-2010 Jean-loup Gailly and Mark Adler + * Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -37,7 +37,7 @@ * REFERENCES * * Deutsch, L.P.,"DEFLATE Compressed Data Format Specification". - * Available in http://www.ietf.org/rfc/rfc1951.txt + * Available in http://tools.ietf.org/html/rfc1951 * * A description of the Rabin and Karp algorithm is given in the book * "Algorithms" by R. Sedgewick, Addison-Wesley, p252. @@ -48,14 +48,20 @@ */ /* @(#) $Id$ */ - #include - #include "deflate.h" -#include "x86.h" +#include "cpu_features.h" +#include "contrib/optimizations/insert_string.h" + +#if (defined(__ARM_NEON__) || defined(__ARM_NEON)) +#include "contrib/optimizations/slide_hash_neon.h" +#endif +#if defined(CRC32_ARMV8_CRC32) +#include "crc32_simd.h" +#endif const char deflate_copyright[] = - " deflate 1.2.5 Copyright 1995-2010 Jean-loup Gailly and Mark Adler "; + " deflate 1.2.11 Copyright 1995-2017 Jean-loup Gailly and Mark Adler "; /* If you use the zlib library in a product, an acknowledgment is welcome in the documentation of your product. If for some reason you cannot @@ -73,51 +79,41 @@ typedef enum { finish_done /* finish done, accept no more input or output */ } block_state; -typedef block_state (*compress_func) OF((deflate_state *s, int flush, - int clas)); +typedef block_state (*compress_func) OF((deflate_state *s, int flush)); /* Compression function. Returns the block state after the call. */ +local int deflateStateCheck OF((z_streamp strm)); +local void slide_hash OF((deflate_state *s)); local void fill_window OF((deflate_state *s)); -local block_state deflate_stored OF((deflate_state *s, int flush, int clas)); -local block_state deflate_fast OF((deflate_state *s, int flush, int clas)); +local block_state deflate_stored OF((deflate_state *s, int flush)); +local block_state deflate_fast OF((deflate_state *s, int flush)); #ifndef FASTEST -local block_state deflate_slow OF((deflate_state *s, int flush, int clas)); +local block_state deflate_slow OF((deflate_state *s, int flush)); #endif local block_state deflate_rle OF((deflate_state *s, int flush)); local block_state deflate_huff OF((deflate_state *s, int flush)); local void lm_init OF((deflate_state *s)); local void putShortMSB OF((deflate_state *s, uInt b)); local void flush_pending OF((z_streamp strm)); - +unsigned ZLIB_INTERNAL deflate_read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); #ifdef ASMV +# pragma message("Assembler code may have bugs -- use at your own risk") void match_init OF((void)); /* asm code initialization */ - uInt longest_match OF((deflate_state *s, IPos cur_match, int clas)); + uInt longest_match OF((deflate_state *s, IPos cur_match)); #else -local uInt longest_match OF((deflate_state *s, IPos cur_match, int clas)); +local uInt longest_match OF((deflate_state *s, IPos cur_match)); #endif -#ifdef DEBUG +#ifdef ZLIB_DEBUG local void check_match OF((deflate_state *s, IPos start, IPos match, int length)); #endif -/* For fill_window_sse.c to use */ -ZLIB_INTERNAL int read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); - /* From crc32.c */ extern void ZLIB_INTERNAL crc_reset(deflate_state *const s); extern void ZLIB_INTERNAL crc_finalize(deflate_state *const s); extern void ZLIB_INTERNAL copy_with_crc(z_streamp strm, Bytef *dst, long size); -#ifdef _MSC_VER -#define INLINE __inline -#else -#define INLINE inline -#endif - -/* Inline optimisation */ -local INLINE Pos insert_string_sse(deflate_state *const s, const Pos str); - /* =========================================================================== * Local data */ @@ -169,62 +165,52 @@ local const config configuration_table[10] = { * meaning. */ -#define EQUAL 0 -/* result of memcmp for equal strings */ - -#ifndef NO_DUMMY_DECL -struct static_tree_desc_s {int dummy;}; /* for buggy compilers */ -#endif +/* rank Z_BLOCK between Z_NO_FLUSH and Z_PARTIAL_FLUSH */ +#define RANK(f) (((f) * 2) - ((f) > 4 ? 9 : 0)) /* =========================================================================== - * Update a hash value with the given input byte - * IN assertion: all calls to to UPDATE_HASH are made with consecutive - * input characters, so that a running hash key can be computed from the - * previous key instead of complete recalculation each time. + * Initialize the hash table (avoiding 64K overflow for 16 bit systems). + * prev[] will be initialized on the fly. */ -#define UPDATE_HASH(s,h,c) (h = (((h)<hash_shift) ^ (c)) & s->hash_mask) +#define CLEAR_HASH(s) \ + s->head[s->hash_size-1] = NIL; \ + zmemzero((Bytef *)s->head, (unsigned)(s->hash_size-1)*sizeof(*s->head)); /* =========================================================================== - * Insert string str in the dictionary and set match_head to the previous head - * of the hash chain (the most recent string with same hash key). Return - * the previous length of the hash chain. - * If this file is compiled with -DFASTEST, the compression level is forced - * to 1, and no hash chains are maintained. - * IN assertion: all calls to to INSERT_STRING are made with consecutive - * input characters and the first MIN_MATCH bytes of str are valid - * (except for the last MIN_MATCH-1 bytes of the input file). + * Slide the hash table when sliding the window down (could be avoided with 32 + * bit values at the expense of memory usage). We slide even when level == 0 to + * keep the hash table consistent if we switch back to level > 0 later. */ -local INLINE Pos insert_string_c(deflate_state *const s, const Pos str) +local void slide_hash(s) + deflate_state *s; { - Pos ret; - - UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]); -#ifdef FASTEST - ret = s->head[s->ins_h]; -#else - ret = s->prev[str & s->w_mask] = s->head[s->ins_h]; +#if (defined(__ARM_NEON__) || defined(__ARM_NEON)) + /* NEON based hash table rebase. */ + return neon_slide_hash(s->head, s->prev, s->w_size, s->hash_size); #endif - s->head[s->ins_h] = str; - - return ret; -} + unsigned n, m; + Posf *p; + uInt wsize = s->w_size; -local INLINE Pos insert_string(deflate_state *const s, const Pos str) -{ - if (x86_cpu_enable_simd) - return insert_string_sse(s, str); - return insert_string_c(s, str); + n = s->hash_size; + p = &s->head[n]; + do { + m = *--p; + *p = (Pos)(m >= wsize ? m - wsize : NIL); + } while (--n); + n = wsize; +#ifndef FASTEST + p = &s->prev[n]; + do { + m = *--p; + *p = (Pos)(m >= wsize ? m - wsize : NIL); + /* If n is not on any hash chain, prev[n] is garbage but + * its value will never be used. + */ + } while (--n); +#endif } - -/* =========================================================================== - * Initialize the hash table (avoiding 64K overflow for 16 bit systems). - * prev[] will be initialized on the fly. - */ -#define CLEAR_HASH(s) \ - s->head[s->hash_size-1] = NIL; \ - zmemzero((Bytef *)s->head, (unsigned)(s->hash_size-1)*sizeof(*s->head)); - /* ========================================================================= */ int ZEXPORT deflateInit_(strm, level, version, stream_size) z_streamp strm; @@ -254,12 +240,13 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, int wrap = 1; static const char my_version[] = ZLIB_VERSION; - ushf *overlay; - /* We overlay pending_buf and d_buf+l_buf. This works since the average - * output size for (length,distance) codes is <= 24 bits. - */ - - x86_check_features(); + // Needed to activate optimized insert_string() that helps compression + // for all wrapper formats (e.g. RAW, ZLIB, GZIP). + // Feature detection is not triggered while using RAW mode (i.e. we never + // call crc32() with a NULL buffer). +#if defined(CRC32_ARMV8_CRC32) || defined(CRC32_SIMD_SSE42_PCLMUL) + cpu_check_features(); +#endif if (version == Z_NULL || version[0] != my_version[0] || stream_size != sizeof(z_stream)) { @@ -269,10 +256,19 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, strm->msg = Z_NULL; if (strm->zalloc == (alloc_func)0) { +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else strm->zalloc = zcalloc; strm->opaque = (voidpf)0; +#endif } - if (strm->zfree == (free_func)0) strm->zfree = zcfree; + if (strm->zfree == (free_func)0) +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zfree = zcfree; +#endif #ifdef FASTEST if (level != 0) level = 1; @@ -292,7 +288,7 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, #endif if (memLevel < 1 || memLevel > MAX_MEM_LEVEL || method != Z_DEFLATED || windowBits < 8 || windowBits > 15 || level < 0 || level > 9 || - strategy < 0 || strategy > Z_FIXED) { + strategy < 0 || strategy > Z_FIXED || (windowBits == 8 && wrap != 1)) { return Z_STREAM_ERROR; } if (windowBits == 8) windowBits = 9; /* until 256-byte window bug fixed */ @@ -300,10 +296,11 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, if (s == Z_NULL) return Z_MEM_ERROR; strm->state = (struct internal_state FAR *)s; s->strm = strm; + s->status = INIT_STATE; /* to pass state test in deflateReset() */ s->wrap = wrap; s->gzhead = Z_NULL; - s->w_bits = windowBits; + s->w_bits = (uInt)windowBits; s->w_size = 1 << s->w_bits; s->w_mask = s->w_size - 1; @@ -317,30 +314,74 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, s->hash_mask = s->hash_size - 1; s->hash_shift = ((s->hash_bits+MIN_MATCH-1)/MIN_MATCH); - s->window = (Bytef *) ZALLOC(strm, s->w_size + window_padding, 2*sizeof(Byte)); + s->window = (Bytef *) ZALLOC(strm, + s->w_size + window_padding, + 2*sizeof(Byte)); s->prev = (Posf *) ZALLOC(strm, s->w_size, sizeof(Pos)); + /* Avoid use of uninitialized value, see: + * https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=11360 + */ + zmemzero(s->prev, s->w_size * sizeof(Pos)); s->head = (Posf *) ZALLOC(strm, s->hash_size, sizeof(Pos)); - s->class_bitmap = NULL; - zmemzero(&s->cookie_locations, sizeof(s->cookie_locations)); - strm->clas = 0; s->high_water = 0; /* nothing written to s->window yet */ s->lit_bufsize = 1 << (memLevel + 6); /* 16K elements by default */ - overlay = (ushf *) ZALLOC(strm, s->lit_bufsize, sizeof(ush)+2); - s->pending_buf = (uchf *) overlay; - s->pending_buf_size = (ulg)s->lit_bufsize * (sizeof(ush)+2L); + /* We overlay pending_buf and sym_buf. This works since the average size + * for length/distance pairs over any compressed block is assured to be 31 + * bits or less. + * + * Analysis: The longest fixed codes are a length code of 8 bits plus 5 + * extra bits, for lengths 131 to 257. The longest fixed distance codes are + * 5 bits plus 13 extra bits, for distances 16385 to 32768. The longest + * possible fixed-codes length/distance pair is then 31 bits total. + * + * sym_buf starts one-fourth of the way into pending_buf. So there are + * three bytes in sym_buf for every four bytes in pending_buf. Each symbol + * in sym_buf is three bytes -- two for the distance and one for the + * literal/length. As each symbol is consumed, the pointer to the next + * sym_buf value to read moves forward three bytes. From that symbol, up to + * 31 bits are written to pending_buf. The closest the written pending_buf + * bits gets to the next sym_buf symbol to read is just before the last + * code is written. At that time, 31*(n-2) bits have been written, just + * after 24*(n-2) bits have been consumed from sym_buf. sym_buf starts at + * 8*n bits into pending_buf. (Note that the symbol buffer fills when n-1 + * symbols are written.) The closest the writing gets to what is unread is + * then n+14 bits. Here n is lit_bufsize, which is 16384 by default, and + * can range from 128 to 32768. + * + * Therefore, at a minimum, there are 142 bits of space between what is + * written and what is read in the overlain buffers, so the symbols cannot + * be overwritten by the compressed data. That space is actually 139 bits, + * due to the three-bit fixed-code block header. + * + * That covers the case where either Z_FIXED is specified, forcing fixed + * codes, or when the use of fixed codes is chosen, because that choice + * results in a smaller compressed block than dynamic codes. That latter + * condition then assures that the above analysis also covers all dynamic + * blocks. A dynamic-code block will only be chosen to be emitted if it has + * fewer bits than a fixed-code block would for the same set of symbols. + * Therefore its average symbol length is assured to be less than 31. So + * the compressed data for a dynamic block also cannot overwrite the + * symbols from which it is being constructed. + */ + s->pending_buf = (uchf *) ZALLOC(strm, s->lit_bufsize, 4); + s->pending_buf_size = (ulg)s->lit_bufsize * 4; if (s->window == Z_NULL || s->prev == Z_NULL || s->head == Z_NULL || s->pending_buf == Z_NULL) { s->status = FINISH_STATE; - strm->msg = (char*)ERR_MSG(Z_MEM_ERROR); + strm->msg = ERR_MSG(Z_MEM_ERROR); deflateEnd (strm); return Z_MEM_ERROR; } - s->d_buf = overlay + s->lit_bufsize/sizeof(ush); - s->l_buf = s->pending_buf + (1+sizeof(ush))*s->lit_bufsize; + s->sym_buf = s->pending_buf + s->lit_bufsize; + s->sym_end = (s->lit_bufsize - 1) * 3; + /* We avoid equality with lit_bufsize*3 because of wraparound at 64K + * on 16 bit machines and because stored blocks are restricted to + * 64K-1 bytes. + */ s->level = level; s->strategy = strategy; @@ -349,6 +390,31 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, return deflateReset(strm); } +/* ========================================================================= + * Check for a valid deflate stream state. Return 0 if ok, 1 if not. + */ +local int deflateStateCheck (strm) + z_streamp strm; +{ + deflate_state *s; + if (strm == Z_NULL || + strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) + return 1; + s = strm->state; + if (s == Z_NULL || s->strm != strm || (s->status != INIT_STATE && +#ifdef GZIP + s->status != GZIP_STATE && +#endif + s->status != EXTRA_STATE && + s->status != NAME_STATE && + s->status != COMMENT_STATE && + s->status != HCRC_STATE && + s->status != BUSY_STATE && + s->status != FINISH_STATE)) + return 1; + return 0; +} + /* ========================================================================= */ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength) z_streamp strm; @@ -356,49 +422,93 @@ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength) uInt dictLength; { deflate_state *s; - uInt length = dictLength; - uInt n; - IPos hash_head = 0; + uInt str, n; + int wrap; + unsigned avail; + z_const unsigned char *next; - if (strm == Z_NULL || strm->state == Z_NULL || dictionary == Z_NULL || - strm->state->wrap == 2 || - (strm->state->wrap == 1 && strm->state->status != INIT_STATE)) + if (deflateStateCheck(strm) || dictionary == Z_NULL) return Z_STREAM_ERROR; - s = strm->state; - if (s->wrap) - strm->adler = adler32(strm->adler, dictionary, dictLength); + wrap = s->wrap; + if (wrap == 2 || (wrap == 1 && s->status != INIT_STATE) || s->lookahead) + return Z_STREAM_ERROR; - if (length < MIN_MATCH) return Z_OK; - if (length > s->w_size) { - length = s->w_size; - dictionary += dictLength - length; /* use the tail of the dictionary */ + /* when using zlib wrappers, compute Adler-32 for provided dictionary */ + if (wrap == 1) + strm->adler = adler32(strm->adler, dictionary, dictLength); + s->wrap = 0; /* avoid computing Adler-32 in deflate_read_buf */ + + /* if dictionary would fill window, just replace the history */ + if (dictLength >= s->w_size) { + if (wrap == 0) { /* already empty otherwise */ + CLEAR_HASH(s); + s->strstart = 0; + s->block_start = 0L; + s->insert = 0; + } + dictionary += dictLength - s->w_size; /* use the tail */ + dictLength = s->w_size; } - zmemcpy(s->window, dictionary, length); - s->strstart = length; - s->block_start = (long)length; - /* Insert all strings in the hash table (except for the last two bytes). - * s->lookahead stays null, so s->ins_h will be recomputed at the next - * call of fill_window. - */ - s->ins_h = s->window[0]; - UPDATE_HASH(s, s->ins_h, s->window[1]); - for (n = 0; n <= length - MIN_MATCH; n++) { - insert_string(s, n); + /* insert dictionary into window and hash */ + avail = strm->avail_in; + next = strm->next_in; + strm->avail_in = dictLength; + strm->next_in = (z_const Bytef *)dictionary; + fill_window(s); + while (s->lookahead >= MIN_MATCH) { + str = s->strstart; + n = s->lookahead - (MIN_MATCH-1); + do { + insert_string(s, str); + str++; + } while (--n); + s->strstart = str; + s->lookahead = MIN_MATCH-1; + fill_window(s); } - if (hash_head) hash_head = 0; /* to make compiler happy */ + s->strstart += s->lookahead; + s->block_start = (long)s->strstart; + s->insert = s->lookahead; + s->lookahead = 0; + s->match_length = s->prev_length = MIN_MATCH-1; + s->match_available = 0; + strm->next_in = next; + strm->avail_in = avail; + s->wrap = wrap; return Z_OK; } /* ========================================================================= */ -int ZEXPORT deflateReset (strm) +int ZEXPORT deflateGetDictionary (strm, dictionary, dictLength) z_streamp strm; + Bytef *dictionary; + uInt *dictLength; { deflate_state *s; + uInt len; - if (strm == Z_NULL || strm->state == Z_NULL || - strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) { + if (deflateStateCheck(strm)) + return Z_STREAM_ERROR; + s = strm->state; + len = s->strstart + s->lookahead; + if (len > s->w_size) + len = s->w_size; + if (dictionary != Z_NULL && len) + zmemcpy(dictionary, s->window + s->strstart + s->lookahead - len, len); + if (dictLength != Z_NULL) + *dictLength = len; + return Z_OK; +} + +/* ========================================================================= */ +int ZEXPORT deflateResetKeep (strm) + z_streamp strm; +{ + deflate_state *s; + + if (deflateStateCheck(strm)) { return Z_STREAM_ERROR; } @@ -409,13 +519,15 @@ int ZEXPORT deflateReset (strm) s = (deflate_state *)strm->state; s->pending = 0; s->pending_out = s->pending_buf; - TRY_FREE(strm, s->class_bitmap); - s->class_bitmap = NULL; if (s->wrap < 0) { s->wrap = -s->wrap; /* was made negative by deflate(..., Z_FINISH); */ } - s->status = s->wrap ? INIT_STATE : BUSY_STATE; + s->status = +#ifdef GZIP + s->wrap == 2 ? GZIP_STATE : +#endif + s->wrap ? INIT_STATE : BUSY_STATE; strm->adler = #ifdef GZIP s->wrap == 2 ? crc32(0L, Z_NULL, 0) : @@ -424,31 +536,70 @@ int ZEXPORT deflateReset (strm) s->last_flush = Z_NO_FLUSH; _tr_init(s); - lm_init(s); return Z_OK; } +/* ========================================================================= */ +int ZEXPORT deflateReset (strm) + z_streamp strm; +{ + int ret; + + ret = deflateResetKeep(strm); + if (ret == Z_OK) + lm_init(strm->state); + return ret; +} + /* ========================================================================= */ int ZEXPORT deflateSetHeader (strm, head) z_streamp strm; gz_headerp head; { - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; - if (strm->state->wrap != 2) return Z_STREAM_ERROR; + if (deflateStateCheck(strm) || strm->state->wrap != 2) + return Z_STREAM_ERROR; strm->state->gzhead = head; return Z_OK; } +/* ========================================================================= */ +int ZEXPORT deflatePending (strm, pending, bits) + unsigned *pending; + int *bits; + z_streamp strm; +{ + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; + if (pending != Z_NULL) + *pending = strm->state->pending; + if (bits != Z_NULL) + *bits = strm->state->bi_valid; + return Z_OK; +} + /* ========================================================================= */ int ZEXPORT deflatePrime (strm, bits, value) z_streamp strm; int bits; int value; { - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; - strm->state->bi_valid = bits; - strm->state->bi_buf = (ush)(value & ((1 << bits) - 1)); + deflate_state *s; + int put; + + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; + s = strm->state; + if (s->sym_buf < s->pending_out + ((Buf_size + 7) >> 3)) + return Z_BUF_ERROR; + do { + put = Buf_size - s->bi_valid; + if (put > bits) + put = bits; + s->bi_buf |= (ush)((value & ((1 << put) - 1)) << s->bi_valid); + s->bi_valid += put; + _tr_flush_bits(s); + value >>= put; + bits -= put; + } while (bits); return Z_OK; } @@ -460,9 +611,8 @@ int ZEXPORT deflateParams(strm, level, strategy) { deflate_state *s; compress_func func; - int err = Z_OK; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; s = strm->state; #ifdef FASTEST @@ -476,11 +626,22 @@ int ZEXPORT deflateParams(strm, level, strategy) func = configuration_table[s->level].func; if ((strategy != s->strategy || func != configuration_table[level].func) && - strm->total_in != 0) { + s->high_water) { /* Flush the last buffer: */ - err = deflate(strm, Z_BLOCK); + int err = deflate(strm, Z_BLOCK); + if (err == Z_STREAM_ERROR) + return err; + if (strm->avail_out == 0) + return Z_BUF_ERROR; } if (s->level != level) { + if (s->level == 0 && s->matches != 0) { + if (s->matches == 1) + slide_hash(s); + else + CLEAR_HASH(s); + s->matches = 0; + } s->level = level; s->max_lazy_match = configuration_table[level].max_lazy; s->good_match = configuration_table[level].good_length; @@ -488,7 +649,7 @@ int ZEXPORT deflateParams(strm, level, strategy) s->max_chain_length = configuration_table[level].max_chain; } s->strategy = strategy; - return err; + return Z_OK; } /* ========================================================================= */ @@ -501,12 +662,12 @@ int ZEXPORT deflateTune(strm, good_length, max_lazy, nice_length, max_chain) { deflate_state *s; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; s = strm->state; - s->good_match = good_length; - s->max_lazy_match = max_lazy; + s->good_match = (uInt)good_length; + s->max_lazy_match = (uInt)max_lazy; s->nice_match = nice_length; - s->max_chain_length = max_chain; + s->max_chain_length = (uInt)max_chain; return Z_OK; } @@ -533,14 +694,13 @@ uLong ZEXPORT deflateBound(strm, sourceLen) { deflate_state *s; uLong complen, wraplen; - Bytef *str; /* conservative upper bound for compressed data */ complen = sourceLen + ((sourceLen + 7) >> 3) + ((sourceLen + 63) >> 6) + 5; /* if can't get parameters, return conservative bound plus zlib wrapper */ - if (strm == Z_NULL || strm->state == Z_NULL) + if (deflateStateCheck(strm)) return complen + 6; /* compute wrapper length */ @@ -552,9 +712,11 @@ uLong ZEXPORT deflateBound(strm, sourceLen) case 1: /* zlib wrapper */ wraplen = 6 + (s->strstart ? 4 : 0); break; +#ifdef GZIP case 2: /* gzip wrapper */ wraplen = 18; if (s->gzhead != Z_NULL) { /* user-supplied gzip header */ + Bytef *str; if (s->gzhead->extra != Z_NULL) wraplen += 2 + s->gzhead->extra_len; str = s->gzhead->name; @@ -571,6 +733,7 @@ uLong ZEXPORT deflateBound(strm, sourceLen) wraplen += 2; } break; +#endif default: /* for compiler happiness */ wraplen = 6; } @@ -598,30 +761,43 @@ local void putShortMSB (s, b) } /* ========================================================================= - * Flush as much pending output as possible. All deflate() output goes - * through this function so some applications may wish to modify it - * to avoid allocating a large strm->next_out buffer and copying into it. - * (See also read_buf()). + * Flush as much pending output as possible. All deflate() output, except for + * some deflate_stored() output, goes through this function so some + * applications may wish to modify it to avoid allocating a large + * strm->next_out buffer and copying into it. (See also deflate_read_buf()). */ local void flush_pending(strm) z_streamp strm; { - unsigned len = strm->state->pending; + unsigned len; + deflate_state *s = strm->state; + _tr_flush_bits(s); + len = s->pending; if (len > strm->avail_out) len = strm->avail_out; if (len == 0) return; - zmemcpy(strm->next_out, strm->state->pending_out, len); + zmemcpy(strm->next_out, s->pending_out, len); strm->next_out += len; - strm->state->pending_out += len; + s->pending_out += len; strm->total_out += len; - strm->avail_out -= len; - strm->state->pending -= len; - if (strm->state->pending == 0) { - strm->state->pending_out = strm->state->pending_buf; + strm->avail_out -= len; + s->pending -= len; + if (s->pending == 0) { + s->pending_out = s->pending_buf; } } +/* =========================================================================== + * Update the header CRC with the bytes s->pending_buf[beg..s->pending - 1]. + */ +#define HCRC_UPDATE(beg) \ + do { \ + if (s->gzhead->hcrc && s->pending > (beg)) \ + strm->adler = crc32(strm->adler, s->pending_buf + (beg), \ + s->pending - (beg)); \ + } while (0) + /* ========================================================================= */ int ZEXPORT deflate (strm, flush) z_streamp strm; @@ -630,230 +806,229 @@ int ZEXPORT deflate (strm, flush) int old_flush; /* value of flush param for previous deflate call */ deflate_state *s; - if (strm == Z_NULL || strm->state == Z_NULL || - flush > Z_BLOCK || flush < 0) { + if (deflateStateCheck(strm) || flush > Z_BLOCK || flush < 0) { return Z_STREAM_ERROR; } s = strm->state; if (strm->next_out == Z_NULL || - (strm->next_in == Z_NULL && strm->avail_in != 0) || + (strm->avail_in != 0 && strm->next_in == Z_NULL) || (s->status == FINISH_STATE && flush != Z_FINISH)) { ERR_RETURN(strm, Z_STREAM_ERROR); } if (strm->avail_out == 0) ERR_RETURN(strm, Z_BUF_ERROR); - s->strm = strm; /* just in case */ old_flush = s->last_flush; s->last_flush = flush; + /* Flush as much pending output as possible */ + if (s->pending != 0) { + flush_pending(strm); + if (strm->avail_out == 0) { + /* Since avail_out is 0, deflate will be called again with + * more output space, but possibly with both pending and + * avail_in equal to zero. There won't be anything to do, + * but this is not an error situation so make sure we + * return OK instead of BUF_ERROR at next call of deflate: + */ + s->last_flush = -1; + return Z_OK; + } + + /* Make sure there is something to do and avoid duplicate consecutive + * flushes. For repeated and useless calls with Z_FINISH, we keep + * returning Z_STREAM_END instead of Z_BUF_ERROR. + */ + } else if (strm->avail_in == 0 && RANK(flush) <= RANK(old_flush) && + flush != Z_FINISH) { + ERR_RETURN(strm, Z_BUF_ERROR); + } + + /* User must not provide more input after the first FINISH: */ + if (s->status == FINISH_STATE && strm->avail_in != 0) { + ERR_RETURN(strm, Z_BUF_ERROR); + } + /* Write the header */ if (s->status == INIT_STATE) { -#ifdef GZIP - if (s->wrap == 2) { - crc_reset(s); - put_byte(s, 31); - put_byte(s, 139); - put_byte(s, 8); - if (s->gzhead == Z_NULL) { - put_byte(s, 0); - put_byte(s, 0); - put_byte(s, 0); - put_byte(s, 0); - put_byte(s, 0); - put_byte(s, s->level == 9 ? 2 : - (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? - 4 : 0)); - put_byte(s, OS_CODE); - s->status = BUSY_STATE; - } - else { - put_byte(s, (s->gzhead->text ? 1 : 0) + - (s->gzhead->hcrc ? 2 : 0) + - (s->gzhead->extra == Z_NULL ? 0 : 4) + - (s->gzhead->name == Z_NULL ? 0 : 8) + - (s->gzhead->comment == Z_NULL ? 0 : 16) - ); - put_byte(s, (Byte)(s->gzhead->time & 0xff)); - put_byte(s, (Byte)((s->gzhead->time >> 8) & 0xff)); - put_byte(s, (Byte)((s->gzhead->time >> 16) & 0xff)); - put_byte(s, (Byte)((s->gzhead->time >> 24) & 0xff)); - put_byte(s, s->level == 9 ? 2 : - (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? - 4 : 0)); - put_byte(s, s->gzhead->os & 0xff); - if (s->gzhead->extra != Z_NULL) { - put_byte(s, s->gzhead->extra_len & 0xff); - put_byte(s, (s->gzhead->extra_len >> 8) & 0xff); - } - if (s->gzhead->hcrc) - strm->adler = crc32(strm->adler, s->pending_buf, - s->pending); - s->gzindex = 0; - s->status = EXTRA_STATE; - } - } + /* zlib header */ + uInt header = (Z_DEFLATED + ((s->w_bits-8)<<4)) << 8; + uInt level_flags; + + if (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2) + level_flags = 0; + else if (s->level < 6) + level_flags = 1; + else if (s->level == 6) + level_flags = 2; else -#endif - { - uInt header = (Z_DEFLATED + ((s->w_bits-8)<<4)) << 8; - uInt level_flags; - - if (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2) - level_flags = 0; - else if (s->level < 6) - level_flags = 1; - else if (s->level == 6) - level_flags = 2; - else - level_flags = 3; - header |= (level_flags << 6); - if (s->strstart != 0) header |= PRESET_DICT; - header += 31 - (header % 31); + level_flags = 3; + header |= (level_flags << 6); + if (s->strstart != 0) header |= PRESET_DICT; + header += 31 - (header % 31); + putShortMSB(s, header); + + /* Save the adler32 of the preset dictionary: */ + if (s->strstart != 0) { + putShortMSB(s, (uInt)(strm->adler >> 16)); + putShortMSB(s, (uInt)(strm->adler & 0xffff)); + } + strm->adler = adler32(0L, Z_NULL, 0); + s->status = BUSY_STATE; + + /* Compression must start with an empty pending buffer */ + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + } +#ifdef GZIP + if (s->status == GZIP_STATE) { + /* gzip header */ + crc_reset(s); + put_byte(s, 31); + put_byte(s, 139); + put_byte(s, 8); + if (s->gzhead == Z_NULL) { + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, s->level == 9 ? 2 : + (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? + 4 : 0)); + put_byte(s, OS_CODE); s->status = BUSY_STATE; - putShortMSB(s, header); - /* Save the adler32 of the preset dictionary: */ - if (s->strstart != 0) { - putShortMSB(s, (uInt)(strm->adler >> 16)); - putShortMSB(s, (uInt)(strm->adler & 0xffff)); + /* Compression must start with an empty pending buffer */ + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; } - strm->adler = adler32(0L, Z_NULL, 0); + } + else { + put_byte(s, (s->gzhead->text ? 1 : 0) + + (s->gzhead->hcrc ? 2 : 0) + + (s->gzhead->extra == Z_NULL ? 0 : 4) + + (s->gzhead->name == Z_NULL ? 0 : 8) + + (s->gzhead->comment == Z_NULL ? 0 : 16) + ); + put_byte(s, (Byte)(s->gzhead->time & 0xff)); + put_byte(s, (Byte)((s->gzhead->time >> 8) & 0xff)); + put_byte(s, (Byte)((s->gzhead->time >> 16) & 0xff)); + put_byte(s, (Byte)((s->gzhead->time >> 24) & 0xff)); + put_byte(s, s->level == 9 ? 2 : + (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? + 4 : 0)); + put_byte(s, s->gzhead->os & 0xff); + if (s->gzhead->extra != Z_NULL) { + put_byte(s, s->gzhead->extra_len & 0xff); + put_byte(s, (s->gzhead->extra_len >> 8) & 0xff); + } + if (s->gzhead->hcrc) + strm->adler = crc32(strm->adler, s->pending_buf, + s->pending); + s->gzindex = 0; + s->status = EXTRA_STATE; } } -#ifdef GZIP if (s->status == EXTRA_STATE) { if (s->gzhead->extra != Z_NULL) { - uInt beg = s->pending; /* start of bytes to update crc */ - - while (s->gzindex < (s->gzhead->extra_len & 0xffff)) { - if (s->pending == s->pending_buf_size) { - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - flush_pending(strm); - beg = s->pending; - if (s->pending == s->pending_buf_size) - break; + ulg beg = s->pending; /* start of bytes to update crc */ + uInt left = (s->gzhead->extra_len & 0xffff) - s->gzindex; + while (s->pending + left > s->pending_buf_size) { + uInt copy = s->pending_buf_size - s->pending; + zmemcpy(s->pending_buf + s->pending, + s->gzhead->extra + s->gzindex, copy); + s->pending = s->pending_buf_size; + HCRC_UPDATE(beg); + s->gzindex += copy; + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; } - put_byte(s, s->gzhead->extra[s->gzindex]); - s->gzindex++; - } - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - if (s->gzindex == s->gzhead->extra_len) { - s->gzindex = 0; - s->status = NAME_STATE; + beg = 0; + left -= copy; } + zmemcpy(s->pending_buf + s->pending, + s->gzhead->extra + s->gzindex, left); + s->pending += left; + HCRC_UPDATE(beg); + s->gzindex = 0; } - else - s->status = NAME_STATE; + s->status = NAME_STATE; } if (s->status == NAME_STATE) { if (s->gzhead->name != Z_NULL) { - uInt beg = s->pending; /* start of bytes to update crc */ + ulg beg = s->pending; /* start of bytes to update crc */ int val; - do { if (s->pending == s->pending_buf_size) { - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); + HCRC_UPDATE(beg); flush_pending(strm); - beg = s->pending; - if (s->pending == s->pending_buf_size) { - val = 1; - break; + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; } + beg = 0; } val = s->gzhead->name[s->gzindex++]; put_byte(s, val); } while (val != 0); - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - if (val == 0) { - s->gzindex = 0; - s->status = COMMENT_STATE; - } + HCRC_UPDATE(beg); + s->gzindex = 0; } - else - s->status = COMMENT_STATE; + s->status = COMMENT_STATE; } if (s->status == COMMENT_STATE) { if (s->gzhead->comment != Z_NULL) { - uInt beg = s->pending; /* start of bytes to update crc */ + ulg beg = s->pending; /* start of bytes to update crc */ int val; - do { if (s->pending == s->pending_buf_size) { - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); + HCRC_UPDATE(beg); flush_pending(strm); - beg = s->pending; - if (s->pending == s->pending_buf_size) { - val = 1; - break; + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; } + beg = 0; } val = s->gzhead->comment[s->gzindex++]; put_byte(s, val); } while (val != 0); - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - if (val == 0) - s->status = HCRC_STATE; + HCRC_UPDATE(beg); } - else - s->status = HCRC_STATE; + s->status = HCRC_STATE; } if (s->status == HCRC_STATE) { if (s->gzhead->hcrc) { - if (s->pending + 2 > s->pending_buf_size) + if (s->pending + 2 > s->pending_buf_size) { flush_pending(strm); - if (s->pending + 2 <= s->pending_buf_size) { - put_byte(s, (Byte)(strm->adler & 0xff)); - put_byte(s, (Byte)((strm->adler >> 8) & 0xff)); - strm->adler = crc32(0L, Z_NULL, 0); - s->status = BUSY_STATE; + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } } + put_byte(s, (Byte)(strm->adler & 0xff)); + put_byte(s, (Byte)((strm->adler >> 8) & 0xff)); + strm->adler = crc32(0L, Z_NULL, 0); } - else - s->status = BUSY_STATE; - } -#endif + s->status = BUSY_STATE; - /* Flush as much pending output as possible */ - if (s->pending != 0) { + /* Compression must start with an empty pending buffer */ flush_pending(strm); - if (strm->avail_out == 0) { - /* Since avail_out is 0, deflate will be called again with - * more output space, but possibly with both pending and - * avail_in equal to zero. There won't be anything to do, - * but this is not an error situation so make sure we - * return OK instead of BUF_ERROR at next call of deflate: - */ + if (s->pending != 0) { s->last_flush = -1; return Z_OK; } - - /* Make sure there is something to do and avoid duplicate consecutive - * flushes. For repeated and useless calls with Z_FINISH, we keep - * returning Z_STREAM_END instead of Z_BUF_ERROR. - */ - } else if (strm->avail_in == 0 && flush <= old_flush && - flush != Z_FINISH) { - ERR_RETURN(strm, Z_BUF_ERROR); - } - - /* User must not provide more input after the first FINISH: */ - if (s->status == FINISH_STATE && strm->avail_in != 0) { - ERR_RETURN(strm, Z_BUF_ERROR); } +#endif /* Start a new block or continue the current one. */ @@ -861,26 +1036,10 @@ int ZEXPORT deflate (strm, flush) (flush != Z_NO_FLUSH && s->status != FINISH_STATE)) { block_state bstate; - if (strm->clas && s->class_bitmap == NULL) { - /* This is the first time that we have seen alternative class - * data. All data up till this point has been standard class. */ - s->class_bitmap = (Bytef*) ZALLOC(strm, s->w_size/4, sizeof(Byte)); - zmemzero(s->class_bitmap, s->w_size/4); - } - - if (strm->clas && s->strategy == Z_RLE) { - /* We haven't patched deflate_rle. */ - ERR_RETURN(strm, Z_BUF_ERROR); - } - - if (s->strategy == Z_HUFFMAN_ONLY) { - bstate = deflate_huff(s, flush); - } else if (s->strategy == Z_RLE) { - bstate = deflate_rle(s, flush); - } else { - bstate = (*(configuration_table[s->level].func)) - (s, flush, strm->clas); - } + bstate = s->level == 0 ? deflate_stored(s, flush) : + s->strategy == Z_HUFFMAN_ONLY ? deflate_huff(s, flush) : + s->strategy == Z_RLE ? deflate_rle(s, flush) : + (*(configuration_table[s->level].func))(s, flush); if (bstate == finish_started || bstate == finish_done) { s->status = FINISH_STATE; @@ -911,6 +1070,7 @@ int ZEXPORT deflate (strm, flush) if (s->lookahead == 0) { s->strstart = 0; s->block_start = 0L; + s->insert = 0; } } } @@ -921,7 +1081,6 @@ int ZEXPORT deflate (strm, flush) } } } - Assert(strm->avail_out > 0, "bug2"); if (flush != Z_FINISH) return Z_OK; if (s->wrap <= 0) return Z_STREAM_END; @@ -959,25 +1118,15 @@ int ZEXPORT deflateEnd (strm) { int status; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; status = strm->state->status; - if (status != INIT_STATE && - status != EXTRA_STATE && - status != NAME_STATE && - status != COMMENT_STATE && - status != HCRC_STATE && - status != BUSY_STATE && - status != FINISH_STATE) { - return Z_STREAM_ERROR; - } /* Deallocate in reverse order of allocations: */ TRY_FREE(strm, strm->state->pending_buf); TRY_FREE(strm, strm->state->head); TRY_FREE(strm, strm->state->prev); TRY_FREE(strm, strm->state->window); - TRY_FREE(strm, strm->state->class_bitmap); ZFREE(strm, strm->state); strm->state = Z_NULL; @@ -999,28 +1148,26 @@ int ZEXPORT deflateCopy (dest, source) #else deflate_state *ds; deflate_state *ss; - ushf *overlay; - if (source == Z_NULL || dest == Z_NULL || source->state == Z_NULL) { + if (deflateStateCheck(source) || dest == Z_NULL) { return Z_STREAM_ERROR; } ss = source->state; - zmemcpy(dest, source, sizeof(z_stream)); + zmemcpy((voidpf)dest, (voidpf)source, sizeof(z_stream)); ds = (deflate_state *) ZALLOC(dest, 1, sizeof(deflate_state)); if (ds == Z_NULL) return Z_MEM_ERROR; dest->state = (struct internal_state FAR *) ds; - zmemcpy(ds, ss, sizeof(deflate_state)); + zmemcpy((voidpf)ds, (voidpf)ss, sizeof(deflate_state)); ds->strm = dest; ds->window = (Bytef *) ZALLOC(dest, ds->w_size, 2*sizeof(Byte)); ds->prev = (Posf *) ZALLOC(dest, ds->w_size, sizeof(Pos)); ds->head = (Posf *) ZALLOC(dest, ds->hash_size, sizeof(Pos)); - overlay = (ushf *) ZALLOC(dest, ds->lit_bufsize, sizeof(ush)+2); - ds->pending_buf = (uchf *) overlay; + ds->pending_buf = (uchf *) ZALLOC(dest, ds->lit_bufsize, 4); if (ds->window == Z_NULL || ds->prev == Z_NULL || ds->head == Z_NULL || ds->pending_buf == Z_NULL) { @@ -1029,13 +1176,12 @@ int ZEXPORT deflateCopy (dest, source) } /* following zmemcpy do not work for 16-bit MSDOS */ zmemcpy(ds->window, ss->window, ds->w_size * 2 * sizeof(Byte)); - zmemcpy(ds->prev, ss->prev, ds->w_size * sizeof(Pos)); - zmemcpy(ds->head, ss->head, ds->hash_size * sizeof(Pos)); + zmemcpy((voidpf)ds->prev, (voidpf)ss->prev, ds->w_size * sizeof(Pos)); + zmemcpy((voidpf)ds->head, (voidpf)ss->head, ds->hash_size * sizeof(Pos)); zmemcpy(ds->pending_buf, ss->pending_buf, (uInt)ds->pending_buf_size); ds->pending_out = ds->pending_buf + (ss->pending_out - ss->pending_buf); - ds->d_buf = overlay + ds->lit_bufsize/sizeof(ush); - ds->l_buf = ds->pending_buf + (1+sizeof(ush))*ds->lit_bufsize; + ds->sym_buf = ds->pending_buf + ds->lit_bufsize; ds->l_desc.dyn_tree = ds->dyn_ltree; ds->d_desc.dyn_tree = ds->dyn_dtree; @@ -1052,7 +1198,7 @@ int ZEXPORT deflateCopy (dest, source) * allocating a large strm->next_in buffer and copying from it. * (See also flush_pending()). */ -ZLIB_INTERNAL int read_buf(strm, buf, size) +ZLIB_INTERNAL unsigned deflate_read_buf(strm, buf, size) z_streamp strm; Bytef *buf; unsigned size; @@ -1065,9 +1211,8 @@ ZLIB_INTERNAL int read_buf(strm, buf, size) strm->avail_in -= len; #ifdef GZIP - if (strm->state->wrap == 2) { + if (strm->state->wrap == 2) copy_with_crc(strm, buf, len); - } else #endif { @@ -1078,7 +1223,7 @@ ZLIB_INTERNAL int read_buf(strm, buf, size) strm->next_in += len; strm->total_in += len; - return (int)len; + return len; } /* =========================================================================== @@ -1101,6 +1246,7 @@ local void lm_init (s) s->strstart = 0; s->block_start = 0L; s->lookahead = 0; + s->insert = 0; s->match_length = s->prev_length = MIN_MATCH-1; s->match_available = 0; s->ins_h = 0; @@ -1111,57 +1257,6 @@ local void lm_init (s) #endif } -/* class_set sets bits [offset,offset+len) in s->class_bitmap to either 1 (if - * class != 0) or 0 (otherwise). */ -local void class_set(s, offset, len, clas) - deflate_state *s; - IPos offset; - uInt len; - int clas; -{ - IPos byte = offset >> 3; - IPos bit = offset & 7; - Bytef class_byte_value = clas ? 0xff : 0x00; - Bytef class_bit_value = clas ? 1 : 0; - static const Bytef mask[8] = {0xfe, 0xfd, 0xfb, 0xf7, - 0xef, 0xdf, 0xbf, 0x7f}; - - if (bit) { - while (len) { - s->class_bitmap[byte] &= mask[bit]; - s->class_bitmap[byte] |= class_bit_value << bit; - bit++; - len--; - if (bit == 8) { - bit = 0; - byte++; - break; - } - } - } - - while (len >= 8) { - s->class_bitmap[byte++] = class_byte_value; - len -= 8; - } - - while (len) { - s->class_bitmap[byte] &= mask[bit]; - s->class_bitmap[byte] |= class_bit_value << bit; - bit++; - len--; - } -} - -local int class_at(s, window_offset) - deflate_state *s; - IPos window_offset; -{ - IPos byte = window_offset >> 3; - IPos bit = window_offset & 7; - return (s->class_bitmap[byte] >> bit) & 1; -} - #ifndef FASTEST /* =========================================================================== * Set match_start to the longest match starting at the given string and @@ -1176,16 +1271,15 @@ local int class_at(s, window_offset) /* For 80x86 and 680x0, an optimized version will be provided in match.asm or * match.S. The code will be functionally equivalent. */ -local uInt longest_match(s, cur_match, clas) +local uInt longest_match(s, cur_match) deflate_state *s; IPos cur_match; /* current match */ - int clas; { unsigned chain_length = s->max_chain_length;/* max hash chain length */ register Bytef *scan = s->window + s->strstart; /* current string */ - register Bytef *match; /* matched string */ + register Bytef *match; /* matched string */ register int len; /* length of current match */ - int best_len = s->prev_length; /* best match length so far */ + int best_len = (int)s->prev_length; /* best match length so far */ int nice_match = s->nice_match; /* stop if match long enough */ IPos limit = s->strstart > (IPos)MAX_DIST(s) ? s->strstart - (IPos)MAX_DIST(s) : NIL; @@ -1220,16 +1314,13 @@ local uInt longest_match(s, cur_match, clas) /* Do not look for matches beyond the end of the input. This is necessary * to make deflate deterministic. */ - if ((uInt)nice_match > s->lookahead) nice_match = s->lookahead; + if ((uInt)nice_match > s->lookahead) nice_match = (int)s->lookahead; Assert((ulg)s->strstart <= s->window_size-MIN_LOOKAHEAD, "need lookahead"); do { Assert(cur_match < s->strstart, "no future"); match = s->window + cur_match; - /* If the matched data is in the wrong class, skip it. */ - if (s->class_bitmap && class_at(s, cur_match) != clas) - continue; /* Skip to next match if the match length cannot increase * or if the match length is less than 2. Note that the checks below @@ -1272,8 +1363,6 @@ local uInt longest_match(s, cur_match, clas) len = (MAX_MATCH - 1) - (int)(strend-scan); scan = strend - (MAX_MATCH-1); -#error "UNALIGNED_OK hasn't been patched." - #else /* UNALIGNED_OK */ if (match[best_len] != scan_end || @@ -1290,23 +1379,15 @@ local uInt longest_match(s, cur_match, clas) scan += 2, match++; Assert(*scan == *match, "match[2]?"); - if (!s->class_bitmap) { - /* We check for insufficient lookahead only every 8th comparison; - * the 256th check will be made at strstart+258. - */ - do { - } while (*++scan == *++match && *++scan == *++match && - *++scan == *++match && *++scan == *++match && - *++scan == *++match && *++scan == *++match && - *++scan == *++match && *++scan == *++match && - scan < strend); - } else { - /* We have to be mindful of the class of the data and not stray. */ - do { - } while (*++scan == *++match && - class_at(s, match - s->window) == clas && - scan < strend); - } + /* We check for insufficient lookahead only every 8th comparison; + * the 256th check will be made at strstart+258. + */ + do { + } while (*++scan == *++match && *++scan == *++match && + *++scan == *++match && *++scan == *++match && + *++scan == *++match && *++scan == *++match && + *++scan == *++match && *++scan == *++match && + scan < strend); Assert(scan <= s->window+(unsigned)(s->window_size-1), "wild scan"); @@ -1334,74 +1415,20 @@ local uInt longest_match(s, cur_match, clas) } #endif /* ASMV */ -/* cookie_match is a replacement for longest_match in the case of cookie data. - * Here we only wish to match the entire value so trying the partial matches in - * longest_match is both wasteful and often fails to find the correct match. - * - * So we take the djb2 hash of the cookie and look up the last position for a - * match in a special hash table. */ -local uInt cookie_match(s, start, len) - deflate_state *s; - IPos start; - unsigned len; -{ - unsigned hash = 5381; - Bytef *str = s->window + start; - unsigned i; - IPos cookie_location; - - if (len >= MAX_MATCH || len == 0) - return 0; - - for (i = 0; i < len; i++) - hash = ((hash << 5) + hash) + str[i]; - - hash &= Z_COOKIE_HASH_MASK; - cookie_location = s->cookie_locations[hash]; - s->cookie_locations[hash] = start; - s->match_start = 0; - if (cookie_location && - (start - cookie_location) > len && - (start - cookie_location) < MAX_DIST(s) && - len <= s->lookahead) { - for (i = 0; i < len; i++) { - if (s->window[start+i] != s->window[cookie_location+i] || - class_at(s, cookie_location+i) != 1) { - return 0; - } - } - /* Check that we aren't matching a prefix of another cookie by ensuring - * that the final byte is either a semicolon (which cannot appear in a - * cookie value), or the match is followed by non-cookie data. */ - if (s->window[cookie_location+len-1] != ';' && - class_at(s, cookie_location+len) != 0) { - return 0; - } - s->match_start = cookie_location; - return len; - } - - return 0; -} - - #else /* FASTEST */ /* --------------------------------------------------------------------------- * Optimized version for FASTEST only */ -local uInt longest_match(s, cur_match, clas) +local uInt longest_match(s, cur_match) deflate_state *s; IPos cur_match; /* current match */ - int clas; { register Bytef *scan = s->window + s->strstart; /* current string */ register Bytef *match; /* matched string */ register int len; /* length of current match */ register Bytef *strend = s->window + s->strstart + MAX_MATCH; -#error "This code not patched" - /* The code is optimized for HASH_BITS >= 8 and MAX_MATCH-2 multiple of 16. * It is easy to get rid of this optimization if necessary. */ @@ -1448,7 +1475,11 @@ local uInt longest_match(s, cur_match, clas) #endif /* FASTEST */ -#ifdef DEBUG +#ifdef ZLIB_DEBUG + +#define EQUAL 0 +/* result of memcmp for equal strings */ + /* =========================================================================== * Check that the match at match_start is indeed a match. */ @@ -1474,7 +1505,7 @@ local void check_match(s, start, match, length) } #else # define check_match(s, start, match, length) -#endif /* DEBUG */ +#endif /* ZLIB_DEBUG */ /* =========================================================================== * Fill the window when the lookahead becomes insufficient. @@ -1490,22 +1521,24 @@ local void fill_window_c(deflate_state *s); local void fill_window(deflate_state *s) { +#ifdef DEFLATE_FILL_WINDOW_SSE2 if (x86_cpu_enable_simd) { fill_window_sse(s); return; } - +#endif fill_window_c(s); } local void fill_window_c(s) deflate_state *s; { - register unsigned n, m; - register Posf *p; + unsigned n; unsigned more; /* Amount of free space at the end of the window. */ uInt wsize = s->w_size; + Assert(s->lookahead < MIN_LOOKAHEAD, "already enough lookahead"); + do { more = (unsigned)(s->window_size -(ulg)s->lookahead -(ulg)s->strstart); @@ -1527,53 +1560,14 @@ local void fill_window_c(s) */ if (s->strstart >= wsize+MAX_DIST(s)) { - zmemcpy(s->window, s->window+wsize, (unsigned)wsize); + zmemcpy(s->window, s->window+wsize, (unsigned)wsize - more); s->match_start -= wsize; s->strstart -= wsize; /* we now have strstart >= MAX_DIST */ s->block_start -= (long) wsize; - - /* Slide the hash table (could be avoided with 32 bit values - at the expense of memory usage). We slide even when level == 0 - to keep the hash table consistent if we switch back to level > 0 - later. (Using level 0 permanently is not an optimal usage of - zlib, so we don't care about this pathological case.) - */ - n = s->hash_size; - p = &s->head[n]; - do { - m = *--p; - *p = (Pos)(m >= wsize ? m-wsize : NIL); - } while (--n); - - n = wsize; -#ifndef FASTEST - p = &s->prev[n]; - do { - m = *--p; - *p = (Pos)(m >= wsize ? m-wsize : NIL); - /* If n is not on any hash chain, prev[n] is garbage but - * its value will never be used. - */ - } while (--n); -#endif - - for (n = 0; n < Z_COOKIE_HASH_SIZE; n++) { - if (s->cookie_locations[n] > wsize) { - s->cookie_locations[n] -= wsize; - } else { - s->cookie_locations[n] = 0; - } - } - - if (s->class_bitmap) { - zmemcpy(s->class_bitmap, s->class_bitmap + s->w_size/8, - s->w_size/8); - zmemzero(s->class_bitmap + s->w_size/8, s->w_size/8); - } - + slide_hash(s); more += wsize; } - if (s->strm->avail_in == 0) return; + if (s->strm->avail_in == 0) break; /* If there was no sliding: * strstart <= WSIZE+MAX_DIST-1 && lookahead <= MIN_LOOKAHEAD - 1 && @@ -1588,19 +1582,28 @@ local void fill_window_c(s) */ Assert(more >= 2, "more < 2"); - n = read_buf(s->strm, s->window + s->strstart + s->lookahead, more); - if (s->class_bitmap != NULL) { - class_set(s, s->strstart + s->lookahead, n, s->strm->clas); - } + n = deflate_read_buf(s->strm, s->window + s->strstart + s->lookahead, more); s->lookahead += n; /* Initialize the hash value now that we have some input: */ - if (s->lookahead >= MIN_MATCH) { - s->ins_h = s->window[s->strstart]; - UPDATE_HASH(s, s->ins_h, s->window[s->strstart+1]); + if (s->lookahead + s->insert >= MIN_MATCH) { + uInt str = s->strstart - s->insert; + s->ins_h = s->window[str]; + UPDATE_HASH(s, s->ins_h, s->window[str + 1]); #if MIN_MATCH != 3 Call UPDATE_HASH() MIN_MATCH-3 more times #endif + while (s->insert) { + UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]); +#ifndef FASTEST + s->prev[str & s->w_mask] = s->head[s->ins_h]; +#endif + s->head[s->ins_h] = (Pos)str; + str++; + s->insert--; + if (s->lookahead + s->insert < MIN_MATCH) + break; + } } /* If the whole input has less than MIN_MATCH bytes, ins_h is garbage, * but this is not important since only literal bytes will be emitted. @@ -1641,6 +1644,9 @@ local void fill_window_c(s) s->high_water += init; } } + + Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD, + "not enough room for search"); } /* =========================================================================== @@ -1664,65 +1670,199 @@ local void fill_window_c(s) if (s->strm->avail_out == 0) return (last) ? finish_started : need_more; \ } +/* Maximum stored block length in deflate format (not including header). */ +#define MAX_STORED 65535 + +/* Minimum of a and b. */ +#define MIN(a, b) ((a) > (b) ? (b) : (a)) + /* =========================================================================== * Copy without compression as much as possible from the input stream, return * the current block state. - * This function does not insert new strings in the dictionary since - * uncompressible data is probably not useful. This function is used - * only for the level=0 compression option. - * NOTE: this function should be optimized to avoid extra copying from - * window to pending_buf. + * + * In case deflateParams() is used to later switch to a non-zero compression + * level, s->matches (otherwise unused when storing) keeps track of the number + * of hash table slides to perform. If s->matches is 1, then one hash table + * slide will be done when switching. If s->matches is 2, the maximum value + * allowed here, then the hash table will be cleared, since two or more slides + * is the same as a clear. + * + * deflate_stored() is written to minimize the number of times an input byte is + * copied. It is most efficient with large input and output buffers, which + * maximizes the opportunites to have a single copy from next_in to next_out. */ -local block_state deflate_stored(s, flush, clas) +local block_state deflate_stored(s, flush) deflate_state *s; int flush; - int clas; { - /* Stored blocks are limited to 0xffff bytes, pending_buf is limited - * to pending_buf_size, and each stored block has a 5 byte header: + /* Smallest worthy block size when not flushing or finishing. By default + * this is 32K. This can be as small as 507 bytes for memLevel == 1. For + * large input and output buffers, the stored block size will be larger. */ - ulg max_block_size = 0xffff; - ulg max_start; - - if (max_block_size > s->pending_buf_size - 5) { - max_block_size = s->pending_buf_size - 5; - } + unsigned min_block = MIN(s->pending_buf_size - 5, s->w_size); - /* Copy as much as possible from input to output: */ - for (;;) { - /* Fill the window as much as possible: */ - if (s->lookahead <= 1) { - - Assert(s->strstart < s->w_size+MAX_DIST(s) || - s->block_start >= (long)s->w_size, "slide too late"); + /* Copy as many min_block or larger stored blocks directly to next_out as + * possible. If flushing, copy the remaining available input to next_out as + * stored blocks, if there is enough space. + */ + unsigned len, left, have, last = 0; + unsigned used = s->strm->avail_in; + do { + /* Set len to the maximum size block that we can copy directly with the + * available input data and output space. Set left to how much of that + * would be copied from what's left in the window. + */ + len = MAX_STORED; /* maximum deflate stored block length */ + have = (s->bi_valid + 42) >> 3; /* number of header bytes */ + if (s->strm->avail_out < have) /* need room for header */ + break; + /* maximum stored block length that will fit in avail_out: */ + have = s->strm->avail_out - have; + left = s->strstart - s->block_start; /* bytes left in window */ + if (len > (ulg)left + s->strm->avail_in) + len = left + s->strm->avail_in; /* limit len to the input */ + if (len > have) + len = have; /* limit len to the output */ + + /* If the stored block would be less than min_block in length, or if + * unable to copy all of the available input when flushing, then try + * copying to the window and the pending buffer instead. Also don't + * write an empty block when flushing -- deflate() does that. + */ + if (len < min_block && ((len == 0 && flush != Z_FINISH) || + flush == Z_NO_FLUSH || + len != left + s->strm->avail_in)) + break; - fill_window(s); - if (s->lookahead == 0 && flush == Z_NO_FLUSH) return need_more; + /* Make a dummy stored block in pending to get the header bytes, + * including any pending bits. This also updates the debugging counts. + */ + last = flush == Z_FINISH && len == left + s->strm->avail_in ? 1 : 0; + _tr_stored_block(s, (char *)0, 0L, last); + + /* Replace the lengths in the dummy stored block with len. */ + s->pending_buf[s->pending - 4] = len; + s->pending_buf[s->pending - 3] = len >> 8; + s->pending_buf[s->pending - 2] = ~len; + s->pending_buf[s->pending - 1] = ~len >> 8; + + /* Write the stored block header bytes. */ + flush_pending(s->strm); + +#ifdef ZLIB_DEBUG + /* Update debugging counts for the data about to be copied. */ + s->compressed_len += len << 3; + s->bits_sent += len << 3; +#endif - if (s->lookahead == 0) break; /* flush the current block */ + /* Copy uncompressed bytes from the window to next_out. */ + if (left) { + if (left > len) + left = len; + zmemcpy(s->strm->next_out, s->window + s->block_start, left); + s->strm->next_out += left; + s->strm->avail_out -= left; + s->strm->total_out += left; + s->block_start += left; + len -= left; } - Assert(s->block_start >= 0L, "block gone"); - - s->strstart += s->lookahead; - s->lookahead = 0; - - /* Emit a stored block if pending_buf will be full: */ - max_start = s->block_start + max_block_size; - if (s->strstart == 0 || (ulg)s->strstart >= max_start) { - /* strstart == 0 is possible when wraparound on 16-bit machine */ - s->lookahead = (uInt)(s->strstart - max_start); - s->strstart = (uInt)max_start; - FLUSH_BLOCK(s, 0); + + /* Copy uncompressed bytes directly from next_in to next_out, updating + * the check value. + */ + if (len) { + deflate_read_buf(s->strm, s->strm->next_out, len); + s->strm->next_out += len; + s->strm->avail_out -= len; + s->strm->total_out += len; } - /* Flush if we may have to slide, otherwise block_start may become - * negative and the data will be gone: + } while (last == 0); + + /* Update the sliding window with the last s->w_size bytes of the copied + * data, or append all of the copied data to the existing window if less + * than s->w_size bytes were copied. Also update the number of bytes to + * insert in the hash tables, in the event that deflateParams() switches to + * a non-zero compression level. + */ + used -= s->strm->avail_in; /* number of input bytes directly copied */ + if (used) { + /* If any input was used, then no unused input remains in the window, + * therefore s->block_start == s->strstart. */ - if (s->strstart - (uInt)s->block_start >= MAX_DIST(s)) { - FLUSH_BLOCK(s, 0); + if (used >= s->w_size) { /* supplant the previous history */ + s->matches = 2; /* clear hash */ + zmemcpy(s->window, s->strm->next_in - s->w_size, s->w_size); + s->strstart = s->w_size; + } + else { + if (s->window_size - s->strstart <= used) { + /* Slide the window down. */ + s->strstart -= s->w_size; + zmemcpy(s->window, s->window + s->w_size, s->strstart); + if (s->matches < 2) + s->matches++; /* add a pending slide_hash() */ + } + zmemcpy(s->window + s->strstart, s->strm->next_in - used, used); + s->strstart += used; } + s->block_start = s->strstart; + s->insert += MIN(used, s->w_size - s->insert); + } + if (s->high_water < s->strstart) + s->high_water = s->strstart; + + /* If the last block was written to next_out, then done. */ + if (last) + return finish_done; + + /* If flushing and all input has been consumed, then done. */ + if (flush != Z_NO_FLUSH && flush != Z_FINISH && + s->strm->avail_in == 0 && (long)s->strstart == s->block_start) + return block_done; + + /* Fill the window with any remaining input. */ + have = s->window_size - s->strstart - 1; + if (s->strm->avail_in > have && s->block_start >= (long)s->w_size) { + /* Slide the window down. */ + s->block_start -= s->w_size; + s->strstart -= s->w_size; + zmemcpy(s->window, s->window + s->w_size, s->strstart); + if (s->matches < 2) + s->matches++; /* add a pending slide_hash() */ + have += s->w_size; /* more space now */ + } + if (have > s->strm->avail_in) + have = s->strm->avail_in; + if (have) { + deflate_read_buf(s->strm, s->window + s->strstart, have); + s->strstart += have; + } + if (s->high_water < s->strstart) + s->high_water = s->strstart; + + /* There was not enough avail_out to write a complete worthy or flushed + * stored block to next_out. Write a stored block to pending instead, if we + * have enough input for a worthy block, or if flushing and there is enough + * room for the remaining input as a stored block in the pending buffer. + */ + have = (s->bi_valid + 42) >> 3; /* number of header bytes */ + /* maximum stored block length that will fit in pending: */ + have = MIN(s->pending_buf_size - have, MAX_STORED); + min_block = MIN(have, s->w_size); + left = s->strstart - s->block_start; + if (left >= min_block || + ((left || flush == Z_FINISH) && flush != Z_NO_FLUSH && + s->strm->avail_in == 0 && left <= have)) { + len = MIN(left, have); + last = flush == Z_FINISH && s->strm->avail_in == 0 && + len == left ? 1 : 0; + _tr_stored_block(s, (charf *)s->window + s->block_start, len, last); + s->block_start += len; + flush_pending(s->strm); } - FLUSH_BLOCK(s, flush == Z_FINISH); - return flush == Z_FINISH ? finish_done : block_done; + + /* We've done all we can with the available input and output. */ + return last ? finish_started : need_more; } /* =========================================================================== @@ -1732,19 +1872,13 @@ local block_state deflate_stored(s, flush, clas) * new strings in the dictionary only for unmatched strings or for short * matches. It is used only for the fast compression options. */ -local block_state deflate_fast(s, flush, clas) +local block_state deflate_fast(s, flush) deflate_state *s; int flush; - int clas; { IPos hash_head; /* head of the hash chain */ int bflush; /* set if current block must be flushed */ - if (clas != 0) { - /* We haven't patched this code for alternative class data. */ - return Z_BUF_ERROR; - } - for (;;) { /* Make sure that we always have enough lookahead, except * at the end of the input file. We need MAX_MATCH bytes @@ -1775,7 +1909,7 @@ local block_state deflate_fast(s, flush, clas) * of window index 0 (in particular we have to avoid a match * of the string with itself at the start of the input file). */ - s->match_length = longest_match (s, hash_head, clas); + s->match_length = longest_match (s, hash_head); /* longest_match() sets match_start */ } if (s->match_length >= MIN_MATCH) { @@ -1824,8 +1958,14 @@ local block_state deflate_fast(s, flush, clas) } if (bflush) FLUSH_BLOCK(s, 0); } - FLUSH_BLOCK(s, flush == Z_FINISH); - return flush == Z_FINISH ? finish_done : block_done; + s->insert = s->strstart < MIN_MATCH-1 ? s->strstart : MIN_MATCH-1; + if (flush == Z_FINISH) { + FLUSH_BLOCK(s, 1); + return finish_done; + } + if (s->sym_next) + FLUSH_BLOCK(s, 0); + return block_done; } #ifndef FASTEST @@ -1834,25 +1974,12 @@ local block_state deflate_fast(s, flush, clas) * evaluation for matches: a match is finally adopted only if there is * no better match at the next window position. */ -local block_state deflate_slow(s, flush, clas) +local block_state deflate_slow(s, flush) deflate_state *s; int flush; - int clas; { IPos hash_head; /* head of hash chain */ int bflush; /* set if current block must be flushed */ - uInt input_length ; - int first = 1; /* first says whether this is the first iteration - of the loop, below. */ - - if (clas == Z_CLASS_COOKIE) { - if (s->lookahead) { - /* Alternative class data must always be presented at the beginning - * of a block. */ - return Z_BUF_ERROR; - } - input_length = s->strm->avail_in; - } /* Process the input block. */ for (;;) { @@ -1882,18 +2009,13 @@ local block_state deflate_slow(s, flush, clas) s->prev_length = s->match_length, s->prev_match = s->match_start; s->match_length = MIN_MATCH-1; - if (clas == Z_CLASS_COOKIE && first) { - s->match_length = cookie_match(s, s->strstart, input_length); - } else if (clas == Z_CLASS_STANDARD && - hash_head != NIL && - s->prev_length < s->max_lazy_match && - s->strstart - hash_head <= MAX_DIST(s)) { + if (hash_head != NIL && s->prev_length < s->max_lazy_match && + s->strstart - hash_head <= MAX_DIST(s)) { /* To simplify the code, we prevent matches with the string * of window index 0 (in particular we have to avoid a match * of the string with itself at the start of the input file). */ - s->match_length = longest_match (s, hash_head, clas); - + s->match_length = longest_match (s, hash_head); /* longest_match() sets match_start */ if (s->match_length <= 5 && (s->strategy == Z_FILTERED @@ -1912,20 +2034,7 @@ local block_state deflate_slow(s, flush, clas) /* If there was a match at the previous step and the current * match is not better, output the previous match: */ - first = 0; - if (s->prev_length >= MIN_MATCH && s->match_length <= s->prev_length && - /* We will only accept an exact match for Z_CLASS_COOKIE data and - * we won't match Z_CLASS_HUFFMAN_ONLY data at all. */ - (clas == Z_CLASS_STANDARD || (clas == Z_CLASS_COOKIE && - s->prev_length == input_length && - s->prev_match > 0 && - /* We require that a Z_CLASS_COOKIE match be - * preceded by either a semicolon (which cannot be - * part of a cookie), or non-cookie data. This is - * to prevent a cookie from being a suffix of - * another. */ - (class_at(s, s->prev_match-1) == Z_CLASS_STANDARD || - *(s->window + s->prev_match-1) == ';')))) { + if (s->prev_length >= MIN_MATCH && s->match_length <= s->prev_length) { uInt max_insert = s->strstart + s->lookahead - MIN_MATCH; /* Do not insert strings in hash table beyond this. */ @@ -1980,8 +2089,14 @@ local block_state deflate_slow(s, flush, clas) _tr_tally_lit(s, s->window[s->strstart-1], bflush); s->match_available = 0; } - FLUSH_BLOCK(s, flush == Z_FINISH); - return flush == Z_FINISH ? finish_done : block_done; + s->insert = s->strstart < MIN_MATCH-1 ? s->strstart : MIN_MATCH-1; + if (flush == Z_FINISH) { + FLUSH_BLOCK(s, 1); + return finish_done; + } + if (s->sym_next) + FLUSH_BLOCK(s, 0); + return block_done; } #endif /* FASTEST */ @@ -2001,11 +2116,11 @@ local block_state deflate_rle(s, flush) for (;;) { /* Make sure that we always have enough lookahead, except * at the end of the input file. We need MAX_MATCH bytes - * for the longest encodable run. + * for the longest run, plus one for the unrolled loop. */ - if (s->lookahead < MAX_MATCH) { + if (s->lookahead <= MAX_MATCH) { fill_window(s); - if (s->lookahead < MAX_MATCH && flush == Z_NO_FLUSH) { + if (s->lookahead <= MAX_MATCH && flush == Z_NO_FLUSH) { return need_more; } if (s->lookahead == 0) break; /* flush the current block */ @@ -2024,10 +2139,11 @@ local block_state deflate_rle(s, flush) prev == *++scan && prev == *++scan && prev == *++scan && prev == *++scan && scan < strend); - s->match_length = MAX_MATCH - (int)(strend - scan); + s->match_length = MAX_MATCH - (uInt)(strend - scan); if (s->match_length > s->lookahead) s->match_length = s->lookahead; } + Assert(scan <= s->window+(uInt)(s->window_size-1), "wild scan"); } /* Emit match if have run of MIN_MATCH or longer, else emit literal */ @@ -2048,8 +2164,14 @@ local block_state deflate_rle(s, flush) } if (bflush) FLUSH_BLOCK(s, 0); } - FLUSH_BLOCK(s, flush == Z_FINISH); - return flush == Z_FINISH ? finish_done : block_done; + s->insert = 0; + if (flush == Z_FINISH) { + FLUSH_BLOCK(s, 1); + return finish_done; + } + if (s->sym_next) + FLUSH_BLOCK(s, 0); + return block_done; } /* =========================================================================== @@ -2081,40 +2203,12 @@ local block_state deflate_huff(s, flush) s->strstart++; if (bflush) FLUSH_BLOCK(s, 0); } - FLUSH_BLOCK(s, flush == Z_FINISH); - return flush == Z_FINISH ? finish_done : block_done; -} - -/* Safe to inline this as GCC/clang will use inline asm and Visual Studio will - * use intrinsic without extra params - */ -local INLINE Pos insert_string_sse(deflate_state *const s, const Pos str) -{ - Pos ret; - unsigned *ip, val, h = 0; - - ip = (unsigned *)&s->window[str]; - val = *ip; - - if (s->level >= 6) - val &= 0xFFFFFF; - -/* Windows clang should use inline asm */ -#if defined(_MSC_VER) && !defined(__clang__) - h = _mm_crc32_u32(h, val); -#elif defined(__i386__) || defined(__amd64__) - __asm__ __volatile__ ( - "crc32 %1,%0\n\t" - : "+r" (h) - : "r" (val) - ); -#else - /* This should never happen */ - assert(0); -#endif - - ret = s->head[h & s->hash_mask]; - s->head[h & s->hash_mask] = str; - s->prev[str & s->w_mask] = ret; - return ret; + s->insert = 0; + if (flush == Z_FINISH) { + FLUSH_BLOCK(s, 1); + return finish_done; + } + if (s->sym_next) + FLUSH_BLOCK(s, 0); + return block_done; } diff --git a/third_party/zlib/deflate.h b/third_party/zlib/deflate.h index d15f2b5d58..78ffb00a29 100644 --- a/third_party/zlib/deflate.h +++ b/third_party/zlib/deflate.h @@ -1,5 +1,5 @@ /* deflate.h -- internal compression state - * Copyright (C) 1995-2010 Jean-loup Gailly + * Copyright (C) 1995-2016 Jean-loup Gailly * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -48,13 +48,19 @@ #define MAX_BITS 15 /* All codes must not exceed MAX_BITS bits */ -#define INIT_STATE 42 -#define EXTRA_STATE 69 -#define NAME_STATE 73 -#define COMMENT_STATE 91 -#define HCRC_STATE 103 -#define BUSY_STATE 113 -#define FINISH_STATE 666 +#define Buf_size 16 +/* size of bit buffer in bi_buf */ + +#define INIT_STATE 42 /* zlib header -> BUSY_STATE */ +#ifdef GZIP +# define GZIP_STATE 57 /* gzip header -> BUSY_STATE | EXTRA_STATE */ +#endif +#define EXTRA_STATE 69 /* gzip extra block -> NAME_STATE */ +#define NAME_STATE 73 /* gzip file name -> COMMENT_STATE */ +#define COMMENT_STATE 91 /* gzip comment -> HCRC_STATE */ +#define HCRC_STATE 103 /* gzip header CRC -> BUSY_STATE */ +#define BUSY_STATE 113 /* deflate -> FINISH_STATE */ +#define FINISH_STATE 666 /* stream complete */ /* Stream status */ @@ -80,7 +86,7 @@ typedef struct static_tree_desc_s static_tree_desc; typedef struct tree_desc_s { ct_data *dyn_tree; /* the dynamic tree */ int max_code; /* largest code with non zero frequency */ - static_tree_desc *stat_desc; /* the corresponding static tree */ + const static_tree_desc *stat_desc; /* the corresponding static tree */ } FAR tree_desc; typedef ush Pos; @@ -91,25 +97,20 @@ typedef unsigned IPos; * save space in the various tables. IPos is used only for parameter passing. */ -#define Z_COOKIE_HASH_SIZE 256 -#define Z_COOKIE_HASH_MASK (Z_COOKIE_HASH_SIZE-1) - typedef struct internal_state { z_streamp strm; /* pointer back to this zlib stream */ int status; /* as the name implies */ Bytef *pending_buf; /* output still pending */ ulg pending_buf_size; /* size of pending_buf */ Bytef *pending_out; /* next pending byte to output to the stream */ - uInt pending; /* nb of bytes in the pending buffer */ + ulg pending; /* nb of bytes in the pending buffer */ int wrap; /* bit 0 true for zlib, bit 1 true for gzip */ gz_headerp gzhead; /* gzip header information to write */ - uInt gzindex; /* where in extra, name, or comment */ - Byte method; /* STORED (for zip only) or DEFLATED */ + ulg gzindex; /* where in extra, name, or comment */ + Byte method; /* can only be DEFLATED */ int last_flush; /* value of flush param for previous deflate call */ - - unsigned zalign(16) crc0[4 * 5]; - - /* used by deflate.c: */ + unsigned crc0[4 * 5]; + /* used by deflate.c: */ uInt w_size; /* LZ77 window size (32K by default) */ uInt w_bits; /* log2(w_size) (8..16) */ @@ -144,8 +145,6 @@ typedef struct internal_state { uInt hash_mask; /* hash_size-1 */ uInt hash_shift; - Bytef *class_bitmap; /* bitmap of class for each byte in window */ - IPos cookie_locations[Z_COOKIE_HASH_SIZE]; /* Number of bits by which ins_h must be shifted at each input * step. It must be such that after MIN_MATCH steps, the oldest * byte no longer takes part in the hash key, that is: @@ -195,7 +194,7 @@ typedef struct internal_state { int nice_match; /* Stop searching when current match exceeds this */ /* used by trees.c: */ - /* Didn't use ct_data typedef below to supress compiler warning */ + /* Didn't use ct_data typedef below to suppress compiler warning */ struct ct_data_s dyn_ltree[HEAP_SIZE]; /* literal and length tree */ struct ct_data_s dyn_dtree[2*D_CODES+1]; /* distance tree */ struct ct_data_s bl_tree[2*BL_CODES+1]; /* Huffman tree for bit lengths */ @@ -218,7 +217,7 @@ typedef struct internal_state { /* Depth of each subtree used as tie breaker for trees of equal frequency */ - uchf *l_buf; /* buffer for literals or lengths */ + uchf *sym_buf; /* buffer for distances and literals/lengths */ uInt lit_bufsize; /* Size of match buffer for literals/lengths. There are 4 reasons for @@ -240,20 +239,15 @@ typedef struct internal_state { * - I can't count above 4 */ - uInt last_lit; /* running index in l_buf */ - - ushf *d_buf; - /* Buffer for distances. To simplify the code, d_buf and l_buf have - * the same number of elements. To use different lengths, an extra flag - * array would be necessary. - */ + uInt sym_next; /* running index in sym_buf */ + uInt sym_end; /* symbol table full when sym_next reaches this */ ulg opt_len; /* bit length of current block with optimal trees */ ulg static_len; /* bit length of current block with static trees */ uInt matches; /* number of string matches in current block */ - int last_eob_len; /* bit length of EOB code for last block */ + uInt insert; /* bytes at end of window left to insert */ -#ifdef DEBUG +#ifdef ZLIB_DEBUG ulg compressed_len; /* total bit length of compressed file mod 2^32 */ ulg bits_sent; /* bit length of compressed data sent mod 2^32 */ #endif @@ -279,7 +273,7 @@ typedef struct internal_state { /* Output a byte on the stream. * IN assertion: there is enough room in pending_buf. */ -#define put_byte(s, c) {s->pending_buf[s->pending++] = (c);} +#define put_byte(s, c) {s->pending_buf[s->pending++] = (Bytef)(c);} #define MIN_LOOKAHEAD (MAX_MATCH+MIN_MATCH+1) @@ -301,6 +295,7 @@ void ZLIB_INTERNAL _tr_init OF((deflate_state *s)); int ZLIB_INTERNAL _tr_tally OF((deflate_state *s, unsigned dist, unsigned lc)); void ZLIB_INTERNAL _tr_flush_block OF((deflate_state *s, charf *buf, ulg stored_len, int last)); +void ZLIB_INTERNAL _tr_flush_bits OF((deflate_state *s)); void ZLIB_INTERNAL _tr_align OF((deflate_state *s)); void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf, ulg stored_len, int last)); @@ -312,7 +307,7 @@ void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf, * used. */ -#ifndef DEBUG +#ifndef ZLIB_DEBUG /* Inline versions of _tr_tally for speed: */ #if defined(GEN_TREES_H) || !defined(STDC) @@ -325,20 +320,22 @@ void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf, # define _tr_tally_lit(s, c, flush) \ { uch cc = (c); \ - s->d_buf[s->last_lit] = 0; \ - s->l_buf[s->last_lit++] = cc; \ + s->sym_buf[s->sym_next++] = 0; \ + s->sym_buf[s->sym_next++] = 0; \ + s->sym_buf[s->sym_next++] = cc; \ s->dyn_ltree[cc].Freq++; \ - flush = (s->last_lit == s->lit_bufsize-1); \ + flush = (s->sym_next == s->sym_end); \ } # define _tr_tally_dist(s, distance, length, flush) \ - { uch len = (length); \ - ush dist = (distance); \ - s->d_buf[s->last_lit] = dist; \ - s->l_buf[s->last_lit++] = len; \ + { uch len = (uch)(length); \ + ush dist = (ush)(distance); \ + s->sym_buf[s->sym_next++] = dist; \ + s->sym_buf[s->sym_next++] = dist >> 8; \ + s->sym_buf[s->sym_next++] = len; \ dist--; \ s->dyn_ltree[_length_code[len]+LITERALS+1].Freq++; \ s->dyn_dtree[d_code(dist)].Freq++; \ - flush = (s->last_lit == s->lit_bufsize-1); \ + flush = (s->sym_next == s->sym_end); \ } #else # define _tr_tally_lit(s, c, flush) flush = _tr_tally(s, 0, c) diff --git a/third_party/zlib/fill_window_sse.c b/third_party/zlib/fill_window_sse.c index 949ccce1ba..a841c99904 100644 --- a/third_party/zlib/fill_window_sse.c +++ b/third_party/zlib/fill_window_sse.c @@ -9,9 +9,10 @@ * For conditions of distribution and use, see copyright notice in zlib.h */ -#include #include "deflate.h" +#ifdef DEFLATE_FILL_WINDOW_SSE2 + #define UPDATE_HASH(s,h,i) \ {\ if (s->level < 6) { \ @@ -26,7 +27,9 @@ }\ }\ -extern int read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); +extern int deflate_read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); + +#include void fill_window_sse(deflate_state *s) { @@ -117,7 +120,9 @@ void fill_window_sse(deflate_state *s) */ Assert(more >= 2, "more < 2"); - n = read_buf(s->strm, s->window + s->strstart + s->lookahead, more); + n = deflate_read_buf(s->strm, + s->window + s->strstart + s->lookahead, + more); s->lookahead += n; /* Initialize the hash value now that we have some input: */ @@ -173,3 +178,5 @@ void fill_window_sse(deflate_state *s) Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD, "not enough room for search"); } + +#endif /* DEFLATE_FILL_WINDOW_SSE2 */ diff --git a/third_party/zlib/google.patch b/third_party/zlib/google.patch deleted file mode 100644 index 3818b11745..0000000000 --- a/third_party/zlib/google.patch +++ /dev/null @@ -1,240 +0,0 @@ -diff -ru zlib-1.2.5/gzlib.c zlib/gzlib.c ---- zlib-1.2.5/gzlib.c -+++ zlib/gzlib.c -@@ -5,7 +5,9 @@ - - #include "gzguts.h" - --#if defined(_LARGEFILE64_SOURCE) && _LFS64_LARGEFILE-0 -+#if defined(_WIN32) -+# define LSEEK (z_off64_t)_lseeki64 -+#elif defined(_LARGEFILE64_SOURCE) && _LFS64_LARGEFILE-0 - # define LSEEK lseek64 - #else - # define LSEEK lseek -diff -ru zlib-1.2.5/mozzconf.h zlib/mozzconf.h ---- zlib-1.2.5/mozzconf.h 2011-12-15 18:10:49.000000000 +0800 -+++ zlib/mozzconf.h 2011-12-16 16:08:00.000000000 +0800 -@@ -0,0 +1,168 @@ -+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -+/* ***** BEGIN LICENSE BLOCK ***** -+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1 -+ * -+ * The contents of this file are subject to the Mozilla Public License Version -+ * 1.1 (the "License"); you may not use this file except in compliance with -+ * the License. You may obtain a copy of the License at -+ * http://www.mozilla.org/MPL/ -+ * -+ * Software distributed under the License is distributed on an "AS IS" basis, -+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License -+ * for the specific language governing rights and limitations under the -+ * License. -+ * -+ * The Original Code is the mozilla zlib configuration. -+ * -+ * The Initial Developer of the Original Code is IBM Corporation. -+ * Portions created by the Initial Developer are Copyright (C) 2004 -+ * the Initial Developer. All Rights Reserved. -+ * -+ * Contributor(s): -+ * -+ * Alternatively, the contents of this file may be used under the terms of -+ * either of the GNU General Public License Version 2 or later (the "GPL"), -+ * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), -+ * in which case the provisions of the GPL or the LGPL are applicable instead -+ * of those above. If you wish to allow use of your version of this file only -+ * under the terms of either the GPL or the LGPL, and not to allow others to -+ * use your version of this file under the terms of the MPL, indicate your -+ * decision by deleting the provisions above and replace them with the notice -+ * and other provisions required by the GPL or the LGPL. If you do not delete -+ * the provisions above, a recipient may use your version of this file under -+ * the terms of any one of the MPL, the GPL or the LGPL. -+ * -+ * ***** END LICENSE BLOCK ***** */ -+ -+#ifndef MOZZCONF_H -+#define MOZZCONF_H -+ -+#if defined(XP_WIN) && defined(ZLIB_DLL) && !defined(MOZ_ENABLE_LIBXUL) -+#undef ZLIB_DLL -+#endif -+ -+#ifdef HAVE_VISIBILITY_ATTRIBUTE -+#define ZEXTERN __attribute__((visibility ("default"))) extern -+#endif -+ -+/* Exported Symbols */ -+#define zlibVersion MOZ_Z_zlibVersion -+#define deflate MOZ_Z_deflate -+#define deflateEnd MOZ_Z_deflateEnd -+#define inflate MOZ_Z_inflate -+#define inflateEnd MOZ_Z_inflateEnd -+#define deflateSetDictionary MOZ_Z_deflateSetDictionary -+#define deflateCopy MOZ_Z_deflateCopy -+#define deflateReset MOZ_Z_deflateReset -+#define deflateParams MOZ_Z_deflateParams -+#define deflateBound MOZ_Z_deflateBound -+#define deflatePrime MOZ_Z_deflatePrime -+#define inflateSetDictionary MOZ_Z_inflateSetDictionary -+#define inflateSync MOZ_Z_inflateSync -+#define inflateCopy MOZ_Z_inflateCopy -+#define inflateReset MOZ_Z_inflateReset -+#define inflateBack MOZ_Z_inflateBack -+#define inflateBackEnd MOZ_Z_inflateBackEnd -+#define zlibCompileFlags MOZ_Z_zlibCompileFlags -+#define compress MOZ_Z_compress -+#define compress2 MOZ_Z_compress2 -+#define compressBound MOZ_Z_compressBound -+#define uncompress MOZ_Z_uncompress -+#define gzopen MOZ_Z_gzopen -+#define gzdopen MOZ_Z_gzdopen -+#define gzsetparams MOZ_Z_gzsetparams -+#define gzread MOZ_Z_gzread -+#define gzwrite MOZ_Z_gzwrite -+#define gzprintf MOZ_Z_gzprintf -+#define gzputs MOZ_Z_gzputs -+#define gzgets MOZ_Z_gzgets -+#define gzputc MOZ_Z_gzputc -+#define gzgetc MOZ_Z_gzgetc -+#define gzungetc MOZ_Z_gzungetc -+#define gzflush MOZ_Z_gzflush -+#define gzseek MOZ_Z_gzseek -+#define gzrewind MOZ_Z_gzrewind -+#define gztell MOZ_Z_gztell -+#define gzeof MOZ_Z_gzeof -+#define gzclose MOZ_Z_gzclose -+#define gzerror MOZ_Z_gzerror -+#define gzclearerr MOZ_Z_gzclearerr -+#define adler32 MOZ_Z_adler32 -+#define crc32 MOZ_Z_crc32 -+#define deflateInit_ MOZ_Z_deflateInit_ -+#define deflateInit2_ MOZ_Z_deflateInit2_ -+#define inflateInit_ MOZ_Z_inflateInit_ -+#define inflateInit2_ MOZ_Z_inflateInit2_ -+#define inflateBackInit_ MOZ_Z_inflateBackInit_ -+#define inflateSyncPoint MOZ_Z_inflateSyncPoint -+#define get_crc_table MOZ_Z_get_crc_table -+#define zError MOZ_Z_zError -+ -+/* Extra global symbols */ -+#define _dist_code MOZ_Z__dist_code -+#define _length_code MOZ_Z__length_code -+#define _tr_align MOZ_Z__tr_align -+#define _tr_flush_block MOZ_Z__tr_flush_block -+#define _tr_init MOZ_Z__tr_init -+#define _tr_stored_block MOZ_Z__tr_stored_block -+#define _tr_tally MOZ_Z__tr_tally -+#define deflate_copyright MOZ_Z_deflate_copyright -+#define inflate_copyright MOZ_Z_inflate_copyright -+#define inflate_fast MOZ_Z_inflate_fast -+#define inflate_table MOZ_Z_inflate_table -+#define z_errmsg MOZ_Z_z_errmsg -+#define zcalloc MOZ_Z_zcalloc -+#define zcfree MOZ_Z_zcfree -+#define alloc_func MOZ_Z_alloc_func -+#define free_func MOZ_Z_free_func -+#define in_func MOZ_Z_in_func -+#define out_func MOZ_Z_out_func -+ -+/* New as of libpng-1.2.3 */ -+#define adler32_combine MOZ_Z_adler32_combine -+#define crc32_combine MOZ_Z_crc32_combine -+#define deflateSetHeader MOZ_Z_deflateSetHeader -+#define deflateTune MOZ_Z_deflateTune -+#define gzdirect MOZ_Z_gzdirect -+#define inflatePrime MOZ_Z_inflatePrime -+#define inflateGetHeader MOZ_Z_inflateGetHeader -+ -+/* New as of zlib 1.2.5 */ -+#define gzoffset MOZ_Z_gzoffset -+#define gzopen64 MOZ_Z_gzopen64 -+#define gzseek64 MOZ_Z_gzseek64 -+#define gztell64 MOZ_Z_gztell64 -+#define gzoffset64 MOZ_Z_gzoffset64 -+#define adler32_combine64 MOZ_Z_adler32_combine64 -+#define crc32_combine64 MOZ_Z_crc32_combine64 -+#define gz_error MOZ_Z_gz_error -+#define gz_intmax MOZ_Z_gz_intmax -+#define gz_strwinerror MOZ_Z_gz_strwinerror -+#define gzbuffer MOZ_Z_gzbuffer -+#define gzclose_r MOZ_Z_gzclose_r -+#define gzclose_w MOZ_Z_gzclose_w -+#define inflateMark MOZ_Z_inflateMark -+#define inflateReset2 MOZ_Z_inflateReset2 -+#define inflateUndermine MOZ_Z_inflateUndermine -+#define charf MOZ_Z_charf -+#define gzFile MOZ_Z_gzFile -+#define gz_header MOZ_Z_gz_header -+#define gz_headerp MOZ_Z_gz_headerp -+#define intf MOZ_Z_intf -+#define uInt MOZ_Z_uInt -+#define uIntf MOZ_Z_uIntf -+#define uLong MOZ_Z_uLong -+#define uLongf MOZ_Z_uLongf -+#define voidp MOZ_Z_voidp -+#define voidpc MOZ_Z_voidpc -+#define voidpf MOZ_Z_voidpf -+#define gz_header_s MOZ_Z_gz_header_s -+#define internal_state MOZ_Z_internal_state -+ -+/* Mangle Byte types except on Mac. */ -+#if !defined(__MACTYPES__) -+#define Byte MOZ_Z_Byte -+#define Bytef MOZ_Z_Bytef -+#endif -+ -+#endif -diff -ru zlib-1.2.5/zconf.h zlib/zconf.h ---- zlib-1.2.5/zconf.h 2010-04-19 01:58:06.000000000 +0800 -+++ zlib/zconf.h 2011-12-14 08:06:37.000000000 +0800 -@@ -8,6 +8,9 @@ - #ifndef ZCONF_H - #define ZCONF_H - -+/* This include does prefixing as below, but with an updated set of names */ -+#include "mozzconf.h" -+ - /* - * If you *really* need a unique prefix for all types and library functions, - * compile with -DZ_PREFIX. The "standard" zlib should be compiled without it. -@@ -359,7 +359,7 @@ typedef uLong FAR uLongf; - typedef Byte *voidp; - #endif - --#ifdef HAVE_UNISTD_H /* may be set to #if 1 by ./configure */ -+#if !defined(_WIN32) - # define Z_HAVE_UNISTD_H - #endif - ---- zlib-1.2.5/zlib.h 2010-04-20 12:12:48.000000000 +0800 -+++ zlib/zlib.h 2013-04-06 16:08:48.000000000 +0800 -@@ -1572,11 +1572,29 @@ - #endif - - #if !defined(ZLIB_INTERNAL) && _FILE_OFFSET_BITS-0 == 64 && _LFS64_LARGEFILE-0 -+# ifdef gzopen -+# undef gzopen -+# endif - # define gzopen gzopen64 -+# ifdef gzseek -+# undef gzseek -+# endif - # define gzseek gzseek64 -+# ifdef gztell -+# undef gztell -+# endif - # define gztell gztell64 -+# ifdef gzoffset -+# undef gzoffset -+# endif - # define gzoffset gzoffset64 -+# ifdef adler32_combine -+# undef adler32_combine -+# endif - # define adler32_combine adler32_combine64 -+# ifdef crc32_combine -+# undef crc32_combine -+# endif - # define crc32_combine crc32_combine64 - # ifdef _LARGEFILE64_SOURCE - ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); diff --git a/third_party/zlib/google/BUILD.gn b/third_party/zlib/google/BUILD.gn index f74628c06d..14b318044f 100644 --- a/third_party/zlib/google/BUILD.gn +++ b/third_party/zlib/google/BUILD.gn @@ -11,5 +11,5 @@ static_library("compression_utils_portable") { "compression_utils_portable.cc", "compression_utils_portable.h", ] - deps = [ "//third_party/zlib" ] + public_deps = [ "//third_party/zlib" ] } diff --git a/third_party/zlib/google/DEPS b/third_party/zlib/google/DEPS index e616fe28cd..03f2cb950b 100644 --- a/third_party/zlib/google/DEPS +++ b/third_party/zlib/google/DEPS @@ -1,4 +1,6 @@ include_rules = [ '+base', + '+build', '+testing', + "+third_party/zlib/zlib.h", ] diff --git a/third_party/zlib/google/compression_utils.cc b/third_party/zlib/google/compression_utils.cc new file mode 100644 index 0000000000..d6ee2b61f7 --- /dev/null +++ b/third_party/zlib/google/compression_utils.cc @@ -0,0 +1,116 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "third_party/zlib/google/compression_utils.h" + +#include "base/bit_cast.h" +#include "base/check_op.h" +#include "base/process/memory.h" +#include "base/strings/string_piece.h" +#include "base/sys_byteorder.h" + +#include "third_party/zlib/google/compression_utils_portable.h" + +namespace compression { + +bool GzipCompress(base::StringPiece input, + char* output_buffer, + size_t output_buffer_size, + size_t* compressed_size, + void* (*malloc_fn)(size_t), + void (*free_fn)(void*)) { + static_assert(sizeof(Bytef) == 1, ""); + + // uLongf can be larger than size_t. + uLongf compressed_size_long = static_cast(output_buffer_size); + if (zlib_internal::GzipCompressHelper( + bit_cast(output_buffer), &compressed_size_long, + bit_cast(input.data()), + static_cast(input.size()), malloc_fn, free_fn) != Z_OK) { + return false; + } + // No overflow, as compressed_size_long <= output.size() which is a size_t. + *compressed_size = static_cast(compressed_size_long); + return true; +} + +bool GzipCompress(base::StringPiece input, std::string* output) { + // Not using std::vector<> because allocation failures are recoverable, + // which is hidden by std::vector<>. + static_assert(sizeof(Bytef) == 1, ""); + const uLongf input_size = static_cast(input.size()); + + uLongf compressed_data_size = + zlib_internal::GzipExpectedCompressedSize(input_size); + + Bytef* compressed_data; + if (!base::UncheckedMalloc(compressed_data_size, + reinterpret_cast(&compressed_data))) { + return false; + } + + if (zlib_internal::GzipCompressHelper(compressed_data, &compressed_data_size, + bit_cast(input.data()), + input_size, nullptr, nullptr) != Z_OK) { + free(compressed_data); + return false; + } + + Bytef* resized_data = + reinterpret_cast(realloc(compressed_data, compressed_data_size)); + if (!resized_data) { + free(compressed_data); + return false; + } + output->assign(resized_data, resized_data + compressed_data_size); + DCHECK_EQ(input_size, GetUncompressedSize(*output)); + + free(resized_data); + return true; +} + +bool GzipUncompress(const std::string& input, std::string* output) { + std::string uncompressed_output; + uLongf uncompressed_size = static_cast(GetUncompressedSize(input)); + if (uncompressed_size > uncompressed_output.max_size()) + return false; + + uncompressed_output.resize(uncompressed_size); + if (zlib_internal::GzipUncompressHelper( + bit_cast(uncompressed_output.data()), &uncompressed_size, + bit_cast(input.data()), + static_cast(input.length())) == Z_OK) { + output->swap(uncompressed_output); + return true; + } + return false; +} + +bool GzipUncompress(base::StringPiece input, base::StringPiece output) { + uLongf uncompressed_size = GetUncompressedSize(input); + if (uncompressed_size > output.size()) + return false; + return zlib_internal::GzipUncompressHelper( + bit_cast(output.data()), &uncompressed_size, + bit_cast(input.data()), + static_cast(input.length())) == Z_OK; +} + +bool GzipUncompress(base::StringPiece input, std::string* output) { + // Disallow in-place usage, i.e., |input| using |*output| as underlying data. + DCHECK_NE(input.data(), output->data()); + uLongf uncompressed_size = GetUncompressedSize(input); + output->resize(uncompressed_size); + return zlib_internal::GzipUncompressHelper( + bit_cast(output->data()), &uncompressed_size, + bit_cast(input.data()), + static_cast(input.length())) == Z_OK; +} + +uint32_t GetUncompressedSize(base::StringPiece compressed_data) { + return zlib_internal::GetGzipUncompressedSize( + bit_cast(compressed_data.data()), compressed_data.length()); +} + +} // namespace compression diff --git a/third_party/zlib/google/compression_utils.h b/third_party/zlib/google/compression_utils.h new file mode 100644 index 0000000000..516220719f --- /dev/null +++ b/third_party/zlib/google/compression_utils.h @@ -0,0 +1,56 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef THIRD_PARTY_ZLIB_GOOGLE_COMPRESSION_UTILS_H_ +#define THIRD_PARTY_ZLIB_GOOGLE_COMPRESSION_UTILS_H_ + +#include + +#include "base/strings/string_piece.h" + +namespace compression { + +// Compresses the data in |input| using gzip, storing the result in +// |output_buffer|, of size |output_buffer_size|. If the buffer is large enough +// and compression succeeds, |compressed_size| points to the compressed data +// size after the call. +// |malloc_fn| and |free_fn| are pointers to malloc() and free()-like functions, +// or nullptr to use the standard ones. +// Returns true for success. +bool GzipCompress(base::StringPiece input, + char* output_buffer, + size_t output_buffer_size, + size_t* compressed_size, + void* (*malloc_fn)(size_t), + void (*free_fn)(void*)); + +// Compresses the data in |input| using gzip, storing the result in |output|. +// |input| and |output| are allowed to point to the same string (in-place +// operation). +// Returns true for success. +bool GzipCompress(base::StringPiece input, std::string* output); + +// Uncompresses the data in |input| using gzip, storing the result in |output|. +// |input| and |output| are allowed to be the same string (in-place operation). +// Returns true for success. +bool GzipUncompress(const std::string& input, std::string* output); + +// Like the above method, but uses base::StringPiece to avoid allocations if +// needed. |output|'s size must be at least as large as the return value from +// GetUncompressedSize. +// Returns true for success. +bool GzipUncompress(base::StringPiece input, base::StringPiece output); + +// Uncompresses the data in |input| using gzip, and writes the results to +// |output|, which must NOT be the underlying string of |input|, and is resized +// if necessary. +// Returns true for success. +bool GzipUncompress(base::StringPiece input, std::string* output); + +// Returns the uncompressed size from GZIP-compressed |compressed_data|. +uint32_t GetUncompressedSize(base::StringPiece compressed_data); + +} // namespace compression + +#endif // THIRD_PARTY_ZLIB_GOOGLE_COMPRESSION_UTILS_H_ diff --git a/third_party/zlib/google/compression_utils_portable.cc b/third_party/zlib/google/compression_utils_portable.cc index 191e349e31..331e41e125 100644 --- a/third_party/zlib/google/compression_utils_portable.cc +++ b/third_party/zlib/google/compression_utils_portable.cc @@ -5,7 +5,7 @@ * found in the Chromium source repository LICENSE file. */ -#include "third_party/zlib/google/compression_utils_portable.h" +#include "compression_utils_portable.h" #include #include @@ -84,7 +84,7 @@ int CompressHelper(WrapperType wrapper_type, int compression_level, void* (*malloc_fn)(size_t), void (*free_fn)(void*)) { - if (compression_level < 1 || compression_level > 9) { + if (compression_level < 0 || compression_level > 9) { compression_level = Z_DEFAULT_COMPRESSION; } diff --git a/third_party/zlib/google/compression_utils_portable.h b/third_party/zlib/google/compression_utils_portable.h index cd004e86cf..c1f377571f 100644 --- a/third_party/zlib/google/compression_utils_portable.h +++ b/third_party/zlib/google/compression_utils_portable.h @@ -9,10 +9,14 @@ #include +/* TODO(cavalcantii): remove support for Chromium ever building with a system + * zlib. + */ #if defined(USE_SYSTEM_ZLIB) #include +/* AOSP build requires relative paths. */ #else -#include "third_party/zlib/zlib.h" +#include "zlib.h" #endif namespace zlib_internal { diff --git a/third_party/zlib/google/compression_utils_unittest.cc b/third_party/zlib/google/compression_utils_unittest.cc new file mode 100644 index 0000000000..398984bb2e --- /dev/null +++ b/third_party/zlib/google/compression_utils_unittest.cc @@ -0,0 +1,103 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "third_party/zlib/google/compression_utils.h" + +#include +#include + +#include + +#include "base/stl_util.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace compression { + +namespace { + +// The data to be compressed by gzip. This is the hex representation of "hello +// world". +const uint8_t kData[] = {0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, + 0x77, 0x6f, 0x72, 0x6c, 0x64}; + +// This is the string representation of gzip compressed string above. It was +// obtained by running echo -n "hello world" | gzip -c | hexdump -e '8 1 ", +// 0x%x"' followed by 0'ing out the OS byte (10th byte) in the header. This is +// so that the test passes on all platforms (that run various OS'es). +const uint8_t kCompressedData[] = { + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcb, + 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0x01, + 0x00, 0x85, 0x11, 0x4a, 0x0d, 0x0b, 0x00, 0x00, 0x00}; + +} // namespace + +TEST(CompressionUtilsTest, GzipCompression) { + std::string data(reinterpret_cast(kData), base::size(kData)); + std::string compressed_data; + EXPECT_TRUE(GzipCompress(data, &compressed_data)); + std::string golden_compressed_data( + reinterpret_cast(kCompressedData), + base::size(kCompressedData)); + EXPECT_EQ(golden_compressed_data, compressed_data); +} + +TEST(CompressionUtilsTest, GzipUncompression) { + std::string compressed_data(reinterpret_cast(kCompressedData), + base::size(kCompressedData)); + + std::string uncompressed_data; + EXPECT_TRUE(GzipUncompress(compressed_data, &uncompressed_data)); + + std::string golden_data(reinterpret_cast(kData), + base::size(kData)); + EXPECT_EQ(golden_data, uncompressed_data); +} + +TEST(CompressionUtilsTest, GzipUncompressionFromStringPieceToString) { + base::StringPiece compressed_data( + reinterpret_cast(kCompressedData), + base::size(kCompressedData)); + + std::string uncompressed_data; + EXPECT_TRUE(GzipUncompress(compressed_data, &uncompressed_data)); + + std::string golden_data(reinterpret_cast(kData), + base::size(kData)); + EXPECT_EQ(golden_data, uncompressed_data); +} + +// Checks that compressing/decompressing input > 256 bytes works as expected. +TEST(CompressionUtilsTest, LargeInput) { + const size_t kSize = 32 * 1024; + + // Generate a data string of |kSize| for testing. + std::string data; + data.resize(kSize); + for (size_t i = 0; i < kSize; ++i) + data[i] = static_cast(i & 0xFF); + + std::string compressed_data; + EXPECT_TRUE(GzipCompress(data, &compressed_data)); + + std::string uncompressed_data; + EXPECT_TRUE(GzipUncompress(compressed_data, &uncompressed_data)); + + EXPECT_EQ(data, uncompressed_data); +} + +TEST(CompressionUtilsTest, InPlace) { + const std::string original_data(reinterpret_cast(kData), + base::size(kData)); + const std::string golden_compressed_data( + reinterpret_cast(kCompressedData), + base::size(kCompressedData)); + + std::string data(original_data); + EXPECT_TRUE(GzipCompress(data, &data)); + EXPECT_EQ(golden_compressed_data, data); + EXPECT_TRUE(GzipUncompress(data, &data)); + EXPECT_EQ(original_data, data); +} + +} // namespace compression diff --git a/third_party/zlib/google/test/data/create_test_zip.sh b/third_party/zlib/google/test/data/create_test_zip.sh index e19d675374..f4cc635c2c 100755 --- a/third_party/zlib/google/test/data/create_test_zip.sh +++ b/third_party/zlib/google/test/data/create_test_zip.sh @@ -4,12 +4,14 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # -# Run this script in its directory to recreate test.zip -# and test_nocompress.zip. +# Run this script in its directory to recreate test.zip, test_nocompress.zip, +# and test_encrypted.zip rm test.zip rm test_nocompress.zip +rm test_encrypted.zip pushd test zip -r ../test.zip . zip -r -0 ../test_nocompress.zip . +zip -P "fake_password" -r ../test_encrypted.zip . popd diff --git a/third_party/zlib/google/test/data/test_encrypted.zip b/third_party/zlib/google/test/data/test_encrypted.zip new file mode 100644 index 0000000000..5e2a75759a Binary files /dev/null and b/third_party/zlib/google/test/data/test_encrypted.zip differ diff --git a/third_party/zlib/google/zip.cc b/third_party/zlib/google/zip.cc index 39e2e534ce..907e5da72e 100644 --- a/third_party/zlib/google/zip.cc +++ b/third_party/zlib/google/zip.cc @@ -4,6 +4,7 @@ #include "third_party/zlib/google/zip.h" +#include #include #include @@ -11,108 +12,223 @@ #include "base/files/file.h" #include "base/files/file_enumerator.h" #include "base/logging.h" -#include "base/strings/string16.h" +#include "base/memory/ptr_util.h" #include "base/strings/string_util.h" +#include "build/build_config.h" #include "third_party/zlib/google/zip_internal.h" #include "third_party/zlib/google/zip_reader.h" +#include "third_party/zlib/google/zip_writer.h" -#if defined(USE_SYSTEM_MINIZIP) -#include -#include -#else -#include "third_party/zlib/contrib/minizip/unzip.h" -#include "third_party/zlib/contrib/minizip/zip.h" -#endif - +namespace zip { namespace { -bool AddFileToZip(zipFile zip_file, const base::FilePath& src_dir) { - base::File file(src_dir, base::File::FLAG_OPEN | base::File::FLAG_READ); - if (!file.IsValid()) { - DLOG(ERROR) << "Could not open file for path " << src_dir.value(); - return false; - } - - int num_bytes; - char buf[zip::internal::kZipBufSize]; - do { - num_bytes = file.ReadAtCurrentPos(buf, zip::internal::kZipBufSize); - if (num_bytes > 0) { - if (ZIP_OK != zipWriteInFileInZip(zip_file, buf, num_bytes)) { - DLOG(ERROR) << "Could not write data to zip for path " - << src_dir.value(); - return false; - } - } - } while (num_bytes > 0); +bool IsHiddenFile(const base::FilePath& file_path) { + return file_path.BaseName().value()[0] == '.'; +} +bool ExcludeNoFilesFilter(const base::FilePath& file_path) { return true; } -bool AddEntryToZip(zipFile zip_file, const base::FilePath& path, - const base::FilePath& root_path) { - base::FilePath relative_path; - bool result = root_path.AppendRelativePath(path, &relative_path); - DCHECK(result); - std::string str_path = relative_path.AsUTF8Unsafe(); -#if defined(OS_WIN) - base::ReplaceSubstringsAfterOffset(&str_path, 0u, "\\", "/"); -#endif +bool ExcludeHiddenFilesFilter(const base::FilePath& file_path) { + return !IsHiddenFile(file_path); +} - bool is_directory = base::DirectoryExists(path); - if (is_directory) - str_path += "/"; +// Creates a directory at |extract_dir|/|entry_path|, including any parents. +bool CreateDirectory(const base::FilePath& extract_dir, + const base::FilePath& entry_path) { + return base::CreateDirectory(extract_dir.Append(entry_path)); +} - zip_fileinfo file_info = zip::internal::GetFileInfoForZipping(path); - if (!zip::internal::ZipOpenNewFileInZip(zip_file, str_path, &file_info)) - return false; +// Creates a WriterDelegate that can write a file at |extract_dir|/|entry_path|. +std::unique_ptr CreateFilePathWriterDelegate( + const base::FilePath& extract_dir, + const base::FilePath& entry_path) { + return std::make_unique( + extract_dir.Append(entry_path)); +} - bool success = true; - if (!is_directory) { - success = AddFileToZip(zip_file, path); +class DirectFileAccessor : public FileAccessor { + public: + explicit DirectFileAccessor(base::FilePath src_dir) : src_dir_(src_dir) {} + ~DirectFileAccessor() override = default; + + std::vector OpenFilesForReading( + const std::vector& paths) override { + std::vector files; + for (const auto& path : paths) { + base::File file; + if (base::PathExists(path) && !base::DirectoryExists(path)) { + file = base::File(path, base::File::FLAG_OPEN | base::File::FLAG_READ); + } + files.push_back(std::move(file)); + } + return files; } - if (ZIP_OK != zipCloseFileInZip(zip_file)) { - DLOG(ERROR) << "Could not close zip file entry " << str_path; - return false; + bool DirectoryExists(const base::FilePath& file) override { + return base::DirectoryExists(file); } - return success; -} + std::vector ListDirectoryContent( + const base::FilePath& dir) override { + std::vector files; + base::FileEnumerator file_enumerator( + dir, false /* recursive */, + base::FileEnumerator::FILES | base::FileEnumerator::DIRECTORIES); + for (base::FilePath path = file_enumerator.Next(); !path.value().empty(); + path = file_enumerator.Next()) { + files.push_back(DirectoryContentEntry(path, base::DirectoryExists(path))); + } + return files; + } -bool ExcludeNoFilesFilter(const base::FilePath& file_path) { - return true; -} + base::Time GetLastModifiedTime(const base::FilePath& path) override { + base::File::Info file_info; + if (!base::GetFileInfo(path, &file_info)) { + LOG(ERROR) << "Failed to retrieve file modification time for " + << path.value(); + } + return file_info.last_modified; + } -bool ExcludeHiddenFilesFilter(const base::FilePath& file_path) { - return file_path.BaseName().value()[0] != '.'; -} + private: + base::FilePath src_dir_; + + DISALLOW_COPY_AND_ASSIGN(DirectFileAccessor); +}; } // namespace -namespace zip { +ZipParams::ZipParams(const base::FilePath& src_dir, + const base::FilePath& dest_file) + : src_dir_(src_dir), + dest_file_(dest_file), + file_accessor_(new DirectFileAccessor(src_dir)) {} + +#if defined(OS_POSIX) +// Does not take ownership of |fd|. +ZipParams::ZipParams(const base::FilePath& src_dir, int dest_fd) + : src_dir_(src_dir), + dest_fd_(dest_fd), + file_accessor_(new DirectFileAccessor(src_dir)) {} +#endif + +bool Zip(const ZipParams& params) { + // Using a pointer to avoid copies of a potentially large array. + const std::vector* files_to_add = ¶ms.files_to_zip(); + std::vector all_files; + if (files_to_add->empty()) { + // Include all files from the src_dir (modulo the src_dir itself and + // filtered and hidden files). + + files_to_add = &all_files; + // Using a list so we can call push_back while iterating. + std::list entries; + entries.push_back(FileAccessor::DirectoryContentEntry( + params.src_dir(), true /* is directory*/)); + const FilterCallback& filter_callback = params.filter_callback(); + for (auto iter = entries.begin(); iter != entries.end(); ++iter) { + const base::FilePath& entry_path = iter->path; + if (iter != entries.begin() && // Don't filter the root dir. + ((!params.include_hidden_files() && IsHiddenFile(entry_path)) || + (filter_callback && !filter_callback.Run(entry_path)))) { + continue; + } + + if (iter != entries.begin()) { // Exclude the root dir from the ZIP file. + // Make the path relative for AddEntryToZip. + base::FilePath relative_path; + bool success = + params.src_dir().AppendRelativePath(entry_path, &relative_path); + DCHECK(success); + all_files.push_back(relative_path); + } + + if (iter->is_directory) { + std::vector subentries = + params.file_accessor()->ListDirectoryContent(entry_path); + entries.insert(entries.end(), subentries.begin(), subentries.end()); + } + } + } + + std::unique_ptr zip_writer; +#if defined(OS_POSIX) + if (params.dest_fd() != base::kInvalidPlatformFile) { + DCHECK(params.dest_file().empty()); + zip_writer = internal::ZipWriter::CreateWithFd( + params.dest_fd(), params.src_dir(), params.file_accessor()); + if (!zip_writer) + return false; + } +#endif + if (!zip_writer) { + zip_writer = internal::ZipWriter::Create( + params.dest_file(), params.src_dir(), params.file_accessor()); + if (!zip_writer) + return false; + } + return zip_writer->WriteEntries(*files_to_add); +} bool Unzip(const base::FilePath& src_file, const base::FilePath& dest_dir) { - ZipReader reader; - if (!reader.Open(src_file)) { + return UnzipWithFilterCallback( + src_file, dest_dir, base::BindRepeating(&ExcludeNoFilesFilter), true); +} + +bool UnzipWithFilterCallback(const base::FilePath& src_file, + const base::FilePath& dest_dir, + const FilterCallback& filter_cb, + bool log_skipped_files) { + base::File file(src_file, base::File::FLAG_OPEN | base::File::FLAG_READ); + if (!file.IsValid()) { DLOG(WARNING) << "Failed to open " << src_file.value(); return false; } + return UnzipWithFilterAndWriters( + file.GetPlatformFile(), + base::BindRepeating(&CreateFilePathWriterDelegate, dest_dir), + base::BindRepeating(&CreateDirectory, dest_dir), filter_cb, + log_skipped_files); +} + +bool UnzipWithFilterAndWriters(const base::PlatformFile& src_file, + const WriterFactory& writer_factory, + const DirectoryCreator& directory_creator, + const FilterCallback& filter_cb, + bool log_skipped_files) { + ZipReader reader; + if (!reader.OpenFromPlatformFile(src_file)) { + DLOG(WARNING) << "Failed to open src_file " << src_file; + return false; + } while (reader.HasMore()) { if (!reader.OpenCurrentEntryInZip()) { DLOG(WARNING) << "Failed to open the current file in zip"; return false; } + const base::FilePath& entry_path = reader.current_entry_info()->file_path(); if (reader.current_entry_info()->is_unsafe()) { - DLOG(WARNING) << "Found an unsafe file in zip " - << reader.current_entry_info()->file_path().value(); + DLOG(WARNING) << "Found an unsafe file in zip " << entry_path; return false; } - if (!reader.ExtractCurrentEntryIntoDirectory(dest_dir)) { - DLOG(WARNING) << "Failed to extract " - << reader.current_entry_info()->file_path().value(); - return false; + if (filter_cb.Run(entry_path)) { + if (reader.current_entry_info()->is_directory()) { + if (!directory_creator.Run(entry_path)) + return false; + } else { + std::unique_ptr writer = writer_factory.Run(entry_path); + if (!reader.ExtractCurrentEntry(writer.get(), + std::numeric_limits::max())) { + DLOG(WARNING) << "Failed to extract " << entry_path; + return false; + } + } + } else if (log_skipped_files) { + DLOG(WARNING) << "Skipped file " << entry_path; } + if (!reader.AdvanceToNextEntry()) { DLOG(WARNING) << "Failed to advance to the next file"; return false; @@ -125,46 +241,19 @@ bool ZipWithFilterCallback(const base::FilePath& src_dir, const base::FilePath& dest_file, const FilterCallback& filter_cb) { DCHECK(base::DirectoryExists(src_dir)); - - zipFile zip_file = internal::OpenForZipping(dest_file.AsUTF8Unsafe(), - APPEND_STATUS_CREATE); - - if (!zip_file) { - DLOG(WARNING) << "couldn't create file " << dest_file.value(); - return false; - } - - bool success = true; - base::FileEnumerator file_enumerator(src_dir, true /* recursive */, - base::FileEnumerator::FILES | base::FileEnumerator::DIRECTORIES); - for (base::FilePath path = file_enumerator.Next(); !path.value().empty(); - path = file_enumerator.Next()) { - if (!filter_cb.Run(path)) { - continue; - } - - if (!AddEntryToZip(zip_file, path, src_dir)) { - success = false; - break; - } - } - - if (ZIP_OK != zipClose(zip_file, NULL)) { - DLOG(ERROR) << "Error closing zip file " << dest_file.value(); - return false; - } - - return success; + ZipParams params(src_dir, dest_file); + params.set_filter_callback(filter_cb); + return Zip(params); } bool Zip(const base::FilePath& src_dir, const base::FilePath& dest_file, bool include_hidden_files) { if (include_hidden_files) { - return ZipWithFilterCallback( - src_dir, dest_file, base::Bind(&ExcludeNoFilesFilter)); + return ZipWithFilterCallback(src_dir, dest_file, + base::BindRepeating(&ExcludeNoFilesFilter)); } else { return ZipWithFilterCallback( - src_dir, dest_file, base::Bind(&ExcludeHiddenFilesFilter)); + src_dir, dest_file, base::BindRepeating(&ExcludeHiddenFilesFilter)); } } @@ -173,31 +262,9 @@ bool ZipFiles(const base::FilePath& src_dir, const std::vector& src_relative_paths, int dest_fd) { DCHECK(base::DirectoryExists(src_dir)); - zipFile zip_file = internal::OpenFdForZipping(dest_fd, APPEND_STATUS_CREATE); - - if (!zip_file) { - DLOG(ERROR) << "couldn't create file for fd " << dest_fd; - return false; - } - - bool success = true; - for (std::vector::const_iterator iter = - src_relative_paths.begin(); - iter != src_relative_paths.end(); ++iter) { - const base::FilePath& path = src_dir.Append(*iter); - if (!AddEntryToZip(zip_file, path, src_dir)) { - // TODO(hshi): clean up the partial zip file when error occurs. - success = false; - break; - } - } - - if (ZIP_OK != zipClose(zip_file, NULL)) { - DLOG(ERROR) << "Error closing zip file for fd " << dest_fd; - success = false; - } - - return success; + ZipParams params(src_dir, dest_fd); + params.set_files_to_zip(src_relative_paths); + return Zip(params); } #endif // defined(OS_POSIX) diff --git a/third_party/zlib/google/zip.gyp b/third_party/zlib/google/zip.gyp deleted file mode 100644 index 9542f9cd66..0000000000 --- a/third_party/zlib/google/zip.gyp +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2013 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -{ - 'targets': [ - { - 'target_name': 'zip', - 'type': 'static_library', - 'dependencies': [ - '../zlib.gyp:minizip', - '../../../base/base.gyp:base', - ], - 'include_dirs': [ - '../../..', - ], - 'sources': [ - 'zip.cc', - 'zip.h', - 'zip_internal.cc', - 'zip_internal.h', - 'zip_reader.cc', - 'zip_reader.h', - ], - }, - ], -} diff --git a/third_party/zlib/google/zip.h b/third_party/zlib/google/zip.h index 216b09e27f..4f64a8aca8 100644 --- a/third_party/zlib/google/zip.h +++ b/third_party/zlib/google/zip.h @@ -9,15 +9,123 @@ #include "base/callback.h" #include "base/files/file_path.h" +#include "base/files/platform_file.h" +#include "base/time/time.h" +#include "build/build_config.h" + +namespace base { +class File; +} namespace zip { +class WriterDelegate; + +// Abstraction for file access operation required by Zip(). +// Can be passed to the ZipParams for providing custom access to the files, +// for example over IPC. +// If none is provided, the files are accessed directly. +// All parameters paths are expected to be absolute. +class FileAccessor { + public: + virtual ~FileAccessor() = default; + + struct DirectoryContentEntry { + DirectoryContentEntry(const base::FilePath& path, bool is_directory) + : path(path), is_directory(is_directory) {} + base::FilePath path; + bool is_directory = false; + }; + + // Opens files specified in |paths|. + // Directories should be mapped to invalid files. + virtual std::vector OpenFilesForReading( + const std::vector& paths) = 0; + + virtual bool DirectoryExists(const base::FilePath& path) = 0; + virtual std::vector ListDirectoryContent( + const base::FilePath& dir_path) = 0; + virtual base::Time GetLastModifiedTime(const base::FilePath& path) = 0; +}; + +class ZipParams { + public: + ZipParams(const base::FilePath& src_dir, const base::FilePath& dest_file); +#if defined(OS_POSIX) + // Does not take ownership of |dest_fd|. + ZipParams(const base::FilePath& src_dir, int dest_fd); + + int dest_fd() const { return dest_fd_; } +#endif + + const base::FilePath& src_dir() const { return src_dir_; } + + const base::FilePath& dest_file() const { return dest_file_; } + + // Restricts the files actually zipped to the paths listed in + // |src_relative_paths|. They must be relative to the |src_dir| passed in the + // constructor and will be used as the file names in the created zip file. All + // source paths must be under |src_dir| in the file system hierarchy. + void set_files_to_zip(const std::vector& src_relative_paths) { + src_files_ = src_relative_paths; + } + const std::vector& files_to_zip() const { return src_files_; } + + using FilterCallback = base::RepeatingCallback; + void set_filter_callback(FilterCallback filter_callback) { + filter_callback_ = filter_callback; + } + const FilterCallback& filter_callback() const { return filter_callback_; } + + void set_include_hidden_files(bool include_hidden_files) { + include_hidden_files_ = include_hidden_files; + } + bool include_hidden_files() const { return include_hidden_files_; } + + // Sets a custom file accessor for file operations. Default is to directly + // access the files (with fopen and the rest). + // Useful in cases where running in a sandbox process and file access has to + // go through IPC, for example. + void set_file_accessor(std::unique_ptr file_accessor) { + file_accessor_ = std::move(file_accessor); + } + FileAccessor* file_accessor() const { return file_accessor_.get(); } + + private: + base::FilePath src_dir_; + + base::FilePath dest_file_; +#if defined(OS_POSIX) + int dest_fd_ = base::kInvalidPlatformFile; +#endif + + // The relative paths to the files that should be included in the zip file. If + // this is empty, all files in |src_dir_| are included. + std::vector src_files_; + + // Filter used to exclude files from the ZIP file. Only effective when + // |src_files_| is empty. + FilterCallback filter_callback_; + + // Whether hidden files should be included in the ZIP file. Only effective + // when |src_files_| is empty. + bool include_hidden_files_ = true; + + // Abstraction around file system access used to read files. An implementation + // that accesses files directly is provided by default. + std::unique_ptr file_accessor_; +}; + +// Zip files specified into a ZIP archives. The source files and ZIP destination +// files (as well as other settings) are specified in |params|. +bool Zip(const ZipParams& params); + // Zip the contents of src_dir into dest_file. src_path must be a directory. // An entry will *not* be created in the zip for the root folder -- children // of src_dir will be at the root level of the created zip. For each file in // src_dir, include it only if the callback |filter_cb| returns true. Otherwise // omit it. -typedef base::Callback FilterCallback; +using FilterCallback = base::RepeatingCallback; bool ZipWithFilterCallback(const base::FilePath& src_dir, const base::FilePath& dest_file, const FilterCallback& filter_cb); @@ -39,6 +147,32 @@ bool ZipFiles(const base::FilePath& src_dir, int dest_fd); #endif // defined(OS_POSIX) +// Unzip the contents of zip_file into dest_dir. +// For each file in zip_file, include it only if the callback |filter_cb| +// returns true. Otherwise omit it. +// If |log_skipped_files| is true, files skipped during extraction are printed +// to debug log. +using FilterCallback = base::RepeatingCallback; +bool UnzipWithFilterCallback(const base::FilePath& zip_file, + const base::FilePath& dest_dir, + const FilterCallback& filter_cb, + bool log_skipped_files); + +// Unzip the contents of zip_file, using the writers provided by writer_factory. +// For each file in zip_file, include it only if the callback |filter_cb| +// returns true. Otherwise omit it. +// If |log_skipped_files| is true, files skipped during extraction are printed +// to debug log. +typedef base::RepeatingCallback( + const base::FilePath&)> + WriterFactory; +typedef base::RepeatingCallback DirectoryCreator; +bool UnzipWithFilterAndWriters(const base::PlatformFile& zip_file, + const WriterFactory& writer_factory, + const DirectoryCreator& directory_creator, + const FilterCallback& filter_cb, + bool log_skipped_files); + // Unzip the contents of zip_file into dest_dir. bool Unzip(const base::FilePath& zip_file, const base::FilePath& dest_dir); diff --git a/third_party/zlib/google/zip_internal.cc b/third_party/zlib/google/zip_internal.cc index 1f026c987b..9cbb78cb58 100644 --- a/third_party/zlib/google/zip_internal.cc +++ b/third_party/zlib/google/zip_internal.cc @@ -4,12 +4,14 @@ #include "third_party/zlib/google/zip_internal.h" +#include +#include + #include -#include "base/files/file_util.h" #include "base/logging.h" +#include "base/notreached.h" #include "base/strings/utf_string_conversions.h" -#include "base/time/time.h" #if defined(USE_SYSTEM_MINIZIP) #include @@ -343,40 +345,32 @@ zipFile OpenFdForZipping(int zip_fd, int append_flag) { } #endif -zip_fileinfo GetFileInfoForZipping(const base::FilePath& path) { - base::Time file_time; - base::File::Info file_info; - if (base::GetFileInfo(path, &file_info)) - file_time = file_info.last_modified; - return TimeToZipFileInfo(file_time); -} - bool ZipOpenNewFileInZip(zipFile zip_file, const std::string& str_path, - const zip_fileinfo* file_info) { + base::Time last_modified_time) { // Section 4.4.4 http://www.pkware.com/documents/casestudies/APPNOTE.TXT // Setting the Language encoding flag so the file is told to be in utf-8. const uLong LANGUAGE_ENCODING_FLAG = 0x1 << 11; - if (ZIP_OK != zipOpenNewFileInZip4( - zip_file, // file - str_path.c_str(), // filename - file_info, // zipfi - NULL, // extrafield_local, - 0u, // size_extrafield_local - NULL, // extrafield_global - 0u, // size_extrafield_global - NULL, // comment - Z_DEFLATED, // method - Z_DEFAULT_COMPRESSION, // level - 0, // raw - -MAX_WBITS, // windowBits - DEF_MEM_LEVEL, // memLevel - Z_DEFAULT_STRATEGY, // strategy - NULL, // password - 0, // crcForCrypting - 0, // versionMadeBy - LANGUAGE_ENCODING_FLAG)) { // flagBase + zip_fileinfo file_info = TimeToZipFileInfo(last_modified_time); + if (ZIP_OK != zipOpenNewFileInZip4(zip_file, // file + str_path.c_str(), // filename + &file_info, // zip_fileinfo + NULL, // extrafield_local, + 0u, // size_extrafield_local + NULL, // extrafield_global + 0u, // size_extrafield_global + NULL, // comment + Z_DEFLATED, // method + Z_DEFAULT_COMPRESSION, // level + 0, // raw + -MAX_WBITS, // windowBits + DEF_MEM_LEVEL, // memLevel + Z_DEFAULT_STRATEGY, // strategy + NULL, // password + 0, // crcForCrypting + 0, // versionMadeBy + LANGUAGE_ENCODING_FLAG)) { // flagBase DLOG(ERROR) << "Could not open zip file entry " << str_path; return false; } diff --git a/third_party/zlib/google/zip_internal.h b/third_party/zlib/google/zip_internal.h index ffd4039316..49fb902a74 100644 --- a/third_party/zlib/google/zip_internal.h +++ b/third_party/zlib/google/zip_internal.h @@ -5,12 +5,15 @@ #ifndef THIRD_PARTY_ZLIB_GOOGLE_ZIP_INTERNAL_H_ #define THIRD_PARTY_ZLIB_GOOGLE_ZIP_INTERNAL_H_ +#include + +#include "base/time/time.h" +#include "build/build_config.h" + #if defined(OS_WIN) #include #endif -#include - #if defined(USE_SYSTEM_MINIZIP) #include #include @@ -57,13 +60,10 @@ zipFile OpenForZipping(const std::string& file_name_utf8, int append_flag); zipFile OpenFdForZipping(int zip_fd, int append_flag); #endif -// Returns a zip_fileinfo with the last modification date of |path| set. -zip_fileinfo GetFileInfoForZipping(const base::FilePath& path); - // Wrapper around zipOpenNewFileInZip4 which passes most common options. bool ZipOpenNewFileInZip(zipFile zip_file, const std::string& str_path, - const zip_fileinfo* file_info); + base::Time last_modified_time); const int kZipMaxPath = 256; const int kZipBufSize = 8192; diff --git a/third_party/zlib/google/zip_reader.cc b/third_party/zlib/google/zip_reader.cc index e0871c8536..8b4bb4aee7 100644 --- a/third_party/zlib/google/zip_reader.cc +++ b/third_party/zlib/google/zip_reader.cc @@ -4,13 +4,17 @@ #include "third_party/zlib/google/zip_reader.h" +#include + #include "base/bind.h" #include "base/files/file.h" #include "base/logging.h" -#include "base/message_loop/message_loop.h" +#include "base/macros.h" +#include "base/single_thread_task_runner.h" #include "base/strings/string_util.h" #include "base/strings/utf_string_conversions.h" -#include "base/thread_task_runner_handle.h" +#include "base/threading/thread_task_runner_handle.h" +#include "build/build_config.h" #include "third_party/zlib/google/zip_internal.h" #if defined(USE_SYSTEM_MINIZIP) @@ -26,54 +30,6 @@ namespace zip { namespace { -// FilePathWriterDelegate ------------------------------------------------------ - -// A writer delegate that writes a file at a given path. -class FilePathWriterDelegate : public WriterDelegate { - public: - explicit FilePathWriterDelegate(const base::FilePath& output_file_path); - ~FilePathWriterDelegate() override; - - // WriterDelegate methods: - - // Creates the output file and any necessary intermediate directories. - bool PrepareOutput() override; - - // Writes |num_bytes| bytes of |data| to the file, returning false if not all - // bytes could be written. - bool WriteBytes(const char* data, int num_bytes) override; - - private: - base::FilePath output_file_path_; - base::File file_; - - DISALLOW_COPY_AND_ASSIGN(FilePathWriterDelegate); -}; - -FilePathWriterDelegate::FilePathWriterDelegate( - const base::FilePath& output_file_path) - : output_file_path_(output_file_path) { -} - -FilePathWriterDelegate::~FilePathWriterDelegate() { -} - -bool FilePathWriterDelegate::PrepareOutput() { - // We can't rely on parent directory entries being specified in the - // zip, so we make sure they are created. - if (!base::CreateDirectory(output_file_path_.DirName())) - return false; - - file_.Initialize(output_file_path_, - base::File::FLAG_CREATE_ALWAYS | base::File::FLAG_WRITE); - return file_.IsValid(); -} - -bool FilePathWriterDelegate::WriteBytes(const char* data, int num_bytes) { - return num_bytes == file_.WriteAtCurrentPos(data, num_bytes); -} - - // StringWriterDelegate -------------------------------------------------------- // A writer delegate that writes no more than |max_read_bytes| to a given @@ -92,6 +48,8 @@ class StringWriterDelegate : public WriterDelegate { // if |num_bytes| will cause the string to exceed |max_read_bytes|. bool WriteBytes(const char* data, int num_bytes) override; + void SetTimeModified(const base::Time& time) override; + private: size_t max_read_bytes_; std::string* output_; @@ -119,6 +77,10 @@ bool StringWriterDelegate::WriteBytes(const char* data, int num_bytes) { return true; } +void StringWriterDelegate::SetTimeModified(const base::Time& time) { + // Do nothing. +} + } // namespace // TODO(satorux): The implementation assumes that file names in zip files @@ -127,7 +89,9 @@ bool StringWriterDelegate::WriteBytes(const char* data, int num_bytes) { ZipReader::EntryInfo::EntryInfo(const std::string& file_name_in_zip, const unz_file_info& raw_file_info) : file_path_(base::FilePath::FromUTF8Unsafe(file_name_in_zip)), - is_directory_(false) { + is_directory_(false), + is_unsafe_(false), + is_encrypted_(false) { original_size_ = raw_file_info.uncompressed_size; // Directory entries in zip files end with "/". @@ -147,9 +111,13 @@ ZipReader::EntryInfo::EntryInfo(const std::string& file_name_in_zip, // We also consider that the file name is unsafe, if it's absolute. // On Windows, IsAbsolute() returns false for paths starting with "/". if (file_path_.IsAbsolute() || - base::StartsWithASCII(file_name_in_zip, "/", false)) + base::StartsWith(file_name_in_zip, "/", + base::CompareCase::INSENSITIVE_ASCII)) is_unsafe_ = true; + // Whether the file is encrypted is bit 0 of the flag. + is_encrypted_ = raw_file_info.flag & 1; + // Construct the last modified time. The timezone info is not present in // zip files, so we construct the time as local time. base::Time::Exploded exploded_time = {}; // Zero-clear. @@ -161,16 +129,12 @@ ZipReader::EntryInfo::EntryInfo(const std::string& file_name_in_zip, exploded_time.minute = raw_file_info.tmu_date.tm_min; exploded_time.second = raw_file_info.tmu_date.tm_sec; exploded_time.millisecond = 0; - if (exploded_time.HasValidValues()) { - last_modified_ = base::Time::FromLocalExploded(exploded_time); - } else { - // Use Unix time epoch if the time stamp data is invalid. + + if (!base::Time::FromLocalExploded(exploded_time, &last_modified_)) last_modified_ = base::Time::UnixEpoch(); - } } -ZipReader::ZipReader() - : weak_ptr_factory_(this) { +ZipReader::ZipReader() { Reset(); } @@ -271,23 +235,8 @@ bool ZipReader::OpenCurrentEntryInZip() { return true; } -bool ZipReader::LocateAndOpenEntry(const base::FilePath& path_in_zip) { - DCHECK(zip_file_); - - current_entry_info_.reset(); - reached_end_ = false; - const int kDefaultCaseSensivityOfOS = 0; - const int result = unzLocateFile(zip_file_, - path_in_zip.AsUTF8Unsafe().c_str(), - kDefaultCaseSensivityOfOS); - if (result != UNZ_OK) - return false; - - // Then Open the entry. - return OpenCurrentEntryInZip(); -} - -bool ZipReader::ExtractCurrentEntry(WriterDelegate* delegate) const { +bool ZipReader::ExtractCurrentEntry(WriterDelegate* delegate, + uint64_t num_bytes_to_extract) const { DCHECK(zip_file_); const int open_result = unzOpenCurrentFile(zip_file_); @@ -296,61 +245,50 @@ bool ZipReader::ExtractCurrentEntry(WriterDelegate* delegate) const { if (!delegate->PrepareOutput()) return false; + std::unique_ptr buf(new char[internal::kZipBufSize]); + + uint64_t remaining_capacity = num_bytes_to_extract; + bool entire_file_extracted = false; + + while (remaining_capacity > 0) { + const int num_bytes_read = + unzReadCurrentFile(zip_file_, buf.get(), internal::kZipBufSize); - bool success = true; // This becomes false when something bad happens. - scoped_ptr buf(new char[internal::kZipBufSize]); - while (true) { - const int num_bytes_read = unzReadCurrentFile(zip_file_, buf.get(), - internal::kZipBufSize); if (num_bytes_read == 0) { - // Reached the end of the file. + entire_file_extracted = true; break; } else if (num_bytes_read < 0) { // If num_bytes_read < 0, then it's a specific UNZ_* error code. - success = false; break; } else if (num_bytes_read > 0) { - // Some data is read. - if (!delegate->WriteBytes(buf.get(), num_bytes_read)) { - success = false; + uint64_t num_bytes_to_write = std::min( + remaining_capacity, base::checked_cast(num_bytes_read)); + if (!delegate->WriteBytes(buf.get(), num_bytes_to_write)) break; + if (remaining_capacity == base::checked_cast(num_bytes_read)) { + // Ensures function returns true if the entire file has been read. + entire_file_extracted = + (unzReadCurrentFile(zip_file_, buf.get(), 1) == 0); } + CHECK_GE(remaining_capacity, num_bytes_to_write); + remaining_capacity -= num_bytes_to_write; } } unzCloseCurrentFile(zip_file_); - return success; -} - -bool ZipReader::ExtractCurrentEntryToFilePath( - const base::FilePath& output_file_path) const { - DCHECK(zip_file_); - - // If this is a directory, just create it and return. - if (current_entry_info()->is_directory()) - return base::CreateDirectory(output_file_path); - - bool success = false; - { - FilePathWriterDelegate writer(output_file_path); - success = ExtractCurrentEntry(&writer); - } - - if (success && + if (entire_file_extracted && current_entry_info()->last_modified() != base::Time::UnixEpoch()) { - base::TouchFile(output_file_path, - base::Time::Now(), - current_entry_info()->last_modified()); + delegate->SetTimeModified(current_entry_info()->last_modified()); } - return success; + return entire_file_extracted; } void ZipReader::ExtractCurrentEntryToFilePathAsync( const base::FilePath& output_file_path, - const SuccessCallback& success_callback, - const FailureCallback& failure_callback, + SuccessCallback success_callback, + FailureCallback failure_callback, const ProgressCallback& progress_callback) { DCHECK(zip_file_); DCHECK(current_entry_info_.get()); @@ -358,24 +296,28 @@ void ZipReader::ExtractCurrentEntryToFilePathAsync( // If this is a directory, just create it and return. if (current_entry_info()->is_directory()) { if (base::CreateDirectory(output_file_path)) { - base::ThreadTaskRunnerHandle::Get()->PostTask(FROM_HERE, success_callback); + base::ThreadTaskRunnerHandle::Get()->PostTask( + FROM_HERE, std::move(success_callback)); } else { DVLOG(1) << "Unzip failed: unable to create directory."; - base::ThreadTaskRunnerHandle::Get()->PostTask(FROM_HERE, failure_callback); + base::ThreadTaskRunnerHandle::Get()->PostTask( + FROM_HERE, std::move(failure_callback)); } return; } if (unzOpenCurrentFile(zip_file_) != UNZ_OK) { DVLOG(1) << "Unzip failed: unable to open current zip entry."; - base::ThreadTaskRunnerHandle::Get()->PostTask(FROM_HERE, failure_callback); + base::ThreadTaskRunnerHandle::Get()->PostTask(FROM_HERE, + std::move(failure_callback)); return; } base::FilePath output_dir_path = output_file_path.DirName(); if (!base::CreateDirectory(output_dir_path)) { DVLOG(1) << "Unzip failed: unable to create containing directory."; - base::ThreadTaskRunnerHandle::Get()->PostTask(FROM_HERE, failure_callback); + base::ThreadTaskRunnerHandle::Get()->PostTask(FROM_HERE, + std::move(failure_callback)); return; } @@ -385,47 +327,28 @@ void ZipReader::ExtractCurrentEntryToFilePathAsync( if (!output_file.IsValid()) { DVLOG(1) << "Unzip failed: unable to create platform file at " << output_file_path.value(); - base::ThreadTaskRunnerHandle::Get()->PostTask(FROM_HERE, failure_callback); + base::ThreadTaskRunnerHandle::Get()->PostTask(FROM_HERE, + std::move(failure_callback)); return; } - base::MessageLoop::current()->PostTask( + base::ThreadTaskRunnerHandle::Get()->PostTask( FROM_HERE, - base::Bind(&ZipReader::ExtractChunk, - weak_ptr_factory_.GetWeakPtr(), - Passed(output_file.Pass()), - success_callback, - failure_callback, - progress_callback, - 0 /* initial offset */)); -} - -bool ZipReader::ExtractCurrentEntryIntoDirectory( - const base::FilePath& output_directory_path) const { - DCHECK(current_entry_info_.get()); - - base::FilePath output_file_path = output_directory_path.Append( - current_entry_info()->file_path()); - return ExtractCurrentEntryToFilePath(output_file_path); -} - -bool ZipReader::ExtractCurrentEntryToFile(base::File* file) const { - DCHECK(zip_file_); - - // If this is a directory, there's nothing to extract to the file, so return - // false. - if (current_entry_info()->is_directory()) - return false; - - FileWriterDelegate writer(file); - return ExtractCurrentEntry(&writer); + base::BindOnce(&ZipReader::ExtractChunk, weak_ptr_factory_.GetWeakPtr(), + Passed(std::move(output_file)), + std::move(success_callback), std::move(failure_callback), + progress_callback, 0 /* initial offset */)); } -bool ZipReader::ExtractCurrentEntryToString(size_t max_read_bytes, +bool ZipReader::ExtractCurrentEntryToString(uint64_t max_read_bytes, std::string* output) const { DCHECK(output); DCHECK(zip_file_); - DCHECK_NE(0U, max_read_bytes); + + if (max_read_bytes == 0) { + output->clear(); + return true; + } if (current_entry_info()->is_directory()) { output->clear(); @@ -437,13 +360,24 @@ bool ZipReader::ExtractCurrentEntryToString(size_t max_read_bytes, // correct. However, we need to assume that the uncompressed size could be // incorrect therefore this function needs to read as much data as possible. std::string contents; - contents.reserve(static_cast(std::min( - static_cast(max_read_bytes), - current_entry_info()->original_size()))); + contents.reserve( + static_cast(std::min(base::checked_cast(max_read_bytes), + current_entry_info()->original_size()))); StringWriterDelegate writer(max_read_bytes, &contents); - if (!ExtractCurrentEntry(&writer)) + if (!ExtractCurrentEntry(&writer, max_read_bytes)) { + if (contents.length() < max_read_bytes) { + // There was an error in extracting entry. If ExtractCurrentEntry() + // returns false, the entire file was not read - in which case + // contents.length() should equal |max_read_bytes| unless an error + // occurred which caused extraction to be aborted. + output->clear(); + } else { + // |num_bytes| is less than the length of current entry. + output->swap(contents); + } return false; + } output->swap(contents); return true; } @@ -472,10 +406,10 @@ void ZipReader::Reset() { } void ZipReader::ExtractChunk(base::File output_file, - const SuccessCallback& success_callback, - const FailureCallback& failure_callback, + SuccessCallback success_callback, + FailureCallback failure_callback, const ProgressCallback& progress_callback, - const int64 offset) { + const int64_t offset) { char buffer[internal::kZipBufSize]; const int num_bytes_read = unzReadCurrentFile(zip_file_, @@ -484,48 +418,42 @@ void ZipReader::ExtractChunk(base::File output_file, if (num_bytes_read == 0) { unzCloseCurrentFile(zip_file_); - success_callback.Run(); + std::move(success_callback).Run(); } else if (num_bytes_read < 0) { DVLOG(1) << "Unzip failed: error while reading zipfile " << "(" << num_bytes_read << ")"; - failure_callback.Run(); + std::move(failure_callback).Run(); } else { if (num_bytes_read != output_file.Write(offset, buffer, num_bytes_read)) { DVLOG(1) << "Unzip failed: unable to write all bytes to target."; - failure_callback.Run(); + std::move(failure_callback).Run(); return; } - int64 current_progress = offset + num_bytes_read; + int64_t current_progress = offset + num_bytes_read; progress_callback.Run(current_progress); - base::MessageLoop::current()->PostTask( + base::ThreadTaskRunnerHandle::Get()->PostTask( FROM_HERE, - base::Bind(&ZipReader::ExtractChunk, - weak_ptr_factory_.GetWeakPtr(), - Passed(output_file.Pass()), - success_callback, - failure_callback, - progress_callback, - current_progress)); - + base::BindOnce(&ZipReader::ExtractChunk, weak_ptr_factory_.GetWeakPtr(), + Passed(std::move(output_file)), + std::move(success_callback), std::move(failure_callback), + progress_callback, current_progress)); } } // FileWriterDelegate ---------------------------------------------------------- -FileWriterDelegate::FileWriterDelegate(base::File* file) - : file_(file), - file_length_(0) { -} +FileWriterDelegate::FileWriterDelegate(base::File* file) : file_(file) {} + +FileWriterDelegate::FileWriterDelegate(std::unique_ptr file) + : file_(file.get()), owned_file_(std::move(file)) {} FileWriterDelegate::~FileWriterDelegate() { -#if !defined(NDEBUG) - const bool success = -#endif - file_->SetLength(file_length_); - DPLOG_IF(ERROR, !success) << "Failed updating length of written file"; + if (!file_->SetLength(file_length_)) { + DVPLOG(1) << "Failed updating length of written file"; + } } bool FileWriterDelegate::PrepareOutput() { @@ -539,4 +467,36 @@ bool FileWriterDelegate::WriteBytes(const char* data, int num_bytes) { return bytes_written == num_bytes; } +void FileWriterDelegate::SetTimeModified(const base::Time& time) { + file_->SetTimes(base::Time::Now(), time); +} + +// FilePathWriterDelegate ------------------------------------------------------ + +FilePathWriterDelegate::FilePathWriterDelegate( + const base::FilePath& output_file_path) + : output_file_path_(output_file_path) {} + +FilePathWriterDelegate::~FilePathWriterDelegate() {} + +bool FilePathWriterDelegate::PrepareOutput() { + // We can't rely on parent directory entries being specified in the + // zip, so we make sure they are created. + if (!base::CreateDirectory(output_file_path_.DirName())) + return false; + + file_.Initialize(output_file_path_, + base::File::FLAG_CREATE_ALWAYS | base::File::FLAG_WRITE); + return file_.IsValid(); +} + +bool FilePathWriterDelegate::WriteBytes(const char* data, int num_bytes) { + return num_bytes == file_.WriteAtCurrentPos(data, num_bytes); +} + +void FilePathWriterDelegate::SetTimeModified(const base::Time& time) { + file_.Close(); + base::TouchFile(output_file_path_, base::Time::Now(), time); +} + } // namespace zip diff --git a/third_party/zlib/google/zip_reader.h b/third_party/zlib/google/zip_reader.h index da6cc93561..d442d42859 100644 --- a/third_party/zlib/google/zip_reader.h +++ b/third_party/zlib/google/zip_reader.h @@ -4,14 +4,17 @@ #ifndef THIRD_PARTY_ZLIB_GOOGLE_ZIP_READER_H_ #define THIRD_PARTY_ZLIB_GOOGLE_ZIP_READER_H_ +#include +#include + +#include #include -#include "base/basictypes.h" #include "base/callback.h" #include "base/files/file.h" #include "base/files/file_path.h" #include "base/files/file_util.h" -#include "base/memory/scoped_ptr.h" +#include "base/macros.h" #include "base/memory/weak_ptr.h" #include "base/time/time.h" @@ -36,6 +39,9 @@ class WriterDelegate { // Invoked to write the next chunk of data. Return false on failure to cancel // extraction. virtual bool WriteBytes(const char* data, int num_bytes) = 0; + + // Sets the last-modified time of the data. + virtual void SetTimeModified(const base::Time& time) = 0; }; // This class is used for reading zip files. A typical use case of this @@ -46,25 +52,25 @@ class WriterDelegate { // reader.Open(zip_file_path); // while (reader.HasMore()) { // reader.OpenCurrentEntryInZip(); -// reader.ExtractCurrentEntryToDirectory(output_directory_path); +// const base::FilePath& entry_path = +// reader.current_entry_info()->file_path(); +// auto writer = CreateFilePathWriterDelegate(extract_dir, entry_path); +// reader.ExtractCurrentEntry(writer, std::numeric_limits::max()); // reader.AdvanceToNextEntry(); // } // // For simplicity, error checking is omitted in the example code above. The // production code should check return values from all of these functions. // -// This calls can also be used for random access of contents in a zip file -// using LocateAndOpenEntry(). -// class ZipReader { public: // A callback that is called when the operation is successful. - typedef base::Closure SuccessCallback; + using SuccessCallback = base::OnceClosure; // A callback that is called when the operation fails. - typedef base::Closure FailureCallback; + using FailureCallback = base::OnceClosure; // A callback that is called periodically during the operation with the number // of bytes that have been processed so far. - typedef base::Callback ProgressCallback; + using ProgressCallback = base::RepeatingCallback; // This class represents information of an entry (file or directory) in // a zip file. @@ -81,7 +87,7 @@ class ZipReader { // Returns 0 if the entry is a directory. // Note: this value should not be trusted, because it is stored as metadata // in the zip archive and can be different from the real uncompressed size. - int64 original_size() const { return original_size_; } + int64_t original_size() const { return original_size_; } // Returns the last modified time. If the time stored in the zip file was // not valid, the unix epoch will be returned. @@ -101,12 +107,16 @@ class ZipReader { // UTF-8 characters in its file name, or the file path is absolute. bool is_unsafe() const { return is_unsafe_; } + // Returns true if the entry is encrypted. + bool is_encrypted() const { return is_encrypted_; } + private: const base::FilePath file_path_; - int64 original_size_; + int64_t original_size_; base::Time last_modified_; bool is_directory_; bool is_unsafe_; + bool is_encrypted_; DISALLOW_COPY_AND_ASSIGN(EntryInfo); }; @@ -151,23 +161,11 @@ class ZipReader { // state is reset automatically as needed. bool OpenCurrentEntryInZip(); - // Locates an entry in the zip file and opens it. Returns true on - // success. This function internally calls OpenCurrentEntryInZip() on - // success. On failure, current_entry_info() becomes NULL. - bool LocateAndOpenEntry(const base::FilePath& path_in_zip); - - // Extracts the current entry in chunks to |delegate|. - bool ExtractCurrentEntry(WriterDelegate* delegate) const; - - // Extracts the current entry to the given output file path. If the - // current file is a directory, just creates a directory - // instead. Returns true on success. OpenCurrentEntryInZip() must be - // called beforehand. - // - // This function preserves the timestamp of the original entry. If that - // timestamp is not valid, the timestamp will be set to the current time. - bool ExtractCurrentEntryToFilePath( - const base::FilePath& output_file_path) const; + // Extracts |num_bytes_to_extract| bytes of the current entry to |delegate|, + // starting from the beginning of the entry. Return value specifies whether + // the entire file was extracted. + bool ExtractCurrentEntry(WriterDelegate* delegate, + uint64_t num_bytes_to_extract) const; // Asynchronously extracts the current entry to the given output file path. // If the current entry is a directory it just creates the directory @@ -177,42 +175,24 @@ class ZipReader { // Callbacks will be posted to the current MessageLoop in-order. void ExtractCurrentEntryToFilePathAsync( const base::FilePath& output_file_path, - const SuccessCallback& success_callback, - const FailureCallback& failure_callback, + SuccessCallback success_callback, + FailureCallback failure_callback, const ProgressCallback& progress_callback); - // Extracts the current entry to the given output directory path using - // ExtractCurrentEntryToFilePath(). Sub directories are created as needed - // based on the file path of the current entry. For example, if the file - // path in zip is "foo/bar.txt", and the output directory is "output", - // "output/foo/bar.txt" will be created. - // - // Returns true on success. OpenCurrentEntryInZip() must be called - // beforehand. - // - // This function preserves the timestamp of the original entry. If that - // timestamp is not valid, the timestamp will be set to the current time. - bool ExtractCurrentEntryIntoDirectory( - const base::FilePath& output_directory_path) const; - - // Extracts the current entry by writing directly to a platform file. - // Does not close the file. Returns true on success. - bool ExtractCurrentEntryToFile(base::File* file) const; - - // Extracts the current entry into memory. If the current entry is a directory - // the |output| parameter is set to the empty string. If the current entry is - // a file, the |output| parameter is filled with its contents. Returns true on - // success. OpenCurrentEntryInZip() must be called beforehand. - // Note: the |output| parameter can be filled with a big amount of data, avoid - // passing it around by value, but by reference or pointer. - // Note: the value returned by EntryInfo::original_size() cannot be - // trusted, so the real size of the uncompressed contents can be different. - // Use max_read_bytes to limit the ammount of memory used to carry the entry. - // If the real size of the uncompressed data is bigger than max_read_bytes - // then false is returned. |max_read_bytes| must be non-zero. - bool ExtractCurrentEntryToString( - size_t max_read_bytes, - std::string* output) const; + // Extracts the current entry into memory. If the current entry is a + // directory, the |output| parameter is set to the empty string. If the + // current entry is a file, the |output| parameter is filled with its + // contents. OpenCurrentEntryInZip() must be called beforehand. Note: the + // |output| parameter can be filled with a big amount of data, avoid passing + // it around by value, but by reference or pointer. Note: the value returned + // by EntryInfo::original_size() cannot be trusted, so the real size of the + // uncompressed contents can be different. |max_read_bytes| limits the ammount + // of memory used to carry the entry. Returns true if the entire content is + // read. If the entry is bigger than |max_read_bytes|, returns false and + // |output| is filled with |max_read_bytes| of data. If an error occurs, + // returns false, and |output| is set to the empty string. + bool ExtractCurrentEntryToString(uint64_t max_read_bytes, + std::string* output) const; // Returns the current entry info. Returns NULL if the current entry is // not yet opened. OpenCurrentEntryInZip() must be called beforehand. @@ -234,17 +214,17 @@ class ZipReader { // Extracts a chunk of the file to the target. Will post a task for the next // chunk and success/failure/progress callbacks as necessary. void ExtractChunk(base::File target_file, - const SuccessCallback& success_callback, - const FailureCallback& failure_callback, + SuccessCallback success_callback, + FailureCallback failure_callback, const ProgressCallback& progress_callback, - const int64 offset); + const int64_t offset); unzFile zip_file_; int num_entries_; bool reached_end_; - scoped_ptr current_entry_info_; + std::unique_ptr current_entry_info_; - base::WeakPtrFactory weak_ptr_factory_; + base::WeakPtrFactory weak_ptr_factory_{this}; DISALLOW_COPY_AND_ASSIGN(ZipReader); }; @@ -252,8 +232,14 @@ class ZipReader { // A writer delegate that writes to a given File. class FileWriterDelegate : public WriterDelegate { public: + // Constructs a FileWriterDelegate that manipulates |file|. The delegate will + // not own |file|, therefore the caller must guarantee |file| will outlive the + // delegate. explicit FileWriterDelegate(base::File* file); + // Constructs a FileWriterDelegate that takes ownership of |file|. + explicit FileWriterDelegate(std::unique_ptr file); + // Truncates the file to the number of bytes written. ~FileWriterDelegate() override; @@ -266,13 +252,50 @@ class FileWriterDelegate : public WriterDelegate { // if not all bytes could be written. bool WriteBytes(const char* data, int num_bytes) override; + // Sets the last-modified time of the data. + void SetTimeModified(const base::Time& time) override; + + // Return the actual size of the file. + int64_t file_length() { return file_length_; } + private: + // The file the delegate modifies. base::File* file_; - int64_t file_length_; + + // The delegate can optionally own the file it modifies, in which case + // owned_file_ is set and file_ is an alias for owned_file_. + std::unique_ptr owned_file_; + + int64_t file_length_ = 0; DISALLOW_COPY_AND_ASSIGN(FileWriterDelegate); }; +// A writer delegate that writes a file at a given path. +class FilePathWriterDelegate : public WriterDelegate { + public: + explicit FilePathWriterDelegate(const base::FilePath& output_file_path); + ~FilePathWriterDelegate() override; + + // WriterDelegate methods: + + // Creates the output file and any necessary intermediate directories. + bool PrepareOutput() override; + + // Writes |num_bytes| bytes of |data| to the file, returning false if not all + // bytes could be written. + bool WriteBytes(const char* data, int num_bytes) override; + + // Sets the last-modified time of the data. + void SetTimeModified(const base::Time& time) override; + + private: + base::FilePath output_file_path_; + base::File file_; + + DISALLOW_COPY_AND_ASSIGN(FilePathWriterDelegate); +}; + } // namespace zip #endif // THIRD_PARTY_ZLIB_GOOGLE_ZIP_READER_H_ diff --git a/third_party/zlib/google/zip_reader_unittest.cc b/third_party/zlib/google/zip_reader_unittest.cc index 89b4ac52e3..bba4365298 100644 --- a/third_party/zlib/google/zip_reader_unittest.cc +++ b/third_party/zlib/google/zip_reader_unittest.cc @@ -4,19 +4,25 @@ #include "third_party/zlib/google/zip_reader.h" +#include +#include +#include + #include #include #include "base/bind.h" +#include "base/check.h" #include "base/files/file.h" #include "base/files/file_util.h" #include "base/files/scoped_temp_dir.h" -#include "base/logging.h" -#include "base/md5.h" +#include "base/hash/md5.h" #include "base/path_service.h" #include "base/run_loop.h" +#include "base/stl_util.h" #include "base/strings/stringprintf.h" #include "base/strings/utf_string_conversions.h" +#include "base/test/task_environment.h" #include "base/time/time.h" #include "testing/gmock/include/gmock/gmock.h" #include "testing/gtest/include/gtest/gtest.h" @@ -80,7 +86,7 @@ class MockUnzipListener : public base::SupportsWeakPtr { } // Progress callback for async functions. - void OnUnzipProgress(int64 progress) { + void OnUnzipProgress(int64_t progress) { DCHECK(progress > current_progress_); progress_calls_++; current_progress_ = progress; @@ -96,15 +102,37 @@ class MockUnzipListener : public base::SupportsWeakPtr { int failure_calls_; int progress_calls_; - int64 current_progress_; + int64_t current_progress_; }; class MockWriterDelegate : public zip::WriterDelegate { public: MOCK_METHOD0(PrepareOutput, bool()); MOCK_METHOD2(WriteBytes, bool(const char*, int)); + MOCK_METHOD1(SetTimeModified, void(const base::Time&)); }; +bool ExtractCurrentEntryToFilePath(zip::ZipReader* reader, + base::FilePath path) { + zip::FilePathWriterDelegate writer(path); + return reader->ExtractCurrentEntry(&writer, + std::numeric_limits::max()); +} + +bool LocateAndOpenEntry(zip::ZipReader* reader, + const base::FilePath& path_in_zip) { + // The underlying library can do O(1) access, but ZipReader does not expose + // that. O(N) access is acceptable for these tests. + while (reader->HasMore()) { + if (!reader->OpenCurrentEntryInZip()) + return false; + if (reader->current_entry_info()->file_path() == path_in_zip) + return true; + reader->AdvanceToNextEntry(); + } + return false; +} + } // namespace namespace zip { @@ -116,11 +144,12 @@ class ZipReaderTest : public PlatformTest { PlatformTest::SetUp(); ASSERT_TRUE(temp_dir_.CreateUniqueTempDir()); - test_dir_ = temp_dir_.path(); + test_dir_ = temp_dir_.GetPath(); ASSERT_TRUE(GetTestDataDirectory(&test_data_dir_)); test_zip_file_ = test_data_dir_.AppendASCII("test.zip"); + encrypted_zip_file_ = test_data_dir_.AppendASCII("test_encrypted.zip"); evil_zip_file_ = test_data_dir_.AppendASCII("evil.zip"); evil_via_invalid_utf8_zip_file_ = test_data_dir_.AppendASCII( "evil_via_invalid_utf8.zip"); @@ -145,7 +174,7 @@ class ZipReaderTest : public PlatformTest { } bool GetTestDataDirectory(base::FilePath* path) { - bool success = PathService::Get(base::DIR_SOURCE_ROOT, path); + bool success = base::PathService::Get(base::DIR_SOURCE_ROOT, path); EXPECT_TRUE(success); if (!success) return false; @@ -173,6 +202,8 @@ class ZipReaderTest : public PlatformTest { base::FilePath test_data_dir_; // The path to test.zip in the test data directory. base::FilePath test_zip_file_; + // The path to test_encrypted.zip in the test data directory. + base::FilePath encrypted_zip_file_; // The path to evil.zip in the test data directory. base::FilePath evil_zip_file_; // The path to evil_via_invalid_utf8.zip in the test data directory. @@ -183,7 +214,7 @@ class ZipReaderTest : public PlatformTest { base::ScopedTempDir temp_dir_; - base::MessageLoop message_loop_; + base::test::TaskEnvironment task_environment_; }; TEST_F(ZipReaderTest, Open_ValidZipFile) { @@ -244,112 +275,11 @@ TEST_F(ZipReaderTest, PlatformFileIteration) { EXPECT_EQ(test_zip_contents_, actual_contents); } -TEST_F(ZipReaderTest, LocateAndOpenEntry_ValidFile) { - std::set actual_contents; - ZipReader reader; - ASSERT_TRUE(reader.Open(test_zip_file_)); - base::FilePath target_path(FILE_PATH_LITERAL("foo/bar/quux.txt")); - ASSERT_TRUE(reader.LocateAndOpenEntry(target_path)); - EXPECT_EQ(target_path, reader.current_entry_info()->file_path()); -} - -TEST_F(ZipReaderTest, LocateAndOpenEntry_NonExistentFile) { - std::set actual_contents; - ZipReader reader; - ASSERT_TRUE(reader.Open(test_zip_file_)); - base::FilePath target_path(FILE_PATH_LITERAL("nonexistent.txt")); - ASSERT_FALSE(reader.LocateAndOpenEntry(target_path)); - EXPECT_EQ(NULL, reader.current_entry_info()); -} - -TEST_F(ZipReaderTest, ExtractCurrentEntryToFilePath_RegularFile) { - ZipReader reader; - ASSERT_TRUE(reader.Open(test_zip_file_)); - base::FilePath target_path(FILE_PATH_LITERAL("foo/bar/quux.txt")); - ASSERT_TRUE(reader.LocateAndOpenEntry(target_path)); - ASSERT_TRUE(reader.ExtractCurrentEntryToFilePath( - test_dir_.AppendASCII("quux.txt"))); - // Read the output file ans compute the MD5. - std::string output; - ASSERT_TRUE(base::ReadFileToString(test_dir_.AppendASCII("quux.txt"), - &output)); - const std::string md5 = base::MD5String(output); - EXPECT_EQ(kQuuxExpectedMD5, md5); - // quux.txt should be larger than kZipBufSize so that we can exercise - // the loop in ExtractCurrentEntry(). - EXPECT_LT(static_cast(internal::kZipBufSize), output.size()); -} - -TEST_F(ZipReaderTest, PlatformFileExtractCurrentEntryToFilePath_RegularFile) { - ZipReader reader; - FileWrapper zip_fd_wrapper(test_zip_file_, FileWrapper::READ_ONLY); - ASSERT_TRUE(reader.OpenFromPlatformFile(zip_fd_wrapper.platform_file())); - base::FilePath target_path(FILE_PATH_LITERAL("foo/bar/quux.txt")); - ASSERT_TRUE(reader.LocateAndOpenEntry(target_path)); - ASSERT_TRUE(reader.ExtractCurrentEntryToFilePath( - test_dir_.AppendASCII("quux.txt"))); - // Read the output file and compute the MD5. - std::string output; - ASSERT_TRUE(base::ReadFileToString(test_dir_.AppendASCII("quux.txt"), - &output)); - const std::string md5 = base::MD5String(output); - EXPECT_EQ(kQuuxExpectedMD5, md5); - // quux.txt should be larger than kZipBufSize so that we can exercise - // the loop in ExtractCurrentEntry(). - EXPECT_LT(static_cast(internal::kZipBufSize), output.size()); -} - -TEST_F(ZipReaderTest, PlatformFileExtractCurrentEntryToFile_RegularFile) { - ZipReader reader; - FileWrapper zip_fd_wrapper(test_zip_file_, FileWrapper::READ_ONLY); - ASSERT_TRUE(reader.OpenFromPlatformFile(zip_fd_wrapper.platform_file())); - base::FilePath target_path(FILE_PATH_LITERAL("foo/bar/quux.txt")); - base::FilePath out_path = test_dir_.AppendASCII("quux.txt"); - FileWrapper out_fd_w(out_path, FileWrapper::READ_WRITE); - ASSERT_TRUE(reader.LocateAndOpenEntry(target_path)); - ASSERT_TRUE(reader.ExtractCurrentEntryToFile(out_fd_w.file())); - // Read the output file and compute the MD5. - std::string output; - ASSERT_TRUE(base::ReadFileToString(out_path, &output)); - const std::string md5 = base::MD5String(output); - EXPECT_EQ(kQuuxExpectedMD5, md5); - // quux.txt should be larger than kZipBufSize so that we can exercise - // the loop in ExtractCurrentEntry(). - EXPECT_LT(static_cast(internal::kZipBufSize), output.size()); -} - -TEST_F(ZipReaderTest, ExtractCurrentEntryToFilePath_Directory) { - ZipReader reader; - ASSERT_TRUE(reader.Open(test_zip_file_)); - base::FilePath target_path(FILE_PATH_LITERAL("foo/")); - ASSERT_TRUE(reader.LocateAndOpenEntry(target_path)); - ASSERT_TRUE(reader.ExtractCurrentEntryToFilePath( - test_dir_.AppendASCII("foo"))); - // The directory should be created. - ASSERT_TRUE(base::DirectoryExists(test_dir_.AppendASCII("foo"))); -} - -TEST_F(ZipReaderTest, ExtractCurrentEntryIntoDirectory_RegularFile) { - ZipReader reader; - ASSERT_TRUE(reader.Open(test_zip_file_)); - base::FilePath target_path(FILE_PATH_LITERAL("foo/bar/quux.txt")); - ASSERT_TRUE(reader.LocateAndOpenEntry(target_path)); - ASSERT_TRUE(reader.ExtractCurrentEntryIntoDirectory(test_dir_)); - // Sub directories should be created. - ASSERT_TRUE(base::DirectoryExists(test_dir_.AppendASCII("foo/bar"))); - // And the file should be created. - std::string output; - ASSERT_TRUE(base::ReadFileToString( - test_dir_.AppendASCII("foo/bar/quux.txt"), &output)); - const std::string md5 = base::MD5String(output); - EXPECT_EQ(kQuuxExpectedMD5, md5); -} - TEST_F(ZipReaderTest, current_entry_info_RegularFile) { ZipReader reader; ASSERT_TRUE(reader.Open(test_zip_file_)); base::FilePath target_path(FILE_PATH_LITERAL("foo/bar/quux.txt")); - ASSERT_TRUE(reader.LocateAndOpenEntry(target_path)); + ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); ZipReader::EntryInfo* current_entry_info = reader.current_entry_info(); EXPECT_EQ(target_path, current_entry_info->file_path()); @@ -375,7 +305,7 @@ TEST_F(ZipReaderTest, current_entry_info_DotDotFile) { ASSERT_TRUE(reader.Open(evil_zip_file_)); base::FilePath target_path(FILE_PATH_LITERAL( "../levilevilevilevilevilevilevilevilevilevilevilevil")); - ASSERT_TRUE(reader.LocateAndOpenEntry(target_path)); + ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); ZipReader::EntryInfo* current_entry_info = reader.current_entry_info(); EXPECT_EQ(target_path, current_entry_info->file_path()); @@ -403,7 +333,7 @@ TEST_F(ZipReaderTest, current_entry_info_AbsoluteFile) { ZipReader reader; ASSERT_TRUE(reader.Open(evil_via_absolute_file_name_zip_file_)); base::FilePath target_path(FILE_PATH_LITERAL("/evil.txt")); - ASSERT_TRUE(reader.LocateAndOpenEntry(target_path)); + ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); ZipReader::EntryInfo* current_entry_info = reader.current_entry_info(); EXPECT_EQ(target_path, current_entry_info->file_path()); @@ -416,7 +346,7 @@ TEST_F(ZipReaderTest, current_entry_info_Directory) { ZipReader reader; ASSERT_TRUE(reader.Open(test_zip_file_)); base::FilePath target_path(FILE_PATH_LITERAL("foo/bar/")); - ASSERT_TRUE(reader.LocateAndOpenEntry(target_path)); + ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); ZipReader::EntryInfo* current_entry_info = reader.current_entry_info(); EXPECT_EQ(base::FilePath(FILE_PATH_LITERAL("foo/bar/")), @@ -439,6 +369,20 @@ TEST_F(ZipReaderTest, current_entry_info_Directory) { EXPECT_TRUE(current_entry_info->is_directory()); } +TEST_F(ZipReaderTest, current_entry_info_EncryptedFile) { + ZipReader reader; + base::FilePath target_path(FILE_PATH_LITERAL("foo/bar/quux.txt")); + + ASSERT_TRUE(reader.Open(encrypted_zip_file_)); + ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); + EXPECT_TRUE(reader.current_entry_info()->is_encrypted()); + reader.Close(); + + ASSERT_TRUE(reader.Open(test_zip_file_)); + ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); + EXPECT_FALSE(reader.current_entry_info()->is_encrypted()); +} + // Verifies that the ZipReader class can extract a file from a zip archive // stored in memory. This test opens a zip archive in a std::string object, // extracts its content, and verifies the content is the same as the expected @@ -459,13 +403,13 @@ TEST_F(ZipReaderTest, OpenFromString) { "\x50\x75\x78\x0b\x00\x01\x04\x8e\xf0\x00\x00\x04\x88\x13\x00\x00" "\x50\x4b\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00\x4e\x00\x00\x00" "\x52\x00\x00\x00\x00\x00"; - std::string data(kTestData, arraysize(kTestData)); + std::string data(kTestData, base::size(kTestData)); ZipReader reader; ASSERT_TRUE(reader.OpenFromString(data)); base::FilePath target_path(FILE_PATH_LITERAL("test.txt")); - ASSERT_TRUE(reader.LocateAndOpenEntry(target_path)); - ASSERT_TRUE(reader.ExtractCurrentEntryToFilePath( - test_dir_.AppendASCII("test.txt"))); + ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); + ASSERT_TRUE(ExtractCurrentEntryToFilePath(&reader, + test_dir_.AppendASCII("test.txt"))); std::string actual; ASSERT_TRUE(base::ReadFileToString( @@ -481,15 +425,13 @@ TEST_F(ZipReaderTest, ExtractToFileAsync_RegularFile) { base::FilePath target_file = test_dir_.AppendASCII("quux.txt"); base::FilePath target_path(FILE_PATH_LITERAL("foo/bar/quux.txt")); ASSERT_TRUE(reader.Open(test_zip_file_)); - ASSERT_TRUE(reader.LocateAndOpenEntry(target_path)); + ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); reader.ExtractCurrentEntryToFilePathAsync( target_file, - base::Bind(&MockUnzipListener::OnUnzipSuccess, - listener.AsWeakPtr()), - base::Bind(&MockUnzipListener::OnUnzipFailure, - listener.AsWeakPtr()), - base::Bind(&MockUnzipListener::OnUnzipProgress, - listener.AsWeakPtr())); + base::BindOnce(&MockUnzipListener::OnUnzipSuccess, listener.AsWeakPtr()), + base::BindOnce(&MockUnzipListener::OnUnzipFailure, listener.AsWeakPtr()), + base::BindRepeating(&MockUnzipListener::OnUnzipProgress, + listener.AsWeakPtr())); EXPECT_EQ(0, listener.success_calls()); EXPECT_EQ(0, listener.failure_calls()); @@ -507,7 +449,7 @@ TEST_F(ZipReaderTest, ExtractToFileAsync_RegularFile) { const std::string md5 = base::MD5String(output); EXPECT_EQ(kQuuxExpectedMD5, md5); - int64 file_size = 0; + int64_t file_size = 0; ASSERT_TRUE(base::GetFileSize(target_file, &file_size)); EXPECT_EQ(file_size, listener.current_progress()); @@ -521,15 +463,13 @@ TEST_F(ZipReaderTest, ExtractToFileAsync_Directory) { base::FilePath target_file = test_dir_.AppendASCII("foo"); base::FilePath target_path(FILE_PATH_LITERAL("foo/")); ASSERT_TRUE(reader.Open(test_zip_file_)); - ASSERT_TRUE(reader.LocateAndOpenEntry(target_path)); + ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); reader.ExtractCurrentEntryToFilePathAsync( target_file, - base::Bind(&MockUnzipListener::OnUnzipSuccess, - listener.AsWeakPtr()), - base::Bind(&MockUnzipListener::OnUnzipFailure, - listener.AsWeakPtr()), - base::Bind(&MockUnzipListener::OnUnzipProgress, - listener.AsWeakPtr())); + base::BindOnce(&MockUnzipListener::OnUnzipSuccess, listener.AsWeakPtr()), + base::BindOnce(&MockUnzipListener::OnUnzipFailure, listener.AsWeakPtr()), + base::BindRepeating(&MockUnzipListener::OnUnzipProgress, + listener.AsWeakPtr())); EXPECT_EQ(0, listener.success_calls()); EXPECT_EQ(0, listener.failure_calls()); @@ -560,7 +500,7 @@ TEST_F(ZipReaderTest, ExtractCurrentEntryToString) { base::FilePath file_name = base::FilePath::FromUTF8Unsafe( base::StringPrintf("%d.txt", static_cast(i))); - ASSERT_TRUE(reader.LocateAndOpenEntry(file_name)); + ASSERT_TRUE(LocateAndOpenEntry(&reader, file_name)); if (i > 1) { // Off by one byte read limit: must fail. @@ -570,18 +510,59 @@ TEST_F(ZipReaderTest, ExtractCurrentEntryToString) { if (i > 0) { // Exact byte read limit: must pass. EXPECT_TRUE(reader.ExtractCurrentEntryToString(i, &contents)); - EXPECT_EQ(i, contents.size()); - EXPECT_EQ(0, memcmp(contents.c_str(), "0123456", i)); + EXPECT_EQ(base::StringPiece("0123456", i).as_string(), contents); } // More than necessary byte read limit: must pass. EXPECT_TRUE(reader.ExtractCurrentEntryToString(16, &contents)); - EXPECT_EQ(i, contents.size()); - EXPECT_EQ(0, memcmp(contents.c_str(), "0123456", i)); + EXPECT_EQ(base::StringPiece("0123456", i).as_string(), contents); } reader.Close(); } +TEST_F(ZipReaderTest, ExtractPartOfCurrentEntry) { + // test_mismatch_size.zip contains files with names from 0.txt to 7.txt with + // sizes from 0 to 7 bytes respectively, being the contents of each file a + // substring of "0123456" starting at '0'. + base::FilePath test_zip_file = + test_data_dir_.AppendASCII("test_mismatch_size.zip"); + + ZipReader reader; + std::string contents; + ASSERT_TRUE(reader.Open(test_zip_file)); + + base::FilePath file_name0 = base::FilePath::FromUTF8Unsafe("0.txt"); + ASSERT_TRUE(LocateAndOpenEntry(&reader, file_name0)); + EXPECT_TRUE(reader.ExtractCurrentEntryToString(0, &contents)); + EXPECT_EQ("", contents); + EXPECT_TRUE(reader.ExtractCurrentEntryToString(1, &contents)); + EXPECT_EQ("", contents); + + base::FilePath file_name1 = base::FilePath::FromUTF8Unsafe("1.txt"); + ASSERT_TRUE(LocateAndOpenEntry(&reader, file_name1)); + EXPECT_TRUE(reader.ExtractCurrentEntryToString(0, &contents)); + EXPECT_EQ("", contents); + EXPECT_TRUE(reader.ExtractCurrentEntryToString(1, &contents)); + EXPECT_EQ("0", contents); + EXPECT_TRUE(reader.ExtractCurrentEntryToString(2, &contents)); + EXPECT_EQ("0", contents); + + base::FilePath file_name4 = base::FilePath::FromUTF8Unsafe("4.txt"); + ASSERT_TRUE(LocateAndOpenEntry(&reader, file_name4)); + EXPECT_TRUE(reader.ExtractCurrentEntryToString(0, &contents)); + EXPECT_EQ("", contents); + EXPECT_FALSE(reader.ExtractCurrentEntryToString(2, &contents)); + EXPECT_EQ("01", contents); + EXPECT_TRUE(reader.ExtractCurrentEntryToString(4, &contents)); + EXPECT_EQ("0123", contents); + // Checks that entire file is extracted and function returns true when + // |max_read_bytes| is larger than file size. + EXPECT_TRUE(reader.ExtractCurrentEntryToString(5, &contents)); + EXPECT_EQ("0123", contents); + + reader.Close(); +} + // This test exposes http://crbug.com/430959, at least on OS X TEST_F(ZipReaderTest, DISABLED_LeakDetectionTest) { for (int i = 0; i < 100000; ++i) { @@ -603,8 +584,9 @@ TEST_F(ZipReaderTest, ExtractCurrentEntryPrepareFailure) { ZipReader reader; ASSERT_TRUE(reader.Open(test_zip_file_)); - ASSERT_TRUE(reader.LocateAndOpenEntry(target_path)); - ASSERT_FALSE(reader.ExtractCurrentEntry(&mock_writer)); + ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); + ASSERT_FALSE(reader.ExtractCurrentEntry( + &mock_writer, std::numeric_limits::max())); } // Test that when WriterDelegate::WriteBytes returns false, no other methods on @@ -621,8 +603,9 @@ TEST_F(ZipReaderTest, ExtractCurrentEntryWriteBytesFailure) { ZipReader reader; ASSERT_TRUE(reader.Open(test_zip_file_)); - ASSERT_TRUE(reader.LocateAndOpenEntry(target_path)); - ASSERT_FALSE(reader.ExtractCurrentEntry(&mock_writer)); + ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); + ASSERT_FALSE(reader.ExtractCurrentEntry( + &mock_writer, std::numeric_limits::max())); } // Test that extraction succeeds when the writer delegate reports all is well. @@ -633,13 +616,15 @@ TEST_F(ZipReaderTest, ExtractCurrentEntrySuccess) { .WillOnce(Return(true)); EXPECT_CALL(mock_writer, WriteBytes(_, _)) .WillRepeatedly(Return(true)); + EXPECT_CALL(mock_writer, SetTimeModified(_)); base::FilePath target_path(FILE_PATH_LITERAL("foo/bar/quux.txt")); ZipReader reader; ASSERT_TRUE(reader.Open(test_zip_file_)); - ASSERT_TRUE(reader.LocateAndOpenEntry(target_path)); - ASSERT_TRUE(reader.ExtractCurrentEntry(&mock_writer)); + ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); + ASSERT_TRUE(reader.ExtractCurrentEntry(&mock_writer, + std::numeric_limits::max())); } class FileWriterDelegateTest : public ::testing::Test { diff --git a/third_party/zlib/google/zip_unittest.cc b/third_party/zlib/google/zip_unittest.cc index eda9a68057..10f2ef7a97 100644 --- a/third_party/zlib/google/zip_unittest.cc +++ b/third_party/zlib/google/zip_unittest.cc @@ -2,18 +2,26 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +#include +#include + +#include #include #include #include +#include "base/bind.h" #include "base/files/file.h" #include "base/files/file_enumerator.h" #include "base/files/file_path.h" #include "base/files/file_util.h" #include "base/files/scoped_temp_dir.h" +#include "base/logging.h" +#include "base/macros.h" #include "base/path_service.h" #include "base/strings/string_util.h" #include "base/strings/stringprintf.h" +#include "build/build_config.h" #include "testing/gtest/include/gtest/gtest.h" #include "testing/platform_test.h" #include "third_party/zlib/google/zip.h" @@ -21,6 +29,105 @@ namespace { +bool CreateFile(const std::string& content, + base::FilePath* file_path, + base::File* file) { + if (!base::CreateTemporaryFile(file_path)) + return false; + + if (base::WriteFile(*file_path, content.data(), content.size()) == -1) + return false; + + *file = base::File( + *file_path, base::File::Flags::FLAG_OPEN | base::File::Flags::FLAG_READ); + return file->IsValid(); +} + +// A virtual file system containing: +// /test +// /test/foo.txt +// /test/bar/bar1.txt +// /test/bar/bar2.txt +// Used to test providing a custom zip::FileAccessor when unzipping. +class VirtualFileSystem : public zip::FileAccessor { + public: + static constexpr char kFooContent[] = "This is foo."; + static constexpr char kBar1Content[] = "This is bar."; + static constexpr char kBar2Content[] = "This is bar too."; + + VirtualFileSystem() { + base::FilePath test_dir(FILE_PATH_LITERAL("/test")); + base::FilePath foo_txt_path = test_dir.Append(FILE_PATH_LITERAL("foo.txt")); + + base::FilePath file_path; + base::File file; + bool success = CreateFile(kFooContent, &file_path, &file); + DCHECK(success); + files_[foo_txt_path] = std::move(file); + + base::FilePath bar_dir = test_dir.Append(FILE_PATH_LITERAL("bar")); + base::FilePath bar1_txt_path = + bar_dir.Append(FILE_PATH_LITERAL("bar1.txt")); + success = CreateFile(kBar1Content, &file_path, &file); + DCHECK(success); + files_[bar1_txt_path] = std::move(file); + + base::FilePath bar2_txt_path = + bar_dir.Append(FILE_PATH_LITERAL("bar2.txt")); + success = CreateFile(kBar2Content, &file_path, &file); + DCHECK(success); + files_[bar2_txt_path] = std::move(file); + + file_tree_[test_dir] = std::vector{ + DirectoryContentEntry(foo_txt_path, /*is_dir=*/false), + DirectoryContentEntry(bar_dir, /*is_dir=*/true)}; + file_tree_[bar_dir] = std::vector{ + DirectoryContentEntry(bar1_txt_path, /*is_dir=*/false), + DirectoryContentEntry(bar2_txt_path, /*is_dir=*/false)}; + } + ~VirtualFileSystem() override = default; + + private: + std::vector OpenFilesForReading( + const std::vector& paths) override { + std::vector files; + for (const auto& path : paths) { + auto iter = files_.find(path); + files.push_back(iter == files_.end() ? base::File() + : std::move(iter->second)); + } + return files; + } + + bool DirectoryExists(const base::FilePath& file) override { + return file_tree_.count(file) == 1 && files_.count(file) == 0; + } + + std::vector ListDirectoryContent( + const base::FilePath& dir) override { + auto iter = file_tree_.find(dir); + if (iter == file_tree_.end()) { + NOTREACHED(); + return std::vector(); + } + return iter->second; + } + + base::Time GetLastModifiedTime(const base::FilePath& path) override { + return base::Time::FromDoubleT(172097977); // Some random date. + } + + std::map> file_tree_; + std::map files_; + + DISALLOW_COPY_AND_ASSIGN(VirtualFileSystem); +}; + +// static +constexpr char VirtualFileSystem::kFooContent[]; +constexpr char VirtualFileSystem::kBar1Content[]; +constexpr char VirtualFileSystem::kBar2Content[]; + // Make the test a PlatformTest to setup autorelease pools properly on Mac. class ZipTest : public PlatformTest { protected: @@ -33,18 +140,18 @@ class ZipTest : public PlatformTest { PlatformTest::SetUp(); ASSERT_TRUE(temp_dir_.CreateUniqueTempDir()); - test_dir_ = temp_dir_.path(); + test_dir_ = temp_dir_.GetPath(); base::FilePath zip_path(test_dir_); - zip_contents_.insert(zip_path.AppendASCII("foo.txt")); - zip_path = zip_path.AppendASCII("foo"); + zip_contents_.insert(zip_path.Append(FILE_PATH_LITERAL("foo.txt"))); + zip_path = zip_path.Append(FILE_PATH_LITERAL("foo")); zip_contents_.insert(zip_path); - zip_contents_.insert(zip_path.AppendASCII("bar.txt")); - zip_path = zip_path.AppendASCII("bar"); + zip_contents_.insert(zip_path.Append(FILE_PATH_LITERAL("bar.txt"))); + zip_path = zip_path.Append(FILE_PATH_LITERAL("bar")); zip_contents_.insert(zip_path); - zip_contents_.insert(zip_path.AppendASCII("baz.txt")); - zip_contents_.insert(zip_path.AppendASCII("quux.txt")); - zip_contents_.insert(zip_path.AppendASCII(".hidden")); + zip_contents_.insert(zip_path.Append(FILE_PATH_LITERAL("baz.txt"))); + zip_contents_.insert(zip_path.Append(FILE_PATH_LITERAL("quux.txt"))); + zip_contents_.insert(zip_path.Append(FILE_PATH_LITERAL(".hidden"))); // Include a subset of files in |zip_file_list_| to test ZipFiles(). zip_file_list_.push_back(base::FilePath(FILE_PATH_LITERAL("foo.txt"))); @@ -59,7 +166,7 @@ class ZipTest : public PlatformTest { } bool GetTestDataDirectory(base::FilePath* path) { - bool success = PathService::Get(base::DIR_SOURCE_ROOT, path); + bool success = base::PathService::Get(base::DIR_SOURCE_ROOT, path); EXPECT_TRUE(success); if (!success) return false; @@ -82,18 +189,42 @@ class ZipTest : public PlatformTest { ASSERT_TRUE(base::PathExists(path)) << "no file " << path.value(); ASSERT_TRUE(zip::Unzip(path, test_dir_)); + base::FilePath original_dir; + ASSERT_TRUE(GetTestDataDirectory(&original_dir)); + original_dir = original_dir.AppendASCII("test"); + base::FileEnumerator files(test_dir_, true, base::FileEnumerator::FILES | base::FileEnumerator::DIRECTORIES); - base::FilePath next_path = files.Next(); + base::FilePath unzipped_entry_path = files.Next(); size_t count = 0; - while (!next_path.value().empty()) { - if (next_path.value().find(FILE_PATH_LITERAL(".svn")) == - base::FilePath::StringType::npos) { - EXPECT_EQ(zip_contents_.count(next_path), 1U) << - "Couldn't find " << next_path.value(); - count++; + while (!unzipped_entry_path.value().empty()) { + EXPECT_EQ(zip_contents_.count(unzipped_entry_path), 1U) + << "Couldn't find " << unzipped_entry_path.value(); + count++; + + if (base::PathExists(unzipped_entry_path) && + !base::DirectoryExists(unzipped_entry_path)) { + // It's a file, check its contents are what we zipped. + // TODO(774156): figure out why the commented out EXPECT_TRUE below + // fails on the build bots (but not on the try-bots). + base::FilePath relative_path; + bool append_relative_path_success = + test_dir_.AppendRelativePath(unzipped_entry_path, &relative_path); + if (!append_relative_path_success) { + LOG(ERROR) << "Append relative path failed, params: " + << test_dir_.value() << " and " + << unzipped_entry_path.value(); + } + base::FilePath original_path = original_dir.Append(relative_path); + LOG(ERROR) << "Comparing original " << original_path.value() + << " and unzipped file " << unzipped_entry_path.value() + << " result: " + << base::ContentsEqual(original_path, unzipped_entry_path); + // EXPECT_TRUE(base::ContentsEqual(original_path, unzipped_entry_path)) + // << "Contents differ between original " << original_path.value() + // << " and unzipped file " << unzipped_entry_path.value(); } - next_path = files.Next(); + unzipped_entry_path = files.Next(); } size_t expected_count = 0; @@ -118,9 +249,9 @@ class ZipTest : public PlatformTest { base::ScopedTempDir temp_dir; ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); - base::FilePath zip_file = temp_dir.path().AppendASCII("out.zip"); - base::FilePath src_dir = temp_dir.path().AppendASCII("input"); - base::FilePath out_dir = temp_dir.path().AppendASCII("output"); + base::FilePath zip_file = temp_dir.GetPath().AppendASCII("out.zip"); + base::FilePath src_dir = temp_dir.GetPath().AppendASCII("input"); + base::FilePath out_dir = temp_dir.GetPath().AppendASCII("output"); base::FilePath src_file = src_dir.AppendASCII("test.txt"); base::FilePath out_file = out_dir.AppendASCII("test.txt"); @@ -138,7 +269,8 @@ class ZipTest : public PlatformTest { base::Time::Now().LocalExplode(&now_parts); now_parts.second = now_parts.second & ~1; now_parts.millisecond = 0; - base::Time now_time = base::Time::FromLocalExploded(now_parts); + base::Time now_time; + EXPECT_TRUE(base::Time::FromLocalExploded(now_parts, &now_time)); EXPECT_EQ(1, base::WriteFile(src_file, "1", 1)); EXPECT_TRUE(base::TouchFile(src_file, base::Time::Now(), test_mtime)); @@ -208,6 +340,75 @@ TEST_F(ZipTest, UnzipEvil2) { ASSERT_FALSE(base::PathExists(evil_file)); } +TEST_F(ZipTest, UnzipWithFilter) { + auto filter = base::BindRepeating([](const base::FilePath& path) { + return path.BaseName().MaybeAsASCII() == "foo.txt"; + }); + base::FilePath path; + ASSERT_TRUE(GetTestDataDirectory(&path)); + ASSERT_TRUE(zip::UnzipWithFilterCallback(path.AppendASCII("test.zip"), + test_dir_, filter, false)); + // Only foo.txt should have been extracted. The following paths should not + // be extracted: + // foo/ + // foo/bar.txt + // foo/bar/ + // foo/bar/.hidden + // foo/bar/baz.txt + // foo/bar/quux.txt + ASSERT_TRUE(base::PathExists(test_dir_.AppendASCII("foo.txt"))); + base::FileEnumerator extractedFiles( + test_dir_, + false, // Do not enumerate recursively - the file must be in the root. + base::FileEnumerator::FileType::FILES); + int extracted_count = 0; + while (!extractedFiles.Next().empty()) + ++extracted_count; + ASSERT_EQ(1, extracted_count); + + base::FileEnumerator extractedDirs( + test_dir_, + false, // Do not enumerate recursively - we require zero directories. + base::FileEnumerator::FileType::DIRECTORIES); + extracted_count = 0; + while (!extractedDirs.Next().empty()) + ++extracted_count; + ASSERT_EQ(0, extracted_count); +} + +TEST_F(ZipTest, UnzipWithDelegates) { + auto filter = + base::BindRepeating([](const base::FilePath& path) { return true; }); + auto dir_creator = base::BindRepeating( + [](const base::FilePath& extract_dir, const base::FilePath& entry_path) { + return base::CreateDirectory(extract_dir.Append(entry_path)); + }, + test_dir_); + auto writer = base::BindRepeating( + [](const base::FilePath& extract_dir, const base::FilePath& entry_path) + -> std::unique_ptr { + return std::make_unique( + extract_dir.Append(entry_path)); + }, + test_dir_); + base::FilePath path; + ASSERT_TRUE(GetTestDataDirectory(&path)); + base::File file(path.AppendASCII("test.zip"), + base::File::Flags::FLAG_OPEN | base::File::Flags::FLAG_READ); + ASSERT_TRUE(zip::UnzipWithFilterAndWriters(file.GetPlatformFile(), writer, + dir_creator, filter, false)); + base::FilePath dir = test_dir_; + base::FilePath dir_foo = dir.AppendASCII("foo"); + base::FilePath dir_foo_bar = dir_foo.AppendASCII("bar"); + ASSERT_TRUE(base::PathExists(dir.AppendASCII("foo.txt"))); + ASSERT_TRUE(base::PathExists(dir_foo)); + ASSERT_TRUE(base::PathExists(dir_foo.AppendASCII("bar.txt"))); + ASSERT_TRUE(base::PathExists(dir_foo_bar)); + ASSERT_TRUE(base::PathExists(dir_foo_bar.AppendASCII(".hidden"))); + ASSERT_TRUE(base::PathExists(dir_foo_bar.AppendASCII("baz.txt"))); + ASSERT_TRUE(base::PathExists(dir_foo_bar.AppendASCII("quux.txt"))); +} + TEST_F(ZipTest, Zip) { base::FilePath src_dir; ASSERT_TRUE(GetTestDataDirectory(&src_dir)); @@ -215,9 +416,9 @@ TEST_F(ZipTest, Zip) { base::ScopedTempDir temp_dir; ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); - base::FilePath zip_file = temp_dir.path().AppendASCII("out.zip"); + base::FilePath zip_file = temp_dir.GetPath().AppendASCII("out.zip"); - EXPECT_TRUE(zip::Zip(src_dir, zip_file, true)); + EXPECT_TRUE(zip::Zip(src_dir, zip_file, /*include_hidden_files=*/true)); TestUnzipFile(zip_file, true); } @@ -228,9 +429,9 @@ TEST_F(ZipTest, ZipIgnoreHidden) { base::ScopedTempDir temp_dir; ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); - base::FilePath zip_file = temp_dir.path().AppendASCII("out.zip"); + base::FilePath zip_file = temp_dir.GetPath().AppendASCII("out.zip"); - EXPECT_TRUE(zip::Zip(src_dir, zip_file, false)); + EXPECT_TRUE(zip::Zip(src_dir, zip_file, /*include_hidden_files=*/false)); TestUnzipFile(zip_file, false); } @@ -242,11 +443,10 @@ TEST_F(ZipTest, ZipNonASCIIDir) { base::ScopedTempDir temp_dir; ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); // Append 'Тест' (in cyrillic). - base::FilePath src_dir_russian = - temp_dir.path().Append(base::FilePath::FromUTF8Unsafe( - "\xD0\xA2\xD0\xB5\xD1\x81\xD1\x82")); + base::FilePath src_dir_russian = temp_dir.GetPath().Append( + base::FilePath::FromUTF8Unsafe("\xD0\xA2\xD0\xB5\xD1\x81\xD1\x82")); base::CopyDirectory(src_dir, src_dir_russian, true); - base::FilePath zip_file = temp_dir.path().AppendASCII("out_russian.zip"); + base::FilePath zip_file = temp_dir.GetPath().AppendASCII("out_russian.zip"); EXPECT_TRUE(zip::Zip(src_dir_russian, zip_file, true)); TestUnzipFile(zip_file, true); @@ -282,7 +482,7 @@ TEST_F(ZipTest, ZipFiles) { base::ScopedTempDir temp_dir; ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); - base::FilePath zip_name = temp_dir.path().AppendASCII("out.zip"); + base::FilePath zip_name = temp_dir.GetPath().AppendASCII("out.zip"); base::File zip_file(zip_name, base::File::FLAG_CREATE | base::File::FLAG_WRITE); @@ -295,10 +495,11 @@ TEST_F(ZipTest, ZipFiles) { EXPECT_TRUE(reader.Open(zip_name)); EXPECT_EQ(zip_file_list_.size(), static_cast(reader.num_entries())); for (size_t i = 0; i < zip_file_list_.size(); ++i) { - EXPECT_TRUE(reader.LocateAndOpenEntry(zip_file_list_[i])); - // Check the path in the entry just in case. + EXPECT_TRUE(reader.HasMore()); + EXPECT_TRUE(reader.OpenCurrentEntryInZip()); const zip::ZipReader::EntryInfo* entry_info = reader.current_entry_info(); EXPECT_EQ(entry_info->file_path(), zip_file_list_[i]); + reader.AdvanceToNextEntry(); } } #endif // defined(OS_POSIX) @@ -316,7 +517,7 @@ TEST_F(ZipTest, UnzipFilesWithIncorrectSize) { base::ScopedTempDir scoped_temp_dir; ASSERT_TRUE(scoped_temp_dir.CreateUniqueTempDir()); - const base::FilePath& temp_dir = scoped_temp_dir.path(); + const base::FilePath& temp_dir = scoped_temp_dir.GetPath(); ASSERT_TRUE(zip::Unzip(test_zip_file, temp_dir)); EXPECT_TRUE(base::DirectoryExists(temp_dir.AppendASCII("d"))); @@ -325,10 +526,35 @@ TEST_F(ZipTest, UnzipFilesWithIncorrectSize) { SCOPED_TRACE(base::StringPrintf("Processing %d.txt", i)); base::FilePath file_path = temp_dir.AppendASCII( base::StringPrintf("%d.txt", i)); - int64 file_size = -1; + int64_t file_size = -1; EXPECT_TRUE(base::GetFileSize(file_path, &file_size)); - EXPECT_EQ(static_cast(i), file_size); + EXPECT_EQ(static_cast(i), file_size); } } +TEST_F(ZipTest, ZipWithFileAccessor) { + base::FilePath zip_file; + ASSERT_TRUE(base::CreateTemporaryFile(&zip_file)); + zip::ZipParams params(base::FilePath(FILE_PATH_LITERAL("/test")), zip_file); + params.set_file_accessor(std::make_unique()); + ASSERT_TRUE(zip::Zip(params)); + + base::ScopedTempDir scoped_temp_dir; + ASSERT_TRUE(scoped_temp_dir.CreateUniqueTempDir()); + const base::FilePath& temp_dir = scoped_temp_dir.GetPath(); + ASSERT_TRUE(zip::Unzip(zip_file, temp_dir)); + base::FilePath bar_dir = temp_dir.Append(FILE_PATH_LITERAL("bar")); + EXPECT_TRUE(base::DirectoryExists(bar_dir)); + std::string file_content; + EXPECT_TRUE(base::ReadFileToString( + temp_dir.Append(FILE_PATH_LITERAL("foo.txt")), &file_content)); + EXPECT_EQ(VirtualFileSystem::kFooContent, file_content); + EXPECT_TRUE(base::ReadFileToString( + bar_dir.Append(FILE_PATH_LITERAL("bar1.txt")), &file_content)); + EXPECT_EQ(VirtualFileSystem::kBar1Content, file_content); + EXPECT_TRUE(base::ReadFileToString( + bar_dir.Append(FILE_PATH_LITERAL("bar2.txt")), &file_content)); + EXPECT_EQ(VirtualFileSystem::kBar2Content, file_content); +} + } // namespace diff --git a/third_party/zlib/google/zip_writer.cc b/third_party/zlib/google/zip_writer.cc new file mode 100644 index 0000000000..6f38d42b6b --- /dev/null +++ b/third_party/zlib/google/zip_writer.cc @@ -0,0 +1,204 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "third_party/zlib/google/zip_writer.h" + +#include "base/files/file.h" +#include "base/logging.h" +#include "base/strings/string_util.h" +#include "third_party/zlib/google/zip_internal.h" + +namespace zip { +namespace internal { + +namespace { + +// Numbers of pending entries that trigger writting them to the ZIP file. +constexpr size_t kMaxPendingEntriesCount = 50; + +bool AddFileContentToZip(zipFile zip_file, + base::File file, + const base::FilePath& file_path) { + int num_bytes; + char buf[zip::internal::kZipBufSize]; + do { + num_bytes = file.ReadAtCurrentPos(buf, zip::internal::kZipBufSize); + + if (num_bytes > 0) { + if (zipWriteInFileInZip(zip_file, buf, num_bytes) != ZIP_OK) { + DLOG(ERROR) << "Could not write data to zip for path " + << file_path.value(); + return false; + } + } + } while (num_bytes > 0); + + return true; +} + +bool OpenNewFileEntry(zipFile zip_file, + const base::FilePath& path, + bool is_directory, + base::Time last_modified) { + std::string str_path = path.AsUTF8Unsafe(); +#if defined(OS_WIN) + base::ReplaceSubstringsAfterOffset(&str_path, 0u, "\\", "/"); +#endif + if (is_directory) + str_path += "/"; + + return zip::internal::ZipOpenNewFileInZip(zip_file, str_path, last_modified); +} + +bool CloseNewFileEntry(zipFile zip_file) { + return zipCloseFileInZip(zip_file) == ZIP_OK; +} + +bool AddFileEntryToZip(zipFile zip_file, + const base::FilePath& path, + base::File file) { + base::File::Info file_info; + if (!file.GetInfo(&file_info)) + return false; + + if (!OpenNewFileEntry(zip_file, path, /*is_directory=*/false, + file_info.last_modified)) + return false; + + bool success = AddFileContentToZip(zip_file, std::move(file), path); + if (!CloseNewFileEntry(zip_file)) + return false; + + return success; +} + +bool AddDirectoryEntryToZip(zipFile zip_file, + const base::FilePath& path, + base::Time last_modified) { + return OpenNewFileEntry(zip_file, path, /*is_directory=*/true, + last_modified) && + CloseNewFileEntry(zip_file); +} + +} // namespace + +#if defined(OS_POSIX) +// static +std::unique_ptr ZipWriter::CreateWithFd( + int zip_file_fd, + const base::FilePath& root_dir, + FileAccessor* file_accessor) { + DCHECK(zip_file_fd != base::kInvalidPlatformFile); + zipFile zip_file = + internal::OpenFdForZipping(zip_file_fd, APPEND_STATUS_CREATE); + if (!zip_file) { + DLOG(ERROR) << "Couldn't create ZIP file for FD " << zip_file_fd; + return nullptr; + } + return std::unique_ptr( + new ZipWriter(zip_file, root_dir, file_accessor)); +} +#endif + +// static +std::unique_ptr ZipWriter::Create( + const base::FilePath& zip_file_path, + const base::FilePath& root_dir, + FileAccessor* file_accessor) { + DCHECK(!zip_file_path.empty()); + zipFile zip_file = internal::OpenForZipping(zip_file_path.AsUTF8Unsafe(), + APPEND_STATUS_CREATE); + if (!zip_file) { + DLOG(ERROR) << "Couldn't create ZIP file at path " << zip_file_path; + return nullptr; + } + return std::unique_ptr( + new ZipWriter(zip_file, root_dir, file_accessor)); +} + +ZipWriter::ZipWriter(zipFile zip_file, + const base::FilePath& root_dir, + FileAccessor* file_accessor) + : zip_file_(zip_file), root_dir_(root_dir), file_accessor_(file_accessor) {} + +ZipWriter::~ZipWriter() { + DCHECK(pending_entries_.empty()); +} + +bool ZipWriter::WriteEntries(const std::vector& paths) { + return AddEntries(paths) && Close(); +} + +bool ZipWriter::AddEntries(const std::vector& paths) { + DCHECK(zip_file_); + pending_entries_.insert(pending_entries_.end(), paths.begin(), paths.end()); + return FlushEntriesIfNeeded(/*force=*/false); +} + +bool ZipWriter::Close() { + bool success = FlushEntriesIfNeeded(/*force=*/true) && + zipClose(zip_file_, nullptr) == ZIP_OK; + zip_file_ = nullptr; + return success; +} + +bool ZipWriter::FlushEntriesIfNeeded(bool force) { + if (pending_entries_.size() < kMaxPendingEntriesCount && !force) + return true; + + while (pending_entries_.size() >= kMaxPendingEntriesCount || + (force && !pending_entries_.empty())) { + size_t entry_count = + std::min(pending_entries_.size(), kMaxPendingEntriesCount); + std::vector relative_paths; + std::vector absolute_paths; + relative_paths.insert(relative_paths.begin(), pending_entries_.begin(), + pending_entries_.begin() + entry_count); + for (auto iter = pending_entries_.begin(); + iter != pending_entries_.begin() + entry_count; ++iter) { + // The FileAccessor requires absolute paths. + absolute_paths.push_back(root_dir_.Append(*iter)); + } + pending_entries_.erase(pending_entries_.begin(), + pending_entries_.begin() + entry_count); + + // We don't know which paths are files and which ones are directories, and + // we want to avoid making a call to file_accessor_ for each entry. Open the + // files instead, invalid files are returned for directories. + std::vector files = + file_accessor_->OpenFilesForReading(absolute_paths); + DCHECK_EQ(files.size(), relative_paths.size()); + for (size_t i = 0; i < files.size(); i++) { + const base::FilePath& relative_path = relative_paths[i]; + const base::FilePath& absolute_path = absolute_paths[i]; + base::File file = std::move(files[i]); + if (file.IsValid()) { + if (!AddFileEntryToZip(zip_file_, relative_path, std::move(file))) { + LOG(ERROR) << "Failed to write file " << relative_path.value() + << " to ZIP file."; + return false; + } + } else { + // Missing file or directory case. + base::Time last_modified = + file_accessor_->GetLastModifiedTime(absolute_path); + if (last_modified.is_null()) { + LOG(ERROR) << "Failed to write entry " << relative_path.value() + << " to ZIP file."; + return false; + } + DCHECK(file_accessor_->DirectoryExists(absolute_path)); + if (!AddDirectoryEntryToZip(zip_file_, relative_path, last_modified)) { + LOG(ERROR) << "Failed to write directory " << relative_path.value() + << " to ZIP file."; + return false; + } + } + } + } + return true; +} + +} // namespace internal +} // namespace zip diff --git a/third_party/zlib/google/zip_writer.h b/third_party/zlib/google/zip_writer.h new file mode 100644 index 0000000000..bd2a727b94 --- /dev/null +++ b/third_party/zlib/google/zip_writer.h @@ -0,0 +1,91 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef THIRD_PARTY_ZLIB_GOOGLE_ZIP_WRITER_H_ +#define THIRD_PARTY_ZLIB_GOOGLE_ZIP_WRITER_H_ + +#include +#include + +#include "base/files/file_path.h" +#include "build/build_config.h" +#include "third_party/zlib/google/zip.h" + +#if defined(USE_SYSTEM_MINIZIP) +#include +#include +#else +#include "third_party/zlib/contrib/minizip/unzip.h" +#include "third_party/zlib/contrib/minizip/zip.h" +#endif + +namespace zip { +namespace internal { + +// A class used to write entries to a ZIP file and buffering the reading of +// files to limit the number of calls to the FileAccessor. This is for +// performance reasons as these calls may be expensive when IPC based). +// This class is so far internal and only used by zip.cc, but could be made +// public if needed. +class ZipWriter { + public: +// Creates a writer that will write a ZIP file to |zip_file_fd|/|zip_file| +// and which entries (specifies with AddEntries) are relative to |root_dir|. +// All file reads are performed using |file_accessor|. +#if defined(OS_POSIX) + static std::unique_ptr CreateWithFd(int zip_file_fd, + const base::FilePath& root_dir, + FileAccessor* file_accessor); +#endif + static std::unique_ptr Create(const base::FilePath& zip_file, + const base::FilePath& root_dir, + FileAccessor* file_accessor); + ~ZipWriter(); + + // Writes the files at |paths| to the ZIP file and closes this Zip file. + // Note that the the FilePaths must be relative to |root_dir| specified in the + // Create method. + // Returns true if all entries were written successfuly. + bool WriteEntries(const std::vector& paths); + + private: + ZipWriter(zipFile zip_file, + const base::FilePath& root_dir, + FileAccessor* file_accessor); + + // Writes the pending entries to the ZIP file if there are at least + // |kMaxPendingEntriesCount| of them. If |force| is true, all pending entries + // are written regardless of how many there are. + // Returns false if writing an entry fails, true if no entry was written or + // there was no error writing entries. + bool FlushEntriesIfNeeded(bool force); + + // Adds the files at |paths| to the ZIP file. These FilePaths must be relative + // to |root_dir| specified in the Create method. + bool AddEntries(const std::vector& paths); + + // Closes the ZIP file. + // Returns true if successful, false otherwise (typically if an entry failed + // to be written). + bool Close(); + + // The entries that have been added but not yet written to the ZIP file. + std::vector pending_entries_; + + // The actual zip file. + zipFile zip_file_; + + // Path to the directory entry paths are relative to. + base::FilePath root_dir_; + + // Abstraction over file access methods used to read files. + FileAccessor* file_accessor_; + + DISALLOW_COPY_AND_ASSIGN(ZipWriter); +}; + +} // namespace internal +} // namespace zip + +#endif // THIRD_PARTY_ZLIB_GOOGLE_ZIP_WRITER_H_ \ No newline at end of file diff --git a/third_party/zlib/gzguts.h b/third_party/zlib/gzguts.h index 0f8fb79f87..990a4d2514 100644 --- a/third_party/zlib/gzguts.h +++ b/third_party/zlib/gzguts.h @@ -1,5 +1,5 @@ /* gzguts.h -- zlib internal header definitions for gz* operations - * Copyright (C) 2004, 2005, 2010 Mark Adler + * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -12,7 +12,7 @@ # endif #endif -#if ((__GNUC__-0) * 10 + __GNUC_MINOR__-0 >= 33) && !defined(NO_VIZ) +#ifdef HAVE_HIDDEN # define ZLIB_INTERNAL __attribute__((visibility ("hidden"))) #else # define ZLIB_INTERNAL @@ -25,21 +25,97 @@ # include # include #endif + +#ifndef _POSIX_SOURCE +# define _POSIX_SOURCE +#endif #include +#ifdef _WIN32 +# include +#endif + +#if defined(__TURBOC__) || defined(_MSC_VER) || defined(_WIN32) +# include +#endif + +#if defined(_WIN32) || defined(__CYGWIN__) +# define WIDECHAR +#endif + +#ifdef WINAPI_FAMILY +# define open _open +# define read _read +# define write _write +# define close _close +#endif + #ifdef NO_DEFLATE /* for compatibility with old definition */ # define NO_GZCOMPRESS #endif -#ifdef _MSC_VER -# include -# define vsnprintf _vsnprintf +#if defined(STDC99) || (defined(__TURBOC__) && __TURBOC__ >= 0x550) +# ifndef HAVE_VSNPRINTF +# define HAVE_VSNPRINTF +# endif +#endif + +#if defined(__CYGWIN__) +# ifndef HAVE_VSNPRINTF +# define HAVE_VSNPRINTF +# endif +#endif + +#if defined(MSDOS) && defined(__BORLANDC__) && (BORLANDC > 0x410) +# ifndef HAVE_VSNPRINTF +# define HAVE_VSNPRINTF +# endif +#endif + +#ifndef HAVE_VSNPRINTF +# ifdef MSDOS +/* vsnprintf may exist on some MS-DOS compilers (DJGPP?), + but for now we just assume it doesn't. */ +# define NO_vsnprintf +# endif +# ifdef __TURBOC__ +# define NO_vsnprintf +# endif +# ifdef WIN32 +/* In Win32, vsnprintf is available as the "non-ANSI" _vsnprintf. */ +# if !defined(vsnprintf) && !defined(NO_vsnprintf) +# if !defined(_MSC_VER) || ( defined(_MSC_VER) && _MSC_VER < 1500 ) +# define vsnprintf _vsnprintf +# endif +# endif +# endif +# ifdef __SASC +# define NO_vsnprintf +# endif +# ifdef VMS +# define NO_vsnprintf +# endif +# ifdef __OS400__ +# define NO_vsnprintf +# endif +# ifdef __MVS__ +# define NO_vsnprintf +# endif +#endif + +/* unlike snprintf (which is required in C99), _snprintf does not guarantee + null termination of the result -- however this is only used in gzlib.c where + the result is assured to fit in the space provided */ +#if defined(_MSC_VER) && _MSC_VER < 1900 +# define snprintf _snprintf #endif #ifndef local # define local static #endif -/* compile with -Dlocal if your debugger can't find static symbols */ +/* since "static" is used to mean two completely different things in C, we + define "local" for the non-static meaning of "static", for readability + (compile with -Dlocal if your debugger can't find static symbols) */ /* gz* functions always use library allocation functions */ #ifndef STDC @@ -52,7 +128,7 @@ # include # define zstrerror() gz_strwinerror((DWORD)GetLastError()) #else -# ifdef STDC +# ifndef NO_STRERROR # include # define zstrerror() strerror(errno) # else @@ -68,7 +144,15 @@ ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile)); #endif -/* default i/o buffer size -- double this for output when reading */ +/* default memLevel */ +#if MAX_MEM_LEVEL >= 8 +# define DEF_MEM_LEVEL 8 +#else +# define DEF_MEM_LEVEL MAX_MEM_LEVEL +#endif + +/* default i/o buffer size -- double this for output when reading (this and + twice this must be able to fit in an unsigned type) */ #define GZBUFSIZE 8192 /* gzip modes, also provide a little integrity check on the passed structure */ @@ -84,23 +168,25 @@ /* internal gzip file state data structure */ typedef struct { + /* exposed contents for gzgetc() macro */ + struct gzFile_s x; /* "x" for exposed */ + /* x.have: number of bytes available at x.next */ + /* x.next: next output data to deliver or write */ + /* x.pos: current position in uncompressed data */ /* used for both reading and writing */ int mode; /* see gzip modes above */ int fd; /* file descriptor */ char *path; /* path or fd for error messages */ - z_off64_t pos; /* current position in uncompressed data */ unsigned size; /* buffer size, zero if not allocated yet */ unsigned want; /* requested buffer size, default is GZBUFSIZE */ - unsigned char *in; /* input buffer */ + unsigned char *in; /* input buffer (double-sized when writing) */ unsigned char *out; /* output buffer (double-sized when reading) */ - unsigned char *next; /* next output data to deliver or write */ + int direct; /* 0 if processing gzip, 1 if transparent */ /* just for reading */ - unsigned have; /* amount of output data unused at next */ - int eof; /* true if end of input file reached */ - z_off64_t start; /* where the gzip data started, for rewinding */ - z_off64_t raw; /* where the raw data started, for seeking */ int how; /* 0: get header, 1: copy, 2: decompress */ - int direct; /* true if last read direct, false if gzip */ + z_off64_t start; /* where the gzip data started, for rewinding */ + int eof; /* true if end of input file reached */ + int past; /* true if read requested past end */ /* just for writing */ int level; /* compression level */ int strategy; /* compression strategy */ diff --git a/third_party/zlib/gzlib.c b/third_party/zlib/gzlib.c index 54e8344429..4105e6aff9 100644 --- a/third_party/zlib/gzlib.c +++ b/third_party/zlib/gzlib.c @@ -1,21 +1,23 @@ /* gzlib.c -- zlib functions common to reading and writing gzip files - * Copyright (C) 2004, 2010 Mark Adler + * Copyright (C) 2004-2017 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ #include "gzguts.h" -#if defined(_WIN32) && !defined(__BORLANDC__) -# define LSEEK (z_off64_t)_lseeki64 -#elif defined(_LARGEFILE64_SOURCE) && _LFS64_LARGEFILE-0 +#if defined(_WIN32) && !defined(__BORLANDC__) && !defined(__MINGW32__) +# define LSEEK _lseeki64 +#else +#if defined(_LARGEFILE64_SOURCE) && _LFS64_LARGEFILE-0 # define LSEEK lseek64 #else # define LSEEK lseek #endif +#endif /* Local functions */ local void gz_reset OF((gz_statep)); -local gzFile gz_open OF((const char *, int, const char *)); +local gzFile gz_open OF((const void *, int, const char *)); #if defined UNDER_CE @@ -73,28 +75,40 @@ char ZLIB_INTERNAL *gz_strwinerror (error) local void gz_reset(state) gz_statep state; { + state->x.have = 0; /* no output data available */ if (state->mode == GZ_READ) { /* for reading ... */ - state->have = 0; /* no output data available */ state->eof = 0; /* not at end of file */ + state->past = 0; /* have not read past end yet */ state->how = LOOK; /* look for gzip header */ - state->direct = 1; /* default for empty file */ } state->seek = 0; /* no seek request pending */ gz_error(state, Z_OK, NULL); /* clear error */ - state->pos = 0; /* no uncompressed data yet */ + state->x.pos = 0; /* no uncompressed data yet */ state->strm.avail_in = 0; /* no input data yet */ } /* Open a gzip file either by name or file descriptor. */ local gzFile gz_open(path, fd, mode) - const char *path; + const void *path; int fd; const char *mode; { gz_statep state; + z_size_t len; + int oflag; +#ifdef O_CLOEXEC + int cloexec = 0; +#endif +#ifdef O_EXCL + int exclusive = 0; +#endif + + /* check input */ + if (path == NULL) + return NULL; /* allocate gzFile structure to return */ - state = malloc(sizeof(gz_state)); + state = (gz_statep)malloc(sizeof(gz_state)); if (state == NULL) return NULL; state->size = 0; /* no buffers allocated yet */ @@ -105,6 +119,7 @@ local gzFile gz_open(path, fd, mode) state->mode = GZ_NONE; state->level = Z_DEFAULT_COMPRESSION; state->strategy = Z_DEFAULT_STRATEGY; + state->direct = 0; while (*mode) { if (*mode >= '0' && *mode <= '9') state->level = *mode - '0'; @@ -126,6 +141,16 @@ local gzFile gz_open(path, fd, mode) return NULL; case 'b': /* ignore -- will request binary anyway */ break; +#ifdef O_CLOEXEC + case 'e': + cloexec = 1; + break; +#endif +#ifdef O_EXCL + case 'x': + exclusive = 1; + break; +#endif case 'f': state->strategy = Z_FILTERED; break; @@ -137,6 +162,10 @@ local gzFile gz_open(path, fd, mode) break; case 'F': state->strategy = Z_FIXED; + break; + case 'T': + state->direct = 1; + break; default: /* could consider as an error, but just ignore */ ; } @@ -149,37 +178,80 @@ local gzFile gz_open(path, fd, mode) return NULL; } + /* can't force transparent read */ + if (state->mode == GZ_READ) { + if (state->direct) { + free(state); + return NULL; + } + state->direct = 1; /* for empty file */ + } + /* save the path name for error messages */ - state->path = malloc(strlen(path) + 1); +#ifdef WIDECHAR + if (fd == -2) { + len = wcstombs(NULL, path, 0); + if (len == (z_size_t)-1) + len = 0; + } + else +#endif + len = strlen((const char *)path); + state->path = (char *)malloc(len + 1); if (state->path == NULL) { free(state); return NULL; } - strcpy(state->path, path); +#ifdef WIDECHAR + if (fd == -2) + if (len) + wcstombs(state->path, path, len + 1); + else + *(state->path) = 0; + else +#endif +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + (void)snprintf(state->path, len + 1, "%s", (const char *)path); +#else + strcpy(state->path, path); +#endif - /* open the file with the appropriate mode (or just use fd) */ - state->fd = fd != -1 ? fd : - open(path, + /* compute the flags for open() */ + oflag = #ifdef O_LARGEFILE - O_LARGEFILE | + O_LARGEFILE | #endif #ifdef O_BINARY - O_BINARY | + O_BINARY | +#endif +#ifdef O_CLOEXEC + (cloexec ? O_CLOEXEC : 0) | #endif - (state->mode == GZ_READ ? - O_RDONLY : - (O_WRONLY | O_CREAT | ( - state->mode == GZ_WRITE ? - O_TRUNC : - O_APPEND))), - 0666); + (state->mode == GZ_READ ? + O_RDONLY : + (O_WRONLY | O_CREAT | +#ifdef O_EXCL + (exclusive ? O_EXCL : 0) | +#endif + (state->mode == GZ_WRITE ? + O_TRUNC : + O_APPEND))); + + /* open the file with the appropriate flags (or just use fd) */ + state->fd = fd > -1 ? fd : ( +#ifdef WIDECHAR + fd == -2 ? _wopen(path, oflag, 0666) : +#endif + open((const char *)path, oflag, 0666)); if (state->fd == -1) { free(state->path); free(state); return NULL; } - if (state->mode == GZ_APPEND) + if (state->mode == GZ_APPEND) { + LSEEK(state->fd, 0, SEEK_END); /* so gzoffset() is correct */ state->mode = GZ_WRITE; /* simplify later checks */ + } /* save the current position for rewinding (only if reading) */ if (state->mode == GZ_READ) { @@ -218,14 +290,28 @@ gzFile ZEXPORT gzdopen(fd, mode) char *path; /* identifier for error messages */ gzFile gz; - if (fd == -1 || (path = malloc(7 + 3 * sizeof(int))) == NULL) + if (fd == -1 || (path = (char *)malloc(7 + 3 * sizeof(int))) == NULL) return NULL; +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + (void)snprintf(path, 7 + 3 * sizeof(int), "", fd); +#else sprintf(path, "", fd); /* for debugging */ +#endif gz = gz_open(path, fd, mode); free(path); return gz; } +/* -- see zlib.h -- */ +#ifdef WIDECHAR +gzFile ZEXPORT gzopen_w(path, mode) + const wchar_t *path; + const char *mode; +{ + return gz_open(path, -2, mode); +} +#endif + /* -- see zlib.h -- */ int ZEXPORT gzbuffer(file, size) gzFile file; @@ -245,8 +331,10 @@ int ZEXPORT gzbuffer(file, size) return -1; /* check and set requested size */ - if (size == 0) - return -1; + if ((size << 1) < size) + return -1; /* need to be able to double it */ + if (size < 2) + size = 2; /* need two bytes to check magic header */ state->want = size; return 0; } @@ -263,7 +351,8 @@ int ZEXPORT gzrewind(file) state = (gz_statep)file; /* check that we're reading and that there's no error */ - if (state->mode != GZ_READ || state->err != Z_OK) + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) return -1; /* back up and start over */ @@ -291,7 +380,7 @@ z_off64_t ZEXPORT gzseek64(file, offset, whence) return -1; /* check that there's no error */ - if (state->err != Z_OK) + if (state->err != Z_OK && state->err != Z_BUF_ERROR) return -1; /* can only seek from start or relative to current position */ @@ -300,31 +389,32 @@ z_off64_t ZEXPORT gzseek64(file, offset, whence) /* normalize offset to a SEEK_CUR specification */ if (whence == SEEK_SET) - offset -= state->pos; + offset -= state->x.pos; else if (state->seek) offset += state->skip; state->seek = 0; /* if within raw area while reading, just go there */ if (state->mode == GZ_READ && state->how == COPY && - state->pos + offset >= state->raw) { - ret = LSEEK(state->fd, offset - state->have, SEEK_CUR); + state->x.pos + offset >= 0) { + ret = LSEEK(state->fd, offset - state->x.have, SEEK_CUR); if (ret == -1) return -1; - state->have = 0; + state->x.have = 0; state->eof = 0; + state->past = 0; state->seek = 0; gz_error(state, Z_OK, NULL); state->strm.avail_in = 0; - state->pos += offset; - return state->pos; + state->x.pos += offset; + return state->x.pos; } /* calculate skip amount, rewinding if needed for back seek when reading */ if (offset < 0) { if (state->mode != GZ_READ) /* writing -- can't go backwards */ return -1; - offset += state->pos; + offset += state->x.pos; if (offset < 0) /* before start of file! */ return -1; if (gzrewind(file) == -1) /* rewind, then skip to offset */ @@ -333,11 +423,11 @@ z_off64_t ZEXPORT gzseek64(file, offset, whence) /* if reading, skip what's in output buffer (one less gzgetc() check) */ if (state->mode == GZ_READ) { - n = GT_OFF(state->have) || (z_off64_t)state->have > offset ? - (unsigned)offset : state->have; - state->have -= n; - state->next += n; - state->pos += n; + n = GT_OFF(state->x.have) || (z_off64_t)state->x.have > offset ? + (unsigned)offset : state->x.have; + state->x.have -= n; + state->x.next += n; + state->x.pos += n; offset -= n; } @@ -346,7 +436,7 @@ z_off64_t ZEXPORT gzseek64(file, offset, whence) state->seek = 1; state->skip = offset; } - return state->pos + offset; + return state->x.pos + offset; } /* -- see zlib.h -- */ @@ -375,7 +465,7 @@ z_off64_t ZEXPORT gztell64(file) return -1; /* return position */ - return state->pos + (state->seek ? state->skip : 0); + return state->x.pos + (state->seek ? state->skip : 0); } /* -- see zlib.h -- */ @@ -435,8 +525,7 @@ int ZEXPORT gzeof(file) return 0; /* return end-of-file state */ - return state->mode == GZ_READ ? - (state->eof && state->strm.avail_in == 0 && state->have == 0) : 0; + return state->mode == GZ_READ ? state->past : 0; } /* -- see zlib.h -- */ @@ -456,7 +545,8 @@ const char * ZEXPORT gzerror(file, errnum) /* return error information */ if (errnum != NULL) *errnum = state->err; - return state->msg == NULL ? "" : state->msg; + return state->err == Z_MEM_ERROR ? "out of memory" : + (state->msg == NULL ? "" : state->msg); } /* -- see zlib.h -- */ @@ -473,8 +563,10 @@ void ZEXPORT gzclearerr(file) return; /* clear error and end-of-file */ - if (state->mode == GZ_READ) + if (state->mode == GZ_READ) { state->eof = 0; + state->past = 0; + } gz_error(state, Z_OK, NULL); } @@ -496,27 +588,33 @@ void ZLIB_INTERNAL gz_error(state, err, msg) state->msg = NULL; } + /* if fatal, set state->x.have to 0 so that the gzgetc() macro fails */ + if (err != Z_OK && err != Z_BUF_ERROR) + state->x.have = 0; + /* set error code, and if no message, then done */ state->err = err; if (msg == NULL) return; - /* for an out of memory error, save as static string */ - if (err == Z_MEM_ERROR) { - state->msg = (char *)msg; + /* for an out of memory error, return literal string when requested */ + if (err == Z_MEM_ERROR) return; - } /* construct error message with path */ - if ((state->msg = malloc(strlen(state->path) + strlen(msg) + 3)) == NULL) { + if ((state->msg = (char *)malloc(strlen(state->path) + strlen(msg) + 3)) == + NULL) { state->err = Z_MEM_ERROR; - state->msg = (char *)"out of memory"; return; } +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + (void)snprintf(state->msg, strlen(state->path) + strlen(msg) + 3, + "%s%s%s", state->path, ": ", msg); +#else strcpy(state->msg, state->path); strcat(state->msg, ": "); strcat(state->msg, msg); - return; +#endif } #ifndef INT_MAX diff --git a/third_party/zlib/gzread.c b/third_party/zlib/gzread.c index 548201ab00..832d3ef98c 100644 --- a/third_party/zlib/gzread.c +++ b/third_party/zlib/gzread.c @@ -1,5 +1,5 @@ /* gzread.c -- zlib functions for reading gzip files - * Copyright (C) 2004, 2005, 2010 Mark Adler + * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -8,11 +8,11 @@ /* Local functions */ local int gz_load OF((gz_statep, unsigned char *, unsigned, unsigned *)); local int gz_avail OF((gz_statep)); -local int gz_next4 OF((gz_statep, unsigned long *)); -local int gz_head OF((gz_statep)); +local int gz_look OF((gz_statep)); local int gz_decomp OF((gz_statep)); -local int gz_make OF((gz_statep)); +local int gz_fetch OF((gz_statep)); local int gz_skip OF((gz_statep, z_off64_t)); +local z_size_t gz_read OF((gz_statep, voidp, z_size_t)); /* Use read() to load a buffer -- return -1 on error, otherwise 0. Read from state->fd, and update state->eof, state->err, and state->msg as appropriate. @@ -25,13 +25,17 @@ local int gz_load(state, buf, len, have) unsigned *have; { int ret; + unsigned get, max = ((unsigned)-1 >> 2) + 1; *have = 0; do { - ret = read(state->fd, buf + *have, len - *have); + get = len - *have; + if (get > max) + get = max; + ret = read(state->fd, buf + *have, get); if (ret <= 0) break; - *have += ret; + *have += (unsigned)ret; } while (*have < len); if (ret < 0) { gz_error(state, Z_ERRNO, zstrerror()); @@ -46,78 +50,57 @@ local int gz_load(state, buf, len, have) error, 0 otherwise. Note that the eof flag is set when the end of the input file is reached, even though there may be unused data in the buffer. Once that data has been used, no more attempts will be made to read the file. - gz_avail() assumes that strm->avail_in == 0. */ + If strm->avail_in != 0, then the current data is moved to the beginning of + the input buffer, and then the remainder of the buffer is loaded with the + available data from the input file. */ local int gz_avail(state) gz_statep state; { + unsigned got; z_streamp strm = &(state->strm); - if (state->err != Z_OK) + if (state->err != Z_OK && state->err != Z_BUF_ERROR) return -1; if (state->eof == 0) { - if (gz_load(state, state->in, state->size, - (unsigned *)&(strm->avail_in)) == -1) + if (strm->avail_in) { /* copy what's there to the start */ + unsigned char *p = state->in; + unsigned const char *q = strm->next_in; + unsigned n = strm->avail_in; + do { + *p++ = *q++; + } while (--n); + } + if (gz_load(state, state->in + strm->avail_in, + state->size - strm->avail_in, &got) == -1) return -1; + strm->avail_in += got; strm->next_in = state->in; } return 0; } -/* Get next byte from input, or -1 if end or error. */ -#define NEXT() ((strm->avail_in == 0 && gz_avail(state) == -1) ? -1 : \ - (strm->avail_in == 0 ? -1 : \ - (strm->avail_in--, *(strm->next_in)++))) - -/* Get a four-byte little-endian integer and return 0 on success and the value - in *ret. Otherwise -1 is returned and *ret is not modified. */ -local int gz_next4(state, ret) - gz_statep state; - unsigned long *ret; -{ - int ch; - unsigned long val; - z_streamp strm = &(state->strm); - - val = NEXT(); - val += (unsigned)NEXT() << 8; - val += (unsigned long)NEXT() << 16; - ch = NEXT(); - if (ch == -1) - return -1; - val += (unsigned long)ch << 24; - *ret = val; - return 0; -} - -/* Look for gzip header, set up for inflate or copy. state->have must be zero. +/* Look for gzip header, set up for inflate or copy. state->x.have must be 0. If this is the first time in, allocate required memory. state->how will be left unchanged if there is no more input data available, will be set to COPY if there is no gzip header and direct copying will be performed, or it will - be set to GZIP for decompression, and the gzip header will be skipped so - that the next available input data is the raw deflate stream. If direct - copying, then leftover input data from the input buffer will be copied to - the output buffer. In that case, all further file reads will be directly to - either the output buffer or a user buffer. If decompressing, the inflate - state and the check value will be initialized. gz_head() will return 0 on - success or -1 on failure. Failures may include read errors or gzip header - errors. */ -local int gz_head(state) + be set to GZIP for decompression. If direct copying, then leftover input + data from the input buffer will be copied to the output buffer. In that + case, all further file reads will be directly to either the output buffer or + a user buffer. If decompressing, the inflate state will be initialized. + gz_look() will return 0 on success or -1 on failure. */ +local int gz_look(state) gz_statep state; { z_streamp strm = &(state->strm); - int flags; - unsigned len; /* allocate read buffers and inflate memory */ if (state->size == 0) { /* allocate buffers */ - state->in = malloc(state->want); - state->out = malloc(state->want << 1); + state->in = (unsigned char *)malloc(state->want); + state->out = (unsigned char *)malloc(state->want << 1); if (state->in == NULL || state->out == NULL) { - if (state->out != NULL) - free(state->out); - if (state->in != NULL) - free(state->in); + free(state->out); + free(state->in); gz_error(state, Z_MEM_ERROR, "out of memory"); return -1; } @@ -129,7 +112,7 @@ local int gz_head(state) state->strm.opaque = Z_NULL; state->strm.avail_in = 0; state->strm.next_in = Z_NULL; - if (inflateInit2(&(state->strm), -15) != Z_OK) { /* raw inflate */ + if (inflateInit2(&(state->strm), 15 + 16) != Z_OK) { /* gunzip */ free(state->out); free(state->in); state->size = 0; @@ -138,83 +121,45 @@ local int gz_head(state) } } - /* get some data in the input buffer */ - if (strm->avail_in == 0) { + /* get at least the magic bytes in the input buffer */ + if (strm->avail_in < 2) { if (gz_avail(state) == -1) return -1; if (strm->avail_in == 0) return 0; } - /* look for the gzip magic header bytes 31 and 139 */ - if (strm->next_in[0] == 31) { - strm->avail_in--; - strm->next_in++; - if (strm->avail_in == 0 && gz_avail(state) == -1) - return -1; - if (strm->avail_in && strm->next_in[0] == 139) { - /* we have a gzip header, woo hoo! */ - strm->avail_in--; - strm->next_in++; - - /* skip rest of header */ - if (NEXT() != 8) { /* compression method */ - gz_error(state, Z_DATA_ERROR, "unknown compression method"); - return -1; - } - flags = NEXT(); - if (flags & 0xe0) { /* reserved flag bits */ - gz_error(state, Z_DATA_ERROR, "unknown header flags set"); - return -1; - } - NEXT(); /* modification time */ - NEXT(); - NEXT(); - NEXT(); - NEXT(); /* extra flags */ - NEXT(); /* operating system */ - if (flags & 4) { /* extra field */ - len = (unsigned)NEXT(); - len += (unsigned)NEXT() << 8; - while (len--) - if (NEXT() < 0) - break; - } - if (flags & 8) /* file name */ - while (NEXT() > 0) - ; - if (flags & 16) /* comment */ - while (NEXT() > 0) - ; - if (flags & 2) { /* header crc */ - NEXT(); - NEXT(); - } - /* an unexpected end of file is not checked for here -- it will be - noticed on the first request for uncompressed data */ - - /* set up for decompression */ - inflateReset(strm); - strm->adler = crc32(0L, Z_NULL, 0); - state->how = GZIP; - state->direct = 0; - return 0; - } - else { - /* not a gzip file -- save first byte (31) and fall to raw i/o */ - state->out[0] = 31; - state->have = 1; - } + /* look for gzip magic bytes -- if there, do gzip decoding (note: there is + a logical dilemma here when considering the case of a partially written + gzip file, to wit, if a single 31 byte is written, then we cannot tell + whether this is a single-byte file, or just a partially written gzip + file -- for here we assume that if a gzip file is being written, then + the header will be written in a single operation, so that reading a + single byte is sufficient indication that it is not a gzip file) */ + if (strm->avail_in > 1 && + strm->next_in[0] == 31 && strm->next_in[1] == 139) { + inflateReset(strm); + state->how = GZIP; + state->direct = 0; + return 0; + } + + /* no gzip header -- if we were decoding gzip before, then this is trailing + garbage. Ignore the trailing garbage and finish. */ + if (state->direct == 0) { + strm->avail_in = 0; + state->eof = 1; + state->x.have = 0; + return 0; } - /* doing raw i/o, save start of raw data for seeking, copy any leftover - input to output -- this assumes that the output buffer is larger than - the input buffer, which also assures space for gzungetc() */ - state->raw = state->pos; - state->next = state->out; + /* doing raw i/o, copy any leftover input to output -- this assumes that + the output buffer is larger than the input buffer, which also assures + space for gzungetc() */ + state->x.next = state->out; if (strm->avail_in) { - memcpy(state->next + state->have, strm->next_in, strm->avail_in); - state->have += strm->avail_in; + memcpy(state->x.next, strm->next_in, strm->avail_in); + state->x.have = strm->avail_in; strm->avail_in = 0; } state->how = COPY; @@ -223,19 +168,15 @@ local int gz_head(state) } /* Decompress from input to the provided next_out and avail_out in the state. - If the end of the compressed data is reached, then verify the gzip trailer - check value and length (modulo 2^32). state->have and state->next are set - to point to the just decompressed data, and the crc is updated. If the - trailer is verified, state->how is reset to LOOK to look for the next gzip - stream or raw data, once state->have is depleted. Returns 0 on success, -1 - on failure. Failures may include invalid compressed data or a failed gzip - trailer verification. */ + On return, state->x.have and state->x.next point to the just decompressed + data. If the gzip stream completes, state->how is reset to LOOK to look for + the next gzip stream or raw data, once state->x.have is depleted. Returns 0 + on success, -1 on failure. */ local int gz_decomp(state) gz_statep state; { - int ret; + int ret = Z_OK; unsigned had; - unsigned long crc, len; z_streamp strm = &(state->strm); /* fill output buffer up to end of deflate stream */ @@ -245,15 +186,15 @@ local int gz_decomp(state) if (strm->avail_in == 0 && gz_avail(state) == -1) return -1; if (strm->avail_in == 0) { - gz_error(state, Z_DATA_ERROR, "unexpected end of file"); - return -1; + gz_error(state, Z_BUF_ERROR, "unexpected end of file"); + break; } /* decompress and handle errors */ ret = inflate(strm, Z_NO_FLUSH); if (ret == Z_STREAM_ERROR || ret == Z_NEED_DICT) { gz_error(state, Z_STREAM_ERROR, - "internal error: inflate stream corrupt"); + "internal error: inflate stream corrupt"); return -1; } if (ret == Z_MEM_ERROR) { @@ -262,67 +203,55 @@ local int gz_decomp(state) } if (ret == Z_DATA_ERROR) { /* deflate stream invalid */ gz_error(state, Z_DATA_ERROR, - strm->msg == NULL ? "compressed data error" : strm->msg); + strm->msg == NULL ? "compressed data error" : strm->msg); return -1; } } while (strm->avail_out && ret != Z_STREAM_END); - /* update available output and crc check value */ - state->have = had - strm->avail_out; - state->next = strm->next_out - state->have; - strm->adler = crc32(strm->adler, state->next, state->have); + /* update available output */ + state->x.have = had - strm->avail_out; + state->x.next = strm->next_out - state->x.have; - /* check gzip trailer if at end of deflate stream */ - if (ret == Z_STREAM_END) { - if (gz_next4(state, &crc) == -1 || gz_next4(state, &len) == -1) { - gz_error(state, Z_DATA_ERROR, "unexpected end of file"); - return -1; - } - if (crc != strm->adler) { - gz_error(state, Z_DATA_ERROR, "incorrect data check"); - return -1; - } - if (len != (strm->total_out & 0xffffffffL)) { - gz_error(state, Z_DATA_ERROR, "incorrect length check"); - return -1; - } - state->how = LOOK; /* ready for next stream, once have is 0 (leave - state->direct unchanged to remember how) */ - } + /* if the gzip stream completed successfully, look for another */ + if (ret == Z_STREAM_END) + state->how = LOOK; /* good decompression */ return 0; } -/* Make data and put in the output buffer. Assumes that state->have == 0. +/* Fetch data and put it in the output buffer. Assumes state->x.have is 0. Data is either copied from the input file or decompressed from the input file depending on state->how. If state->how is LOOK, then a gzip header is - looked for (and skipped if found) to determine wither to copy or decompress. - Returns -1 on error, otherwise 0. gz_make() will leave state->have as COPY - or GZIP unless the end of the input file has been reached and all data has - been processed. */ -local int gz_make(state) + looked for to determine whether to copy or decompress. Returns -1 on error, + otherwise 0. gz_fetch() will leave state->how as COPY or GZIP unless the + end of the input file has been reached and all data has been processed. */ +local int gz_fetch(state) gz_statep state; { z_streamp strm = &(state->strm); - if (state->how == LOOK) { /* look for gzip header */ - if (gz_head(state) == -1) - return -1; - if (state->have) /* got some data from gz_head() */ + do { + switch(state->how) { + case LOOK: /* -> LOOK, COPY (only if never GZIP), or GZIP */ + if (gz_look(state) == -1) + return -1; + if (state->how == LOOK) + return 0; + break; + case COPY: /* -> COPY */ + if (gz_load(state, state->out, state->size << 1, &(state->x.have)) + == -1) + return -1; + state->x.next = state->out; return 0; - } - if (state->how == COPY) { /* straight copy */ - if (gz_load(state, state->out, state->size << 1, &(state->have)) == -1) - return -1; - state->next = state->out; - } - else if (state->how == GZIP) { /* decompress */ - strm->avail_out = state->size << 1; - strm->next_out = state->out; - if (gz_decomp(state) == -1) - return -1; - } + case GZIP: /* -> GZIP or LOOK (if end of gzip stream) */ + strm->avail_out = state->size << 1; + strm->next_out = state->out; + if (gz_decomp(state) == -1) + return -1; + } + } while (state->x.have == 0 && (!state->eof || strm->avail_in)); return 0; } @@ -336,12 +265,12 @@ local int gz_skip(state, len) /* skip over len bytes or reach end-of-file, whichever comes first */ while (len) /* skip over whatever is in output buffer */ - if (state->have) { - n = GT_OFF(state->have) || (z_off64_t)state->have > len ? - (unsigned)len : state->have; - state->have -= n; - state->next += n; - state->pos += n; + if (state->x.have) { + n = GT_OFF(state->x.have) || (z_off64_t)state->x.have > len ? + (unsigned)len : state->x.have; + state->x.have -= n; + state->x.next += n; + state->x.pos += n; len -= n; } @@ -352,38 +281,23 @@ local int gz_skip(state, len) /* need more data to skip -- load up output buffer */ else { /* get more output, looking for header if required */ - if (gz_make(state) == -1) + if (gz_fetch(state) == -1) return -1; } return 0; } -/* -- see zlib.h -- */ -int ZEXPORT gzread(file, buf, len) - gzFile file; +/* Read len bytes into buf from file, or less than len up to the end of the + input. Return the number of bytes read. If zero is returned, either the + end of file was reached, or there was an error. state->err must be + consulted in that case to determine which. */ +local z_size_t gz_read(state, buf, len) + gz_statep state; voidp buf; - unsigned len; + z_size_t len; { - unsigned got, n; - gz_statep state; - z_streamp strm; - - /* get internal structure */ - if (file == NULL) - return -1; - state = (gz_statep)file; - strm = &(state->strm); - - /* check that we're reading and that there's no error */ - if (state->mode != GZ_READ || state->err != Z_OK) - return -1; - - /* since an int is returned, make sure len fits in one, otherwise return - with an error (this avoids the flaw in the interface) */ - if ((int)len < 0) { - gz_error(state, Z_BUF_ERROR, "requested length does not fit in int"); - return -1; - } + z_size_t got; + unsigned n; /* if len is zero, avoid unnecessary operations */ if (len == 0) @@ -393,63 +307,147 @@ int ZEXPORT gzread(file, buf, len) if (state->seek) { state->seek = 0; if (gz_skip(state, state->skip) == -1) - return -1; + return 0; } /* get len bytes to buf, or less than len if at the end */ got = 0; do { + /* set n to the maximum amount of len that fits in an unsigned int */ + n = -1; + if (n > len) + n = len; + /* first just try copying data from the output buffer */ - if (state->have) { - n = state->have > len ? len : state->have; - memcpy(buf, state->next, n); - state->next += n; - state->have -= n; + if (state->x.have) { + if (state->x.have < n) + n = state->x.have; + memcpy(buf, state->x.next, n); + state->x.next += n; + state->x.have -= n; } /* output buffer empty -- return if we're at the end of the input */ - else if (state->eof && strm->avail_in == 0) + else if (state->eof && state->strm.avail_in == 0) { + state->past = 1; /* tried to read past end */ break; + } /* need output data -- for small len or new stream load up our output buffer */ - else if (state->how == LOOK || len < (state->size << 1)) { + else if (state->how == LOOK || n < (state->size << 1)) { /* get more output, looking for header if required */ - if (gz_make(state) == -1) - return -1; - continue; /* no progress yet -- go back to memcpy() above */ + if (gz_fetch(state) == -1) + return 0; + continue; /* no progress yet -- go back to copy above */ /* the copy above assures that we will leave with space in the output buffer, allowing at least one gzungetc() to succeed */ } /* large len -- read directly into user buffer */ else if (state->how == COPY) { /* read directly */ - if (gz_load(state, buf, len, &n) == -1) - return -1; + if (gz_load(state, (unsigned char *)buf, n, &n) == -1) + return 0; } /* large len -- decompress directly into user buffer */ else { /* state->how == GZIP */ - strm->avail_out = len; - strm->next_out = buf; + state->strm.avail_out = n; + state->strm.next_out = (unsigned char *)buf; if (gz_decomp(state) == -1) - return -1; - n = state->have; - state->have = 0; + return 0; + n = state->x.have; + state->x.have = 0; } /* update progress */ len -= n; buf = (char *)buf + n; got += n; - state->pos += n; + state->x.pos += n; } while (len); - /* return number of bytes read into user buffer (will fit in int) */ - return (int)got; + /* return number of bytes read into user buffer */ + return got; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzread(file, buf, len) + gzFile file; + voidp buf; + unsigned len; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + + /* check that we're reading and that there's no (serious) error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) + return -1; + + /* since an int is returned, make sure len fits in one, otherwise return + with an error (this avoids a flaw in the interface) */ + if ((int)len < 0) { + gz_error(state, Z_STREAM_ERROR, "request does not fit in an int"); + return -1; + } + + /* read len or fewer bytes to buf */ + len = gz_read(state, buf, len); + + /* check for an error */ + if (len == 0 && state->err != Z_OK && state->err != Z_BUF_ERROR) + return -1; + + /* return the number of bytes read (this is assured to fit in an int) */ + return (int)len; +} + +/* -- see zlib.h -- */ +z_size_t ZEXPORT gzfread(buf, size, nitems, file) + voidp buf; + z_size_t size; + z_size_t nitems; + gzFile file; +{ + z_size_t len; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return 0; + state = (gz_statep)file; + + /* check that we're reading and that there's no (serious) error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) + return 0; + + /* compute bytes to read -- error on overflow */ + len = nitems * size; + if (size && len / size != nitems) { + gz_error(state, Z_STREAM_ERROR, "request does not fit in a size_t"); + return 0; + } + + /* read len or fewer bytes to buf, return the number of full items read */ + return len ? gz_read(state, buf, len) / size : 0; } /* -- see zlib.h -- */ +#ifdef Z_PREFIX_SET +# undef z_gzgetc +#else +# undef gzgetc +# ifdef Z_CR_PREFIX_SET +# define gzgetc Cr_z_gzgetc +# endif +#endif + int ZEXPORT gzgetc(file) gzFile file; { @@ -462,22 +460,29 @@ int ZEXPORT gzgetc(file) return -1; state = (gz_statep)file; - /* check that we're reading and that there's no error */ - if (state->mode != GZ_READ || state->err != Z_OK) + /* check that we're reading and that there's no (serious) error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) return -1; /* try output buffer (no need to check for skip request) */ - if (state->have) { - state->have--; - state->pos++; - return *(state->next)++; + if (state->x.have) { + state->x.have--; + state->x.pos++; + return *(state->x.next)++; } - /* nothing there -- try gzread() */ - ret = gzread(file, buf, 1); + /* nothing there -- try gz_read() */ + ret = gz_read(state, buf, 1); return ret < 1 ? -1 : buf[0]; } +int ZEXPORT gzgetc_(file) +gzFile file; +{ + return gzgetc(file); +} + /* -- see zlib.h -- */ int ZEXPORT gzungetc(c, file) int c; @@ -490,8 +495,9 @@ int ZEXPORT gzungetc(c, file) return -1; state = (gz_statep)file; - /* check that we're reading and that there's no error */ - if (state->mode != GZ_READ || state->err != Z_OK) + /* check that we're reading and that there's no (serious) error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) return -1; /* process a skip request */ @@ -506,32 +512,34 @@ int ZEXPORT gzungetc(c, file) return -1; /* if output buffer empty, put byte at end (allows more pushing) */ - if (state->have == 0) { - state->have = 1; - state->next = state->out + (state->size << 1) - 1; - state->next[0] = c; - state->pos--; + if (state->x.have == 0) { + state->x.have = 1; + state->x.next = state->out + (state->size << 1) - 1; + state->x.next[0] = (unsigned char)c; + state->x.pos--; + state->past = 0; return c; } /* if no room, give up (must have already done a gzungetc()) */ - if (state->have == (state->size << 1)) { - gz_error(state, Z_BUF_ERROR, "out of room to push characters"); + if (state->x.have == (state->size << 1)) { + gz_error(state, Z_DATA_ERROR, "out of room to push characters"); return -1; } /* slide output data if needed and insert byte before existing data */ - if (state->next == state->out) { - unsigned char *src = state->out + state->have; + if (state->x.next == state->out) { + unsigned char *src = state->out + state->x.have; unsigned char *dest = state->out + (state->size << 1); while (src > state->out) *--dest = *--src; - state->next = dest; + state->x.next = dest; } - state->have++; - state->next--; - state->next[0] = c; - state->pos--; + state->x.have++; + state->x.next--; + state->x.next[0] = (unsigned char)c; + state->x.pos--; + state->past = 0; return c; } @@ -551,8 +559,9 @@ char * ZEXPORT gzgets(file, buf, len) return NULL; state = (gz_statep)file; - /* check that we're reading and that there's no error */ - if (state->mode != GZ_READ || state->err != Z_OK) + /* check that we're reading and that there's no (serious) error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) return NULL; /* process a skip request */ @@ -569,32 +578,31 @@ char * ZEXPORT gzgets(file, buf, len) left = (unsigned)len - 1; if (left) do { /* assure that something is in the output buffer */ - if (state->have == 0) { - if (gz_make(state) == -1) - return NULL; /* error */ - if (state->have == 0) { /* end of file */ - if (buf == str) /* got bupkus */ - return NULL; - break; /* got something -- return it */ - } + if (state->x.have == 0 && gz_fetch(state) == -1) + return NULL; /* error */ + if (state->x.have == 0) { /* end of file */ + state->past = 1; /* read past end */ + break; /* return what we have */ } /* look for end-of-line in current output buffer */ - n = state->have > left ? left : state->have; - eol = memchr(state->next, '\n', n); + n = state->x.have > left ? left : state->x.have; + eol = (unsigned char *)memchr(state->x.next, '\n', n); if (eol != NULL) - n = (unsigned)(eol - state->next) + 1; + n = (unsigned)(eol - state->x.next) + 1; /* copy through end-of-line, or remainder if not found */ - memcpy(buf, state->next, n); - state->have -= n; - state->next += n; - state->pos += n; + memcpy(buf, state->x.next, n); + state->x.have -= n; + state->x.next += n; + state->x.pos += n; left -= n; buf += n; } while (left && eol == NULL); - /* found end-of-line or out of space -- terminate string and return it */ + /* return terminated string, or if nothing, end of file */ + if (buf == str) + return NULL; buf[0] = 0; return str; } @@ -610,16 +618,12 @@ int ZEXPORT gzdirect(file) return 0; state = (gz_statep)file; - /* check that we're reading */ - if (state->mode != GZ_READ) - return 0; - /* if the state is not known, but we can find out, then do so (this is mainly for right after a gzopen() or gzdopen()) */ - if (state->how == LOOK && state->have == 0) - (void)gz_head(state); + if (state->mode == GZ_READ && state->how == LOOK && state->x.have == 0) + (void)gz_look(state); - /* return 1 if reading direct, 0 if decompressing a gzip stream */ + /* return 1 if transparent, 0 if processing a gzip stream */ return state->direct; } @@ -627,7 +631,7 @@ int ZEXPORT gzdirect(file) int ZEXPORT gzclose_r(file) gzFile file; { - int ret; + int ret, err; gz_statep state; /* get internal structure */ @@ -645,9 +649,10 @@ int ZEXPORT gzclose_r(file) free(state->out); free(state->in); } + err = state->err == Z_BUF_ERROR ? Z_BUF_ERROR : Z_OK; gz_error(state, Z_OK, NULL); free(state->path); ret = close(state->fd); free(state); - return ret ? Z_ERRNO : Z_OK; + return ret ? Z_ERRNO : err; } diff --git a/third_party/zlib/gzwrite.c b/third_party/zlib/gzwrite.c index e8defc6887..c7b5651d70 100644 --- a/third_party/zlib/gzwrite.c +++ b/third_party/zlib/gzwrite.c @@ -1,5 +1,5 @@ /* gzwrite.c -- zlib functions for writing gzip files - * Copyright (C) 2004, 2005, 2010 Mark Adler + * Copyright (C) 2004-2017 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -9,65 +9,94 @@ local int gz_init OF((gz_statep)); local int gz_comp OF((gz_statep, int)); local int gz_zero OF((gz_statep, z_off64_t)); +local z_size_t gz_write OF((gz_statep, voidpc, z_size_t)); /* Initialize state for writing a gzip file. Mark initialization by setting - state->size to non-zero. Return -1 on failure or 0 on success. */ + state->size to non-zero. Return -1 on a memory allocation failure, or 0 on + success. */ local int gz_init(state) gz_statep state; { int ret; z_streamp strm = &(state->strm); - /* allocate input and output buffers */ - state->in = malloc(state->want); - state->out = malloc(state->want); - if (state->in == NULL || state->out == NULL) { - if (state->out != NULL) - free(state->out); - if (state->in != NULL) - free(state->in); + /* allocate input buffer (double size for gzprintf) */ + state->in = (unsigned char *)malloc(state->want << 1); + if (state->in == NULL) { gz_error(state, Z_MEM_ERROR, "out of memory"); return -1; } - /* allocate deflate memory, set up for gzip compression */ - strm->zalloc = Z_NULL; - strm->zfree = Z_NULL; - strm->opaque = Z_NULL; - ret = deflateInit2(strm, state->level, Z_DEFLATED, - 15 + 16, 8, state->strategy); - if (ret != Z_OK) { - free(state->in); - gz_error(state, Z_MEM_ERROR, "out of memory"); - return -1; + /* only need output buffer and deflate state if compressing */ + if (!state->direct) { + /* allocate output buffer */ + state->out = (unsigned char *)malloc(state->want); + if (state->out == NULL) { + free(state->in); + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + + /* allocate deflate memory, set up for gzip compression */ + strm->zalloc = Z_NULL; + strm->zfree = Z_NULL; + strm->opaque = Z_NULL; + ret = deflateInit2(strm, state->level, Z_DEFLATED, + MAX_WBITS + 16, DEF_MEM_LEVEL, state->strategy); + if (ret != Z_OK) { + free(state->out); + free(state->in); + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + strm->next_in = NULL; } /* mark state as initialized */ state->size = state->want; - /* initialize write buffer */ - strm->avail_out = state->size; - strm->next_out = state->out; - state->next = strm->next_out; + /* initialize write buffer if compressing */ + if (!state->direct) { + strm->avail_out = state->size; + strm->next_out = state->out; + state->x.next = strm->next_out; + } return 0; } /* Compress whatever is at avail_in and next_in and write to the output file. - Return -1 if there is an error writing to the output file, otherwise 0. - flush is assumed to be a valid deflate() flush value. If flush is Z_FINISH, - then the deflate() state is reset to start a new gzip stream. */ + Return -1 if there is an error writing to the output file or if gz_init() + fails to allocate memory, otherwise 0. flush is assumed to be a valid + deflate() flush value. If flush is Z_FINISH, then the deflate() state is + reset to start a new gzip stream. If gz->direct is true, then simply write + to the output file without compressing, and ignore flush. */ local int gz_comp(state, flush) gz_statep state; int flush; { - int ret, got; - unsigned have; + int ret, writ; + unsigned have, put, max = ((unsigned)-1 >> 2) + 1; z_streamp strm = &(state->strm); /* allocate memory if this is the first time through */ if (state->size == 0 && gz_init(state) == -1) return -1; + /* write directly if requested */ + if (state->direct) { + while (strm->avail_in) { + put = strm->avail_in > max ? max : strm->avail_in; + writ = write(state->fd, strm->next_in, put); + if (writ < 0) { + gz_error(state, Z_ERRNO, zstrerror()); + return -1; + } + strm->avail_in -= (unsigned)writ; + strm->next_in += writ; + } + return 0; + } + /* run deflate() on provided input until it produces no more output */ ret = Z_OK; do { @@ -75,17 +104,21 @@ local int gz_comp(state, flush) doing Z_FINISH then don't write until we get to Z_STREAM_END */ if (strm->avail_out == 0 || (flush != Z_NO_FLUSH && (flush != Z_FINISH || ret == Z_STREAM_END))) { - have = (unsigned)(strm->next_out - state->next); - if (have && ((got = write(state->fd, state->next, have)) < 0 || - (unsigned)got != have)) { - gz_error(state, Z_ERRNO, zstrerror()); - return -1; + while (strm->next_out > state->x.next) { + put = strm->next_out - state->x.next > (int)max ? max : + (unsigned)(strm->next_out - state->x.next); + writ = write(state->fd, state->x.next, put); + if (writ < 0) { + gz_error(state, Z_ERRNO, zstrerror()); + return -1; + } + state->x.next += writ; } if (strm->avail_out == 0) { strm->avail_out = state->size; strm->next_out = state->out; + state->x.next = state->out; } - state->next = strm->next_out; } /* compress */ @@ -107,7 +140,8 @@ local int gz_comp(state, flush) return 0; } -/* Compress len zeros to output. Return -1 on error, 0 on success. */ +/* Compress len zeros to output. Return -1 on a write error or memory + allocation failure by gz_comp(), or 0 on success. */ local int gz_zero(state, len) gz_statep state; z_off64_t len; @@ -131,7 +165,7 @@ local int gz_zero(state, len) } strm->avail_in = n; strm->next_in = state->in; - state->pos += n; + state->x.pos += n; if (gz_comp(state, Z_NO_FLUSH) == -1) return -1; len -= n; @@ -139,33 +173,14 @@ local int gz_zero(state, len) return 0; } -/* -- see zlib.h -- */ -int ZEXPORT gzwrite(file, buf, len) - gzFile file; +/* Write len bytes from buf to file. Return the number of bytes written. If + the returned value is less than len, then there was an error. */ +local z_size_t gz_write(state, buf, len) + gz_statep state; voidpc buf; - unsigned len; + z_size_t len; { - unsigned put = len; - unsigned n; - gz_statep state; - z_streamp strm; - - /* get internal structure */ - if (file == NULL) - return 0; - state = (gz_statep)file; - strm = &(state->strm); - - /* check that we're writing and that there's no error */ - if (state->mode != GZ_WRITE || state->err != Z_OK) - return 0; - - /* since an int is returned, make sure len fits in one, otherwise return - with an error (this avoids the flaw in the interface) */ - if ((int)len < 0) { - gz_error(state, Z_BUF_ERROR, "requested length does not fit in int"); - return 0; - } + z_size_t put = len; /* if len is zero, avoid unnecessary operations */ if (len == 0) @@ -186,35 +201,103 @@ int ZEXPORT gzwrite(file, buf, len) if (len < state->size) { /* copy to input buffer, compress when full */ do { - if (strm->avail_in == 0) - strm->next_in = state->in; - n = state->size - strm->avail_in; - if (n > len) - n = len; - memcpy(strm->next_in + strm->avail_in, buf, n); - strm->avail_in += n; - state->pos += n; - buf = (char *)buf + n; - len -= n; + unsigned have, copy; + + if (state->strm.avail_in == 0) + state->strm.next_in = state->in; + have = (unsigned)((state->strm.next_in + state->strm.avail_in) - + state->in); + copy = state->size - have; + if (copy > len) + copy = len; + memcpy(state->in + have, buf, copy); + state->strm.avail_in += copy; + state->x.pos += copy; + buf = (const char *)buf + copy; + len -= copy; if (len && gz_comp(state, Z_NO_FLUSH) == -1) return 0; } while (len); } else { /* consume whatever's left in the input buffer */ - if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1) + if (state->strm.avail_in && gz_comp(state, Z_NO_FLUSH) == -1) return 0; /* directly compress user buffer to file */ - strm->avail_in = len; - strm->next_in = (voidp)buf; - state->pos += len; - if (gz_comp(state, Z_NO_FLUSH) == -1) - return 0; + state->strm.next_in = (z_const Bytef *)buf; + do { + unsigned n = (unsigned)-1; + if (n > len) + n = len; + state->strm.avail_in = n; + state->x.pos += n; + if (gz_comp(state, Z_NO_FLUSH) == -1) + return 0; + len -= n; + } while (len); + } + + /* input was all buffered or compressed */ + return put; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzwrite(file, buf, len) + gzFile file; + voidpc buf; + unsigned len; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return 0; + state = (gz_statep)file; + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return 0; + + /* since an int is returned, make sure len fits in one, otherwise return + with an error (this avoids a flaw in the interface) */ + if ((int)len < 0) { + gz_error(state, Z_DATA_ERROR, "requested length does not fit in int"); + return 0; } - /* input was all buffered or compressed (put will fit in int) */ - return (int)put; + /* write len bytes from buf (the return value will fit in an int) */ + return (int)gz_write(state, buf, len); +} + +/* -- see zlib.h -- */ +z_size_t ZEXPORT gzfwrite(buf, size, nitems, file) + voidpc buf; + z_size_t size; + z_size_t nitems; + gzFile file; +{ + z_size_t len; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return 0; + state = (gz_statep)file; + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return 0; + + /* compute bytes to read -- error on overflow */ + len = nitems * size; + if (size && len / size != nitems) { + gz_error(state, Z_STREAM_ERROR, "request does not fit in a size_t"); + return 0; + } + + /* write len bytes to buf, return the number of full items written */ + return len ? gz_write(state, buf, len) / size : 0; } /* -- see zlib.h -- */ @@ -222,6 +305,7 @@ int ZEXPORT gzputc(file, c) gzFile file; int c; { + unsigned have; unsigned char buf[1]; gz_statep state; z_streamp strm; @@ -245,19 +329,23 @@ int ZEXPORT gzputc(file, c) /* try writing to input buffer for speed (state->size == 0 if buffer not initialized) */ - if (strm->avail_in < state->size) { + if (state->size) { if (strm->avail_in == 0) strm->next_in = state->in; - strm->next_in[strm->avail_in++] = c; - state->pos++; - return c; + have = (unsigned)((strm->next_in + strm->avail_in) - state->in); + if (have < state->size) { + state->in[have] = (unsigned char)c; + strm->avail_in++; + state->x.pos++; + return c & 0xff; + } } /* no room in buffer or not initialized, use gz_write() */ - buf[0] = c; - if (gzwrite(file, buf, 1) != 1) + buf[0] = (unsigned char)c; + if (gz_write(state, buf, 1) != 1) return -1; - return c; + return c & 0xff; } /* -- see zlib.h -- */ @@ -266,87 +354,112 @@ int ZEXPORT gzputs(file, str) const char *str; { int ret; - unsigned len; + z_size_t len; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return -1; /* write string */ - len = (unsigned)strlen(str); - ret = gzwrite(file, str, len); + len = strlen(str); + ret = gz_write(state, str, len); return ret == 0 && len != 0 ? -1 : ret; } -#ifdef STDC +#if defined(STDC) || defined(Z_HAVE_STDARG_H) #include /* -- see zlib.h -- */ -int ZEXPORTVA gzprintf (gzFile file, const char *format, ...) +int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va) { - int size, len; + int len; + unsigned left; + char *next; gz_statep state; z_streamp strm; - va_list va; /* get internal structure */ if (file == NULL) - return -1; + return Z_STREAM_ERROR; state = (gz_statep)file; strm = &(state->strm); /* check that we're writing and that there's no error */ if (state->mode != GZ_WRITE || state->err != Z_OK) - return 0; + return Z_STREAM_ERROR; /* make sure we have some buffer space */ if (state->size == 0 && gz_init(state) == -1) - return 0; + return state->err; /* check for seek request */ if (state->seek) { state->seek = 0; if (gz_zero(state, state->skip) == -1) - return 0; + return state->err; } - /* consume whatever's left in the input buffer */ - if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1) - return 0; - - /* do the printf() into the input buffer, put length in len */ - size = (int)(state->size); - state->in[size - 1] = 0; - va_start(va, format); + /* do the printf() into the input buffer, put length in len -- the input + buffer is double-sized just for this function, so there is guaranteed to + be state->size bytes available after the current contents */ + if (strm->avail_in == 0) + strm->next_in = state->in; + next = (char *)(state->in + (strm->next_in - state->in) + strm->avail_in); + next[state->size - 1] = 0; #ifdef NO_vsnprintf # ifdef HAS_vsprintf_void - (void)vsprintf(state->in, format, va); - va_end(va); - for (len = 0; len < size; len++) - if (state->in[len] == 0) break; + (void)vsprintf(next, format, va); + for (len = 0; len < state->size; len++) + if (next[len] == 0) break; # else - len = vsprintf(state->in, format, va); - va_end(va); + len = vsprintf(next, format, va); # endif #else # ifdef HAS_vsnprintf_void - (void)vsnprintf(state->in, size, format, va); - va_end(va); - len = strlen(state->in); + (void)vsnprintf(next, state->size, format, va); + len = strlen(next); # else - len = vsnprintf((char *)(state->in), size, format, va); - va_end(va); + len = vsnprintf(next, state->size, format, va); # endif #endif /* check that printf() results fit in buffer */ - if (len <= 0 || len >= (int)size || state->in[size - 1] != 0) + if (len == 0 || (unsigned)len >= state->size || next[state->size - 1] != 0) return 0; - /* update buffer and position, defer compression until needed */ - strm->avail_in = (unsigned)len; - strm->next_in = state->in; - state->pos += len; + /* update buffer and position, compress first half if past that */ + strm->avail_in += (unsigned)len; + state->x.pos += len; + if (strm->avail_in >= state->size) { + left = strm->avail_in - state->size; + strm->avail_in = state->size; + if (gz_comp(state, Z_NO_FLUSH) == -1) + return state->err; + memcpy(state->in, state->in + state->size, left); + strm->next_in = state->in; + strm->avail_in = left; + } return len; } -#else /* !STDC */ +int ZEXPORTVA gzprintf(gzFile file, const char *format, ...) +{ + va_list va; + int ret; + + va_start(va, format); + ret = gzvprintf(file, format, va); + va_end(va); + return ret; +} + +#else /* !STDC && !Z_HAVE_STDARG_H */ /* -- see zlib.h -- */ int ZEXPORTVA gzprintf (file, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, @@ -356,68 +469,82 @@ int ZEXPORTVA gzprintf (file, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, int a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20; { - int size, len; + unsigned len, left; + char *next; gz_statep state; z_streamp strm; /* get internal structure */ if (file == NULL) - return -1; + return Z_STREAM_ERROR; state = (gz_statep)file; strm = &(state->strm); + /* check that can really pass pointer in ints */ + if (sizeof(int) != sizeof(void *)) + return Z_STREAM_ERROR; + /* check that we're writing and that there's no error */ if (state->mode != GZ_WRITE || state->err != Z_OK) - return 0; + return Z_STREAM_ERROR; /* make sure we have some buffer space */ if (state->size == 0 && gz_init(state) == -1) - return 0; + return state->error; /* check for seek request */ if (state->seek) { state->seek = 0; if (gz_zero(state, state->skip) == -1) - return 0; + return state->error; } - /* consume whatever's left in the input buffer */ - if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1) - return 0; - - /* do the printf() into the input buffer, put length in len */ - size = (int)(state->size); - state->in[size - 1] = 0; + /* do the printf() into the input buffer, put length in len -- the input + buffer is double-sized just for this function, so there is guaranteed to + be state->size bytes available after the current contents */ + if (strm->avail_in == 0) + strm->next_in = state->in; + next = (char *)(strm->next_in + strm->avail_in); + next[state->size - 1] = 0; #ifdef NO_snprintf # ifdef HAS_sprintf_void - sprintf(state->in, format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); + sprintf(next, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, + a13, a14, a15, a16, a17, a18, a19, a20); for (len = 0; len < size; len++) - if (state->in[len] == 0) break; + if (next[len] == 0) + break; # else - len = sprintf(state->in, format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); + len = sprintf(next, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, + a12, a13, a14, a15, a16, a17, a18, a19, a20); # endif #else # ifdef HAS_snprintf_void - snprintf(state->in, size, format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); - len = strlen(state->in); + snprintf(next, state->size, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, + a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); + len = strlen(next); # else - len = snprintf(state->in, size, format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); + len = snprintf(next, state->size, format, a1, a2, a3, a4, a5, a6, a7, a8, + a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); # endif #endif /* check that printf() results fit in buffer */ - if (len <= 0 || len >= (int)size || state->in[size - 1] != 0) + if (len == 0 || len >= state->size || next[state->size - 1] != 0) return 0; - /* update buffer and position, defer compression until needed */ - strm->avail_in = (unsigned)len; - strm->next_in = state->in; - state->pos += len; - return len; + /* update buffer and position, compress first half if past that */ + strm->avail_in += len; + state->x.pos += len; + if (strm->avail_in >= state->size) { + left = strm->avail_in - state->size; + strm->avail_in = state->size; + if (gz_comp(state, Z_NO_FLUSH) == -1) + return state->err; + memcpy(state->in, state->in + state->size, left); + strm->next_in = state->in; + strm->avail_in = left; + } + return (int)len; } #endif @@ -431,7 +558,7 @@ int ZEXPORT gzflush(file, flush) /* get internal structure */ if (file == NULL) - return -1; + return Z_STREAM_ERROR; state = (gz_statep)file; /* check that we're writing and that there's no error */ @@ -446,11 +573,11 @@ int ZEXPORT gzflush(file, flush) if (state->seek) { state->seek = 0; if (gz_zero(state, state->skip) == -1) - return -1; + return state->err; } /* compress remaining data with requested flush */ - gz_comp(state, flush); + (void)gz_comp(state, flush); return state->err; } @@ -481,13 +608,13 @@ int ZEXPORT gzsetparams(file, level, strategy) if (state->seek) { state->seek = 0; if (gz_zero(state, state->skip) == -1) - return -1; + return state->err; } /* change compression parameters for subsequent input */ if (state->size) { /* flush previous input with previous parameters before changing */ - if (strm->avail_in && gz_comp(state, Z_PARTIAL_FLUSH) == -1) + if (strm->avail_in && gz_comp(state, Z_BLOCK) == -1) return state->err; deflateParams(strm, level, strategy); } @@ -500,7 +627,7 @@ int ZEXPORT gzsetparams(file, level, strategy) int ZEXPORT gzclose_w(file) gzFile file; { - int ret = 0; + int ret = Z_OK; gz_statep state; /* get internal structure */ @@ -515,17 +642,24 @@ int ZEXPORT gzclose_w(file) /* check for seek request */ if (state->seek) { state->seek = 0; - ret += gz_zero(state, state->skip); + if (gz_zero(state, state->skip) == -1) + ret = state->err; } /* flush, free memory, and close file */ - ret += gz_comp(state, Z_FINISH); - (void)deflateEnd(&(state->strm)); - free(state->out); - free(state->in); + if (gz_comp(state, Z_FINISH) == -1) + ret = state->err; + if (state->size) { + if (!state->direct) { + (void)deflateEnd(&(state->strm)); + free(state->out); + } + free(state->in); + } gz_error(state, Z_OK, NULL); free(state->path); - ret += close(state->fd); + if (close(state->fd) == -1) + ret = Z_ERRNO; free(state); - return ret ? Z_ERRNO : Z_OK; + return ret; } diff --git a/third_party/zlib/infback.c b/third_party/zlib/infback.c index af3a8c965d..aab14b4848 100644 --- a/third_party/zlib/infback.c +++ b/third_party/zlib/infback.c @@ -1,5 +1,5 @@ /* infback.c -- inflate using a call-back interface - * Copyright (C) 1995-2009 Mark Adler + * Copyright (C) 1995-2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -42,17 +42,26 @@ int stream_size; return Z_STREAM_ERROR; strm->msg = Z_NULL; /* in case we return an error */ if (strm->zalloc == (alloc_func)0) { +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else strm->zalloc = zcalloc; strm->opaque = (voidpf)0; +#endif } - if (strm->zfree == (free_func)0) strm->zfree = zcfree; + if (strm->zfree == (free_func)0) +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zfree = zcfree; +#endif state = (struct inflate_state FAR *)ZALLOC(strm, 1, sizeof(struct inflate_state)); if (state == Z_NULL) return Z_MEM_ERROR; Tracev((stderr, "inflate: allocated\n")); strm->state = (struct internal_state FAR *)state; state->dmax = 32768U; - state->wbits = windowBits; + state->wbits = (uInt)windowBits; state->wsize = 1U << windowBits; state->window = window; state->wnext = 0; @@ -246,7 +255,7 @@ out_func out; void FAR *out_desc; { struct inflate_state FAR *state; - unsigned char FAR *next; /* next input */ + z_const unsigned char FAR *next; /* next input */ unsigned char FAR *put; /* next output */ unsigned have, left; /* available input and output */ unsigned long hold; /* bit buffer */ @@ -394,7 +403,6 @@ void FAR *out_desc; PULLBYTE(); } if (here.val < 16) { - NEEDBITS(here.bits); DROPBITS(here.bits); state->lens[state->have++] = here.val; } @@ -472,7 +480,8 @@ void FAR *out_desc; case LEN: /* use inflate_fast() if we have enough input and output */ - if (have >= 6 && left >= 258) { + if (have >= INFLATE_FAST_MIN_INPUT && + left >= INFLATE_FAST_MIN_OUTPUT) { RESTORE(); if (state->whave < state->wsize) state->whave = state->wsize - left; diff --git a/third_party/zlib/inffast.c b/third_party/zlib/inffast.c index 2f1d60b43b..2797e8a03c 100644 --- a/third_party/zlib/inffast.c +++ b/third_party/zlib/inffast.c @@ -1,5 +1,5 @@ /* inffast.c -- fast decoding - * Copyright (C) 1995-2008, 2010 Mark Adler + * Copyright (C) 1995-2017 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -8,26 +8,9 @@ #include "inflate.h" #include "inffast.h" -#ifndef ASMINF - -/* Allow machine dependent optimization for post-increment or pre-increment. - Based on testing to date, - Pre-increment preferred for: - - PowerPC G3 (Adler) - - MIPS R5000 (Randers-Pehrson) - Post-increment preferred for: - - none - No measurable difference: - - Pentium III (Anderson) - - M68060 (Nikl) - */ -#ifdef POSTINC -# define OFF 0 -# define PUP(a) *(a)++ +#ifdef ASMINF +# pragma message("Assembler code may have bugs -- use at your own risk") #else -# define OFF 1 -# define PUP(a) *++(a) -#endif /* Decode literal, length, and distance codes and write out the resulting @@ -35,13 +18,13 @@ available, an end-of-block is encountered, or a data error is encountered. When large enough input and output buffers are supplied to inflate(), for example, a 16K input buffer and a 64K output buffer, more than 95% of the - inflate execution time is spent in this routine. + inflate() execution time is spent in this routine. Entry assumptions: state->mode == LEN - strm->avail_in >= 6 - strm->avail_out >= 258 + strm->avail_in >= INFLATE_FAST_MIN_INPUT (6 bytes) + strm->avail_out >= INFLATE_FAST_MIN_OUTPUT (258 bytes) start >= strm->avail_out state->bits < 8 @@ -53,24 +36,28 @@ Notes: + INFLATE_FAST_MIN_INPUT: 6 bytes + - The maximum input bits used by a length/distance pair is 15 bits for the length code, 5 bits for the length extra, 15 bits for the distance code, and 13 bits for the distance extra. This totals 48 bits, or six bytes. Therefore if strm->avail_in >= 6, then there is enough input to avoid checking for available input while decoding. + INFLATE_FAST_MIN_OUTPUT: 258 bytes + - The maximum bytes that a single length/distance pair can output is 258 bytes, which is the maximum length that can be coded. inflate_fast() requires strm->avail_out >= 258 for each loop to avoid checking for - output space. + available output space while decoding. */ void ZLIB_INTERNAL inflate_fast(strm, start) z_streamp strm; unsigned start; /* inflate()'s starting value for strm->avail_out */ { struct inflate_state FAR *state; - unsigned char FAR *in; /* local strm->next_in */ - unsigned char FAR *last; /* while in < last, enough input available */ + z_const unsigned char FAR *in; /* local strm->next_in */ + z_const unsigned char FAR *last; /* have enough input while in < last */ unsigned char FAR *out; /* local strm->next_out */ unsigned char FAR *beg; /* inflate()'s initial strm->next_out */ unsigned char FAR *end; /* while out < end, enough space available */ @@ -96,11 +83,11 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ /* copy state to local variables */ state = (struct inflate_state FAR *)strm->state; - in = strm->next_in - OFF; - last = in + (strm->avail_in - 5); - out = strm->next_out - OFF; + in = strm->next_in; + last = in + (strm->avail_in - (INFLATE_FAST_MIN_INPUT - 1)); + out = strm->next_out; beg = out - (start - strm->avail_out); - end = out + (strm->avail_out - 257); + end = out + (strm->avail_out - (INFLATE_FAST_MIN_OUTPUT - 1)); #ifdef INFLATE_STRICT dmax = state->dmax; #endif @@ -119,9 +106,9 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ input data or output space */ do { if (bits < 15) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } here = lcode[hold & lmask]; @@ -134,14 +121,14 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? "inflate: literal '%c'\n" : "inflate: literal 0x%02x\n", here.val)); - PUP(out) = (unsigned char)(here.val); + *out++ = (unsigned char)(here.val); } else if (op & 16) { /* length base */ len = (unsigned)(here.val); op &= 15; /* number of extra bits */ if (op) { if (bits < op) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } len += (unsigned)hold & ((1U << op) - 1); @@ -150,9 +137,9 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ } Tracevv((stderr, "inflate: length %u\n", len)); if (bits < 15) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } here = dcode[hold & dmask]; @@ -165,10 +152,10 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ dist = (unsigned)(here.val); op &= 15; /* number of extra bits */ if (bits < op) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; if (bits < op) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } } @@ -196,30 +183,30 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ #ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR if (len <= op - whave) { do { - PUP(out) = 0; + *out++ = 0; } while (--len); continue; } len -= op - whave; do { - PUP(out) = 0; + *out++ = 0; } while (--op > whave); if (op == 0) { from = out - dist; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--len); continue; } #endif } - from = window - OFF; + from = window; if (wnext == 0) { /* very common case */ from += wsize - op; if (op < len) { /* some from window */ len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); from = out - dist; /* rest from output */ } @@ -230,14 +217,14 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ if (op < len) { /* some from end of window */ len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); - from = window - OFF; + from = window; if (wnext < len) { /* some from start of window */ op = wnext; len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); from = out - dist; /* rest from output */ } @@ -248,35 +235,35 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ if (op < len) { /* some from window */ len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); from = out - dist; /* rest from output */ } } while (len > 2) { - PUP(out) = PUP(from); - PUP(out) = PUP(from); - PUP(out) = PUP(from); + *out++ = *from++; + *out++ = *from++; + *out++ = *from++; len -= 3; } if (len) { - PUP(out) = PUP(from); + *out++ = *from++; if (len > 1) - PUP(out) = PUP(from); + *out++ = *from++; } } else { from = out - dist; /* copy direct from output */ do { /* minimum length is three */ - PUP(out) = PUP(from); - PUP(out) = PUP(from); - PUP(out) = PUP(from); + *out++ = *from++; + *out++ = *from++; + *out++ = *from++; len -= 3; } while (len > 2); if (len) { - PUP(out) = PUP(from); + *out++ = *from++; if (len > 1) - PUP(out) = PUP(from); + *out++ = *from++; } } } @@ -313,11 +300,14 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ hold &= (1U << bits) - 1; /* update state and return */ - strm->next_in = in + OFF; - strm->next_out = out + OFF; - strm->avail_in = (unsigned)(in < last ? 5 + (last - in) : 5 - (in - last)); + strm->next_in = in; + strm->next_out = out; + strm->avail_in = (unsigned)(in < last ? + (INFLATE_FAST_MIN_INPUT - 1) + (last - in) : + (INFLATE_FAST_MIN_INPUT - 1) - (in - last)); strm->avail_out = (unsigned)(out < end ? - 257 + (end - out) : 257 - (out - end)); + (INFLATE_FAST_MIN_OUTPUT - 1) + (end - out) : + (INFLATE_FAST_MIN_OUTPUT - 1) - (out - end)); state->hold = hold; state->bits = bits; return; diff --git a/third_party/zlib/inffast.h b/third_party/zlib/inffast.h index e5c1aa4ca8..c7c1c09808 100644 --- a/third_party/zlib/inffast.h +++ b/third_party/zlib/inffast.h @@ -8,4 +8,19 @@ subject to change. Applications should only use zlib.h. */ +/* INFLATE_FAST_MIN_INPUT: the minimum number of input bytes needed so that + we can safely call inflate_fast() with only one up-front bounds check. One + length/distance code pair (15 bits for the length code, 5 bits for length + extra, 15 bits for the distance code, 13 bits for distance extra) requires + reading up to 48 input bits (6 bytes). +*/ +#define INFLATE_FAST_MIN_INPUT 6 + +/* INFLATE_FAST_MIN_OUTPUT: the minimum number of output bytes needed so that + we can safely call inflate_fast() with only one up-front bounds check. One + length/distance code pair can output up to 258 bytes, which is the maximum + length that can be coded. + */ +#define INFLATE_FAST_MIN_OUTPUT 258 + void ZLIB_INTERNAL inflate_fast OF((z_streamp strm, unsigned start)); diff --git a/third_party/zlib/inffixed.h b/third_party/zlib/inffixed.h index 75ed4b5978..d628327769 100644 --- a/third_party/zlib/inffixed.h +++ b/third_party/zlib/inffixed.h @@ -2,9 +2,9 @@ * Generated automatically by makefixed(). */ - /* WARNING: this file should *not* be used by applications. It - is part of the implementation of the compression library and - is subject to change. Applications should only use zlib.h. + /* WARNING: this file should *not* be used by applications. + It is part of the implementation of this library and is + subject to change. Applications should only use zlib.h. */ static const code lenfix[512] = { diff --git a/third_party/zlib/inflate.c b/third_party/zlib/inflate.c index a8431abeac..68902e81bd 100644 --- a/third_party/zlib/inflate.c +++ b/third_party/zlib/inflate.c @@ -1,5 +1,5 @@ /* inflate.c -- zlib decompression - * Copyright (C) 1995-2010 Mark Adler + * Copyright (C) 1995-2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -92,32 +92,46 @@ #endif /* function prototypes */ +local int inflateStateCheck OF((z_streamp strm)); local void fixedtables OF((struct inflate_state FAR *state)); -local int updatewindow OF((z_streamp strm, unsigned out)); +local int updatewindow OF((z_streamp strm, const unsigned char FAR *end, + unsigned copy)); #ifdef BUILDFIXED void makefixed OF((void)); #endif -local unsigned syncsearch OF((unsigned FAR *have, unsigned char FAR *buf, +local unsigned syncsearch OF((unsigned FAR *have, const unsigned char FAR *buf, unsigned len)); -int ZEXPORT inflateReset(strm) +local int inflateStateCheck(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (strm == Z_NULL || + strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) + return 1; + state = (struct inflate_state FAR *)strm->state; + if (state == Z_NULL || state->strm != strm || + state->mode < HEAD || state->mode > SYNC) + return 1; + return 0; +} + +int ZEXPORT inflateResetKeep(strm) z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; strm->total_in = strm->total_out = state->total = 0; strm->msg = Z_NULL; - strm->adler = 1; /* to support ill-conceived Java test suite */ + if (state->wrap) /* to support ill-conceived Java test suite */ + strm->adler = state->wrap & 1; state->mode = HEAD; state->last = 0; state->havedict = 0; state->dmax = 32768U; state->head = Z_NULL; - state->wsize = 0; - state->whave = 0; - state->wnext = 0; state->hold = 0; state->bits = 0; state->lencode = state->distcode = state->next = state->codes; @@ -127,6 +141,19 @@ z_streamp strm; return Z_OK; } +int ZEXPORT inflateReset(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + state->wsize = 0; + state->whave = 0; + state->wnext = 0; + return inflateResetKeep(strm); +} + int ZEXPORT inflateReset2(strm, windowBits) z_streamp strm; int windowBits; @@ -135,7 +162,7 @@ int windowBits; struct inflate_state FAR *state; /* get the state */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; /* extract wrap request from windowBits parameter */ @@ -144,7 +171,7 @@ int windowBits; windowBits = -windowBits; } else { - wrap = (windowBits >> 4) + 1; + wrap = (windowBits >> 4) + 5; #ifdef GUNZIP if (windowBits < 48) windowBits &= 15; @@ -180,16 +207,28 @@ int stream_size; if (strm == Z_NULL) return Z_STREAM_ERROR; strm->msg = Z_NULL; /* in case we return an error */ if (strm->zalloc == (alloc_func)0) { +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else strm->zalloc = zcalloc; strm->opaque = (voidpf)0; +#endif } - if (strm->zfree == (free_func)0) strm->zfree = zcfree; + if (strm->zfree == (free_func)0) +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zfree = zcfree; +#endif state = (struct inflate_state FAR *) ZALLOC(strm, 1, sizeof(struct inflate_state)); if (state == Z_NULL) return Z_MEM_ERROR; Tracev((stderr, "inflate: allocated\n")); strm->state = (struct internal_state FAR *)state; + state->strm = strm; state->window = Z_NULL; + state->mode = HEAD; /* to pass state test in inflateReset2() */ + state->check = 1L; /* 1L is the result of adler32() zero length data */ ret = inflateReset2(strm, windowBits); if (ret != Z_OK) { ZFREE(strm, state); @@ -213,17 +252,17 @@ int value; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (bits < 0) { state->hold = 0; state->bits = 0; return Z_OK; } - if (bits > 16 || state->bits + bits > 32) return Z_STREAM_ERROR; + if (bits > 16 || state->bits + (uInt)bits > 32) return Z_STREAM_ERROR; value &= (1L << bits) - 1; - state->hold += value << state->bits; - state->bits += bits; + state->hold += (unsigned)value << state->bits; + state->bits += (uInt)bits; return Z_OK; } @@ -321,8 +360,8 @@ void makefixed() low = 0; for (;;) { if ((low % 7) == 0) printf("\n "); - printf("{%u,%u,%d}", state.lencode[low].op, state.lencode[low].bits, - state.lencode[low].val); + printf("{%u,%u,%d}", (low & 127) == 99 ? 64 : state.lencode[low].op, + state.lencode[low].bits, state.lencode[low].val); if (++low == size) break; putchar(','); } @@ -355,12 +394,13 @@ void makefixed() output will fall in the output data, making match copies simpler and faster. The advantage may be dependent on the size of the processor's data caches. */ -local int updatewindow(strm, out) +local int updatewindow(strm, end, copy) z_streamp strm; -unsigned out; +const Bytef *end; +unsigned copy; { struct inflate_state FAR *state; - unsigned copy, dist; + unsigned dist; state = (struct inflate_state FAR *)strm->state; @@ -380,19 +420,18 @@ unsigned out; } /* copy state->wsize or less output bytes into the circular window */ - copy = out - strm->avail_out; if (copy >= state->wsize) { - zmemcpy(state->window, strm->next_out - state->wsize, state->wsize); + zmemcpy(state->window, end - state->wsize, state->wsize); state->wnext = 0; state->whave = state->wsize; } else { dist = state->wsize - state->wnext; if (dist > copy) dist = copy; - zmemcpy(state->window + state->wnext, strm->next_out - copy, dist); + zmemcpy(state->window + state->wnext, end - copy, dist); copy -= dist; if (copy) { - zmemcpy(state->window, strm->next_out - copy, copy); + zmemcpy(state->window, end - copy, copy); state->wnext = copy; state->whave = state->wsize; } @@ -499,11 +538,6 @@ unsigned out; bits -= bits & 7; \ } while (0) -/* Reverse the bytes in a 32-bit value */ -#define REVERSE(q) \ - ((((q) >> 24) & 0xff) + (((q) >> 8) & 0xff00) + \ - (((q) & 0xff00) << 8) + (((q) & 0xff) << 24)) - /* inflate() uses a state machine to process as much input data and generate as much output data as possible before returning. The state machine is @@ -591,7 +625,7 @@ z_streamp strm; int flush; { struct inflate_state FAR *state; - unsigned char FAR *next; /* next input */ + z_const unsigned char FAR *next; /* next input */ unsigned char FAR *put; /* next output */ unsigned have, left; /* available input and output */ unsigned long hold; /* bit buffer */ @@ -609,7 +643,7 @@ int flush; static const unsigned short order[19] = /* permutation of code lengths */ {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; - if (strm == Z_NULL || strm->state == Z_NULL || strm->next_out == Z_NULL || + if (inflateStateCheck(strm) || strm->next_out == Z_NULL || (strm->next_in == Z_NULL && strm->avail_in != 0)) return Z_STREAM_ERROR; @@ -629,6 +663,8 @@ int flush; NEEDBITS(16); #ifdef GUNZIP if ((state->wrap & 2) && hold == 0x8b1f) { /* gzip header */ + if (state->wbits == 0) + state->wbits = 15; state->check = crc32(0L, Z_NULL, 0); CRC2(state->check, hold); INITBITS(); @@ -656,7 +692,7 @@ int flush; len = BITS(4) + 8; if (state->wbits == 0) state->wbits = len; - else if (len > state->wbits) { + if (len > 15 || len > state->wbits) { strm->msg = (char *)"invalid window size"; state->mode = BAD; break; @@ -683,14 +719,16 @@ int flush; } if (state->head != Z_NULL) state->head->text = (int)((hold >> 8) & 1); - if (state->flags & 0x0200) CRC2(state->check, hold); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); INITBITS(); state->mode = TIME; case TIME: NEEDBITS(32); if (state->head != Z_NULL) state->head->time = hold; - if (state->flags & 0x0200) CRC4(state->check, hold); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC4(state->check, hold); INITBITS(); state->mode = OS; case OS: @@ -699,7 +737,8 @@ int flush; state->head->xflags = (int)(hold & 0xff); state->head->os = (int)(hold >> 8); } - if (state->flags & 0x0200) CRC2(state->check, hold); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); INITBITS(); state->mode = EXLEN; case EXLEN: @@ -708,7 +747,8 @@ int flush; state->length = (unsigned)(hold); if (state->head != Z_NULL) state->head->extra_len = (unsigned)hold; - if (state->flags & 0x0200) CRC2(state->check, hold); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); INITBITS(); } else if (state->head != Z_NULL) @@ -726,7 +766,7 @@ int flush; len + copy > state->head->extra_max ? state->head->extra_max - len : copy); } - if (state->flags & 0x0200) + if ((state->flags & 0x0200) && (state->wrap & 4)) state->check = crc32(state->check, next, copy); have -= copy; next += copy; @@ -745,9 +785,9 @@ int flush; if (state->head != Z_NULL && state->head->name != Z_NULL && state->length < state->head->name_max) - state->head->name[state->length++] = len; + state->head->name[state->length++] = (Bytef)len; } while (len && copy < have); - if (state->flags & 0x0200) + if ((state->flags & 0x0200) && (state->wrap & 4)) state->check = crc32(state->check, next, copy); have -= copy; next += copy; @@ -766,9 +806,9 @@ int flush; if (state->head != Z_NULL && state->head->comment != Z_NULL && state->length < state->head->comm_max) - state->head->comment[state->length++] = len; + state->head->comment[state->length++] = (Bytef)len; } while (len && copy < have); - if (state->flags & 0x0200) + if ((state->flags & 0x0200) && (state->wrap & 4)) state->check = crc32(state->check, next, copy); have -= copy; next += copy; @@ -780,7 +820,7 @@ int flush; case HCRC: if (state->flags & 0x0200) { NEEDBITS(16); - if (hold != (state->check & 0xffff)) { + if ((state->wrap & 4) && hold != (state->check & 0xffff)) { strm->msg = (char *)"header crc mismatch"; state->mode = BAD; break; @@ -797,7 +837,7 @@ int flush; #endif case DICTID: NEEDBITS(32); - strm->adler = state->check = REVERSE(hold); + strm->adler = state->check = ZSWAP32(hold); INITBITS(); state->mode = DICT; case DICT: @@ -905,7 +945,7 @@ int flush; while (state->have < 19) state->lens[order[state->have++]] = 0; state->next = state->codes; - state->lencode = (code const FAR *)(state->next); + state->lencode = (const code FAR *)(state->next); state->lenbits = 7; ret = inflate_table(CODES, state->lens, 19, &(state->next), &(state->lenbits), state->work); @@ -925,7 +965,6 @@ int flush; PULLBYTE(); } if (here.val < 16) { - NEEDBITS(here.bits); DROPBITS(here.bits); state->lens[state->have++] = here.val; } @@ -980,7 +1019,7 @@ int flush; values here (9 and 6) without reading the comments in inftrees.h concerning the ENOUGH constants, which depend on those values */ state->next = state->codes; - state->lencode = (code const FAR *)(state->next); + state->lencode = (const code FAR *)(state->next); state->lenbits = 9; ret = inflate_table(LENS, state->lens, state->nlen, &(state->next), &(state->lenbits), state->work); @@ -989,7 +1028,7 @@ int flush; state->mode = BAD; break; } - state->distcode = (code const FAR *)(state->next); + state->distcode = (const code FAR *)(state->next); state->distbits = 6; ret = inflate_table(DISTS, state->lens + state->nlen, state->ndist, &(state->next), &(state->distbits), state->work); @@ -1004,7 +1043,8 @@ int flush; case LEN_: state->mode = LEN; case LEN: - if (have >= 6 && left >= 258) { + if (have >= INFLATE_FAST_MIN_INPUT && + left >= INFLATE_FAST_MIN_OUTPUT) { RESTORE(); inflate_fast(strm, out); LOAD(); @@ -1162,15 +1202,15 @@ int flush; out -= left; strm->total_out += out; state->total += out; - if (out) + if ((state->wrap & 4) && out) strm->adler = state->check = UPDATE(state->check, put - out, out); out = left; - if (( + if ((state->wrap & 4) && ( #ifdef GUNZIP state->flags ? hold : #endif - REVERSE(hold)) != state->check) { + ZSWAP32(hold)) != state->check) { strm->msg = (char *)"incorrect data check"; state->mode = BAD; break; @@ -1214,8 +1254,9 @@ int flush; */ inf_leave: RESTORE(); - if (state->wsize || (state->mode < CHECK && out != strm->avail_out)) - if (updatewindow(strm, out)) { + if (state->wsize || (out != strm->avail_out && state->mode < BAD && + (state->mode < CHECK || flush != Z_FINISH))) + if (updatewindow(strm, strm->next_out, out - strm->avail_out)) { state->mode = MEM; return Z_MEM_ERROR; } @@ -1224,10 +1265,10 @@ int flush; strm->total_in += in; strm->total_out += out; state->total += out; - if (state->wrap && out) + if ((state->wrap & 4) && out) strm->adler = state->check = UPDATE(state->check, strm->next_out - out, out); - strm->data_type = state->bits + (state->last ? 64 : 0) + + strm->data_type = (int)state->bits + (state->last ? 64 : 0) + (state->mode == TYPE ? 128 : 0) + (state->mode == LEN_ || state->mode == COPY_ ? 256 : 0); if (((in == 0 && out == 0) || flush == Z_FINISH) && ret == Z_OK) @@ -1239,7 +1280,7 @@ int ZEXPORT inflateEnd(strm) z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL || strm->zfree == (free_func)0) + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (state->window != Z_NULL) ZFREE(strm, state->window); @@ -1249,43 +1290,59 @@ z_streamp strm; return Z_OK; } +int ZEXPORT inflateGetDictionary(strm, dictionary, dictLength) +z_streamp strm; +Bytef *dictionary; +uInt *dictLength; +{ + struct inflate_state FAR *state; + + /* check state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + + /* copy dictionary */ + if (state->whave && dictionary != Z_NULL) { + zmemcpy(dictionary, state->window + state->wnext, + state->whave - state->wnext); + zmemcpy(dictionary + state->whave - state->wnext, + state->window, state->wnext); + } + if (dictLength != Z_NULL) + *dictLength = state->whave; + return Z_OK; +} + int ZEXPORT inflateSetDictionary(strm, dictionary, dictLength) z_streamp strm; const Bytef *dictionary; uInt dictLength; { struct inflate_state FAR *state; - unsigned long id; + unsigned long dictid; + int ret; /* check state */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (state->wrap != 0 && state->mode != DICT) return Z_STREAM_ERROR; - /* check for correct dictionary id */ + /* check for correct dictionary identifier */ if (state->mode == DICT) { - id = adler32(0L, Z_NULL, 0); - id = adler32(id, dictionary, dictLength); - if (id != state->check) + dictid = adler32(0L, Z_NULL, 0); + dictid = adler32(dictid, dictionary, dictLength); + if (dictid != state->check) return Z_DATA_ERROR; } - /* copy dictionary to window */ - if (updatewindow(strm, strm->avail_out)) { + /* copy dictionary to window using updatewindow(), which will amend the + existing dictionary if appropriate */ + ret = updatewindow(strm, dictionary + dictLength, dictLength); + if (ret) { state->mode = MEM; return Z_MEM_ERROR; } - if (dictLength > state->wsize) { - zmemcpy(state->window, dictionary + dictLength - state->wsize, - state->wsize); - state->whave = state->wsize; - } - else { - zmemcpy(state->window + state->wsize - dictLength, dictionary, - dictLength); - state->whave = dictLength; - } state->havedict = 1; Tracev((stderr, "inflate: dictionary set\n")); return Z_OK; @@ -1298,7 +1355,7 @@ gz_headerp head; struct inflate_state FAR *state; /* check state */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if ((state->wrap & 2) == 0) return Z_STREAM_ERROR; @@ -1321,7 +1378,7 @@ gz_headerp head; */ local unsigned syncsearch(have, buf, len) unsigned FAR *have; -unsigned char FAR *buf; +const unsigned char FAR *buf; unsigned len; { unsigned got; @@ -1351,7 +1408,7 @@ z_streamp strm; struct inflate_state FAR *state; /* check parameters */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (strm->avail_in == 0 && state->bits < 8) return Z_BUF_ERROR; @@ -1398,7 +1455,7 @@ z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; return state->mode == STORED && state->bits == 0; } @@ -1413,8 +1470,7 @@ z_streamp source; unsigned wsize; /* check input */ - if (dest == Z_NULL || source == Z_NULL || source->state == Z_NULL || - source->zalloc == (alloc_func)0 || source->zfree == (free_func)0) + if (inflateStateCheck(source) || dest == Z_NULL) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)source->state; @@ -1433,8 +1489,9 @@ z_streamp source; } /* copy state */ - zmemcpy(dest, source, sizeof(z_stream)); - zmemcpy(copy, state, sizeof(struct inflate_state)); + zmemcpy((voidpf)dest, (voidpf)source, sizeof(z_stream)); + zmemcpy((voidpf)copy, (voidpf)state, sizeof(struct inflate_state)); + copy->strm = dest; if (state->lencode >= state->codes && state->lencode <= state->codes + ENOUGH - 1) { copy->lencode = copy->codes + (state->lencode - state->codes); @@ -1456,25 +1513,51 @@ int subvert; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; - state->sane = !subvert; #ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + state->sane = !subvert; return Z_OK; #else + (void)subvert; state->sane = 1; return Z_DATA_ERROR; #endif } +int ZEXPORT inflateValidate(strm, check) +z_streamp strm; +int check; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (check) + state->wrap |= 4; + else + state->wrap &= ~4; + return Z_OK; +} + long ZEXPORT inflateMark(strm) z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return -1L << 16; + if (inflateStateCheck(strm)) + return -(1L << 16); state = (struct inflate_state FAR *)strm->state; - return ((long)(state->back) << 16) + + return (long)(((unsigned long)((long)state->back)) << 16) + (state->mode == COPY ? state->length : (state->mode == MATCH ? state->was - state->length : 0)); } + +unsigned long ZEXPORT inflateCodesUsed(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (inflateStateCheck(strm)) return (unsigned long)-1; + state = (struct inflate_state FAR *)strm->state; + return (unsigned long)(state->next - state->codes); +} diff --git a/third_party/zlib/inflate.h b/third_party/zlib/inflate.h index 95f4986d40..a46cce6b6d 100644 --- a/third_party/zlib/inflate.h +++ b/third_party/zlib/inflate.h @@ -1,5 +1,5 @@ /* inflate.h -- internal inflate state definition - * Copyright (C) 1995-2009 Mark Adler + * Copyright (C) 1995-2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -18,7 +18,7 @@ /* Possible inflate modes between inflate() calls */ typedef enum { - HEAD, /* i: waiting for magic header */ + HEAD = 16180, /* i: waiting for magic header */ FLAGS, /* i: waiting for method and flags (gzip) */ TIME, /* i: waiting for modification time (gzip) */ OS, /* i: waiting for extra flags and operating system (gzip) */ @@ -77,11 +77,14 @@ typedef enum { CHECK -> LENGTH -> DONE */ -/* state maintained between inflate() calls. Approximately 10K bytes. */ +/* State maintained between inflate() calls -- approximately 7K bytes, not + including the allocated sliding window, which is up to 32K bytes. */ struct inflate_state { + z_streamp strm; /* pointer back to this zlib stream */ inflate_mode mode; /* current inflate mode */ int last; /* true if processing last block */ - int wrap; /* bit 0 true for zlib, bit 1 true for gzip */ + int wrap; /* bit 0 true for zlib, bit 1 true for gzip, + bit 2 true to validate check value */ int havedict; /* true if dictionary provided */ int flags; /* gzip header method and flags (0 if zlib) */ unsigned dmax; /* zlib header max distance (INFLATE_STRICT) */ diff --git a/third_party/zlib/inftrees.c b/third_party/zlib/inftrees.c index 11e9c52acc..2ea08fc13e 100644 --- a/third_party/zlib/inftrees.c +++ b/third_party/zlib/inftrees.c @@ -1,5 +1,5 @@ /* inftrees.c -- generate Huffman trees for efficient decoding - * Copyright (C) 1995-2010 Mark Adler + * Copyright (C) 1995-2017 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -9,7 +9,7 @@ #define MAXBITS 15 const char inflate_copyright[] = - " inflate 1.2.5 Copyright 1995-2010 Mark Adler "; + " inflate 1.2.11 Copyright 1995-2017 Mark Adler "; /* If you use the zlib library in a product, an acknowledgment is welcome in the documentation of your product. If for some reason you cannot @@ -54,7 +54,7 @@ unsigned short FAR *work; code FAR *next; /* next available space in table */ const unsigned short FAR *base; /* base value table to use */ const unsigned short FAR *extra; /* extra bits table to use */ - int end; /* use base and extra for symbol > end */ + unsigned match; /* use base and extra for symbol >= match */ unsigned short count[MAXBITS+1]; /* number of codes of each length */ unsigned short offs[MAXBITS+1]; /* offsets in table for each length */ static const unsigned short lbase[31] = { /* Length codes 257..285 base */ @@ -62,7 +62,7 @@ unsigned short FAR *work; 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0}; static const unsigned short lext[31] = { /* Length codes 257..285 extra */ 16, 16, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, - 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 73, 195}; + 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 77, 202}; static const unsigned short dbase[32] = { /* Distance codes 0..29 base */ 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, @@ -181,19 +181,17 @@ unsigned short FAR *work; switch (type) { case CODES: base = extra = work; /* dummy value--not used */ - end = 19; + match = 20; break; case LENS: base = lbase; - base -= 257; extra = lext; - extra -= 257; - end = 256; + match = 257; break; - default: /* DISTS */ + default: /* DISTS */ base = dbase; extra = dext; - end = -1; + match = 0; } /* initialize state for loop */ @@ -208,21 +206,21 @@ unsigned short FAR *work; mask = used - 1; /* mask for comparing low */ /* check available table space */ - if ((type == LENS && used >= ENOUGH_LENS) || - (type == DISTS && used >= ENOUGH_DISTS)) + if ((type == LENS && used > ENOUGH_LENS) || + (type == DISTS && used > ENOUGH_DISTS)) return 1; /* process all codes and make table entries */ for (;;) { /* create table entry */ here.bits = (unsigned char)(len - drop); - if ((int)(work[sym]) < end) { + if (work[sym] + 1U < match) { here.op = (unsigned char)0; here.val = work[sym]; } - else if ((int)(work[sym]) > end) { - here.op = (unsigned char)(extra[work[sym]]); - here.val = base[work[sym]]; + else if (work[sym] >= match) { + here.op = (unsigned char)(extra[work[sym] - match]); + here.val = base[work[sym] - match]; } else { here.op = (unsigned char)(32 + 64); /* end of block */ @@ -277,8 +275,8 @@ unsigned short FAR *work; /* check for enough space */ used += 1U << curr; - if ((type == LENS && used >= ENOUGH_LENS) || - (type == DISTS && used >= ENOUGH_DISTS)) + if ((type == LENS && used > ENOUGH_LENS) || + (type == DISTS && used > ENOUGH_DISTS)) return 1; /* point entry in root table to sub-table */ @@ -289,38 +287,14 @@ unsigned short FAR *work; } } - /* - Fill in rest of table for incomplete codes. This loop is similar to the - loop above in incrementing huff for table indices. It is assumed that - len is equal to curr + drop, so there is no loop needed to increment - through high index bits. When the current sub-table is filled, the loop - drops back to the root table to fill in any remaining entries there. - */ - here.op = (unsigned char)64; /* invalid code marker */ - here.bits = (unsigned char)(len - drop); - here.val = (unsigned short)0; - while (huff != 0) { - /* when done with sub-table, drop back to root table */ - if (drop != 0 && (huff & mask) != low) { - drop = 0; - len = root; - next = *table; - here.bits = (unsigned char)len; - } - - /* put invalid code marker in table */ - next[huff >> drop] = here; - - /* backwards increment the len-bit code huff */ - incr = 1U << (len - 1); - while (huff & incr) - incr >>= 1; - if (incr != 0) { - huff &= incr - 1; - huff += incr; - } - else - huff = 0; + /* fill in remaining table entry if code is incomplete (guaranteed to have + at most one remaining entry, since if the code is incomplete, the + maximum code length that was allowed to get this far is one bit) */ + if (huff != 0) { + here.op = (unsigned char)64; /* invalid code marker */ + here.bits = (unsigned char)(len - drop); + here.val = (unsigned short)0; + next[huff] = here; } /* set return parameters */ diff --git a/third_party/zlib/mixed-source.patch b/third_party/zlib/mixed-source.patch deleted file mode 100644 index ed54802c09..0000000000 --- a/third_party/zlib/mixed-source.patch +++ /dev/null @@ -1,496 +0,0 @@ -diff --git a/third_party/zlib/deflate.c b/third_party/zlib/deflate.c -index 5c4022f..88b2ec0 100644 ---- a/third_party/zlib/deflate.c -+++ b/third_party/zlib/deflate.c -@@ -70,14 +70,15 @@ typedef enum { - finish_done /* finish done, accept no more input or output */ - } block_state; - --typedef block_state (*compress_func) OF((deflate_state *s, int flush)); -+typedef block_state (*compress_func) OF((deflate_state *s, int flush, -+ int clas)); - /* Compression function. Returns the block state after the call. */ - - local void fill_window OF((deflate_state *s)); --local block_state deflate_stored OF((deflate_state *s, int flush)); --local block_state deflate_fast OF((deflate_state *s, int flush)); -+local block_state deflate_stored OF((deflate_state *s, int flush, int clas)); -+local block_state deflate_fast OF((deflate_state *s, int flush, int clas)); - #ifndef FASTEST --local block_state deflate_slow OF((deflate_state *s, int flush)); -+local block_state deflate_slow OF((deflate_state *s, int flush, int clas)); - #endif - local block_state deflate_rle OF((deflate_state *s, int flush)); - local block_state deflate_huff OF((deflate_state *s, int flush)); -@@ -87,9 +88,9 @@ local void flush_pending OF((z_streamp strm)); - local int read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); - #ifdef ASMV - void match_init OF((void)); /* asm code initialization */ -- uInt longest_match OF((deflate_state *s, IPos cur_match)); -+ uInt longest_match OF((deflate_state *s, IPos cur_match, int clas)); - #else --local uInt longest_match OF((deflate_state *s, IPos cur_match)); -+local uInt longest_match OF((deflate_state *s, IPos cur_match, int clas)); - #endif - - #ifdef DEBUG -@@ -281,6 +282,9 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, - s->window = (Bytef *) ZALLOC(strm, s->w_size, 2*sizeof(Byte)); - s->prev = (Posf *) ZALLOC(strm, s->w_size, sizeof(Pos)); - s->head = (Posf *) ZALLOC(strm, s->hash_size, sizeof(Pos)); -+ s->class_bitmap = NULL; -+ zmemzero(&s->cookie_locations, sizeof(s->cookie_locations)); -+ strm->clas = 0; - - s->high_water = 0; /* nothing written to s->window yet */ - -@@ -367,6 +371,8 @@ int ZEXPORT deflateReset (strm) - s = (deflate_state *)strm->state; - s->pending = 0; - s->pending_out = s->pending_buf; -+ TRY_FREE(strm, s->class_bitmap); -+ s->class_bitmap = NULL; - - if (s->wrap < 0) { - s->wrap = -s->wrap; /* was made negative by deflate(..., Z_FINISH); */ -@@ -817,9 +823,26 @@ int ZEXPORT deflate (strm, flush) - (flush != Z_NO_FLUSH && s->status != FINISH_STATE)) { - block_state bstate; - -- bstate = s->strategy == Z_HUFFMAN_ONLY ? deflate_huff(s, flush) : -- (s->strategy == Z_RLE ? deflate_rle(s, flush) : -- (*(configuration_table[s->level].func))(s, flush)); -+ if (strm->clas && s->class_bitmap == NULL) { -+ /* This is the first time that we have seen alternative class -+ * data. All data up till this point has been standard class. */ -+ s->class_bitmap = (Bytef*) ZALLOC(strm, s->w_size/4, sizeof(Byte)); -+ zmemzero(s->class_bitmap, s->w_size/4); -+ } -+ -+ if (strm->clas && s->strategy == Z_RLE) { -+ /* We haven't patched deflate_rle. */ -+ ERR_RETURN(strm, Z_BUF_ERROR); -+ } -+ -+ if (s->strategy == Z_HUFFMAN_ONLY) { -+ bstate = deflate_huff(s, flush); -+ } else if (s->strategy == Z_RLE) { -+ bstate = deflate_rle(s, flush); -+ } else { -+ bstate = (*(configuration_table[s->level].func)) -+ (s, flush, strm->clas); -+ } - - if (bstate == finish_started || bstate == finish_done) { - s->status = FINISH_STATE; -@@ -915,6 +938,7 @@ int ZEXPORT deflateEnd (strm) - TRY_FREE(strm, strm->state->head); - TRY_FREE(strm, strm->state->prev); - TRY_FREE(strm, strm->state->window); -+ TRY_FREE(strm, strm->state->class_bitmap); - - ZFREE(strm, strm->state); - strm->state = Z_NULL; -@@ -1046,6 +1070,57 @@ local void lm_init (s) - #endif - } - -+/* class_set sets bits [offset,offset+len) in s->class_bitmap to either 1 (if -+ * class != 0) or 0 (otherwise). */ -+local void class_set(s, offset, len, clas) -+ deflate_state *s; -+ IPos offset; -+ uInt len; -+ int clas; -+{ -+ IPos byte = offset >> 3; -+ IPos bit = offset & 7; -+ Bytef class_byte_value = clas ? 0xff : 0x00; -+ Bytef class_bit_value = clas ? 1 : 0; -+ static const Bytef mask[8] = {0xfe, 0xfd, 0xfb, 0xf7, -+ 0xef, 0xdf, 0xbf, 0x7f}; -+ -+ if (bit) { -+ while (len) { -+ s->class_bitmap[byte] &= mask[bit]; -+ s->class_bitmap[byte] |= class_bit_value << bit; -+ bit++; -+ len--; -+ if (bit == 8) { -+ bit = 0; -+ byte++; -+ break; -+ } -+ } -+ } -+ -+ while (len >= 8) { -+ s->class_bitmap[byte++] = class_byte_value; -+ len -= 8; -+ } -+ -+ while (len) { -+ s->class_bitmap[byte] &= mask[bit]; -+ s->class_bitmap[byte] |= class_bit_value << bit; -+ bit++; -+ len--; -+ } -+} -+ -+local int class_at(s, window_offset) -+ deflate_state *s; -+ IPos window_offset; -+{ -+ IPos byte = window_offset >> 3; -+ IPos bit = window_offset & 7; -+ return (s->class_bitmap[byte] >> bit) & 1; -+} -+ - #ifndef FASTEST - /* =========================================================================== - * Set match_start to the longest match starting at the given string and -@@ -1060,9 +1135,10 @@ local void lm_init (s) - /* For 80x86 and 680x0, an optimized version will be provided in match.asm or - * match.S. The code will be functionally equivalent. - */ --local uInt longest_match(s, cur_match) -+local uInt longest_match(s, cur_match, clas) - deflate_state *s; - IPos cur_match; /* current match */ -+ int clas; - { - unsigned chain_length = s->max_chain_length;/* max hash chain length */ - register Bytef *scan = s->window + s->strstart; /* current string */ -@@ -1110,6 +1186,9 @@ local uInt longest_match(s, cur_match) - do { - Assert(cur_match < s->strstart, "no future"); - match = s->window + cur_match; -+ /* If the matched data is in the wrong class, skip it. */ -+ if (s->class_bitmap && class_at(s, cur_match) != clas) -+ continue; - - /* Skip to next match if the match length cannot increase - * or if the match length is less than 2. Note that the checks below -@@ -1152,6 +1231,8 @@ local uInt longest_match(s, cur_match) - len = (MAX_MATCH - 1) - (int)(strend-scan); - scan = strend - (MAX_MATCH-1); - -+#error "UNALIGNED_OK hasn't been patched." -+ - #else /* UNALIGNED_OK */ - - if (match[best_len] != scan_end || -@@ -1168,15 +1249,23 @@ local uInt longest_match(s, cur_match) - scan += 2, match++; - Assert(*scan == *match, "match[2]?"); - -- /* We check for insufficient lookahead only every 8th comparison; -- * the 256th check will be made at strstart+258. -- */ -- do { -- } while (*++scan == *++match && *++scan == *++match && -- *++scan == *++match && *++scan == *++match && -- *++scan == *++match && *++scan == *++match && -- *++scan == *++match && *++scan == *++match && -- scan < strend); -+ if (!s->class_bitmap) { -+ /* We check for insufficient lookahead only every 8th comparison; -+ * the 256th check will be made at strstart+258. -+ */ -+ do { -+ } while (*++scan == *++match && *++scan == *++match && -+ *++scan == *++match && *++scan == *++match && -+ *++scan == *++match && *++scan == *++match && -+ *++scan == *++match && *++scan == *++match && -+ scan < strend); -+ } else { -+ /* We have to be mindful of the class of the data and not stray. */ -+ do { -+ } while (*++scan == *++match && -+ class_at(s, match - s->window) == clas && -+ scan < strend); -+ } - - Assert(scan <= s->window+(unsigned)(s->window_size-1), "wild scan"); - -@@ -1204,20 +1293,74 @@ local uInt longest_match(s, cur_match) - } - #endif /* ASMV */ - -+/* cookie_match is a replacement for longest_match in the case of cookie data. -+ * Here we only wish to match the entire value so trying the partial matches in -+ * longest_match is both wasteful and often fails to find the correct match. -+ * -+ * So we take the djb2 hash of the cookie and look up the last position for a -+ * match in a special hash table. */ -+local uInt cookie_match(s, start, len) -+ deflate_state *s; -+ IPos start; -+ unsigned len; -+{ -+ unsigned hash = 5381; -+ Bytef *str = s->window + start; -+ unsigned i; -+ IPos cookie_location; -+ -+ if (len >= MAX_MATCH || len == 0) -+ return 0; -+ -+ for (i = 0; i < len; i++) -+ hash = ((hash << 5) + hash) + str[i]; -+ -+ hash &= Z_COOKIE_HASH_MASK; -+ cookie_location = s->cookie_locations[hash]; -+ s->cookie_locations[hash] = start; -+ s->match_start = 0; -+ if (cookie_location && -+ (start - cookie_location) > len && -+ (start - cookie_location) < MAX_DIST(s) && -+ len <= s->lookahead) { -+ for (i = 0; i < len; i++) { -+ if (s->window[start+i] != s->window[cookie_location+i] || -+ class_at(s, cookie_location+i) != 1) { -+ return 0; -+ } -+ } -+ /* Check that we aren't matching a prefix of another cookie by ensuring -+ * that the final byte is either a semicolon (which cannot appear in a -+ * cookie value), or non-cookie data. */ -+ if (s->window[cookie_location+len-1] != ';' && -+ class_at(s, cookie_location+len) != 0) { -+ return 0; -+ } -+ s->match_start = cookie_location; -+ return len; -+ } -+ -+ return 0; -+} -+ -+ - #else /* FASTEST */ - - /* --------------------------------------------------------------------------- - * Optimized version for FASTEST only - */ --local uInt longest_match(s, cur_match) -+local uInt longest_match(s, cur_match, clas) - deflate_state *s; - IPos cur_match; /* current match */ -+ int clas; - { - register Bytef *scan = s->window + s->strstart; /* current string */ - register Bytef *match; /* matched string */ - register int len; /* length of current match */ - register Bytef *strend = s->window + s->strstart + MAX_MATCH; - -+#error "This code not patched" -+ - /* The code is optimized for HASH_BITS >= 8 and MAX_MATCH-2 multiple of 16. - * It is easy to get rid of this optimization if necessary. - */ -@@ -1360,6 +1503,21 @@ local void fill_window(s) - */ - } while (--n); - #endif -+ -+ for (n = 0; n < Z_COOKIE_HASH_SIZE; n++) { -+ if (s->cookie_locations[n] > wsize) { -+ s->cookie_locations[n] -= wsize; -+ } else { -+ s->cookie_locations[n] = 0; -+ } -+ } -+ -+ if (s->class_bitmap) { -+ zmemcpy(s->class_bitmap, s->class_bitmap + s->w_size/8, -+ s->w_size/8); -+ zmemzero(s->class_bitmap + s->w_size/8, s->w_size/8); -+ } -+ - more += wsize; - } - if (s->strm->avail_in == 0) return; -@@ -1378,6 +1536,9 @@ local void fill_window(s) - Assert(more >= 2, "more < 2"); - - n = read_buf(s->strm, s->window + s->strstart + s->lookahead, more); -+ if (s->class_bitmap != NULL) { -+ class_set(s, s->strstart + s->lookahead, n, s->strm->clas); -+ } - s->lookahead += n; - - /* Initialize the hash value now that we have some input: */ -@@ -1459,9 +1620,10 @@ local void fill_window(s) - * NOTE: this function should be optimized to avoid extra copying from - * window to pending_buf. - */ --local block_state deflate_stored(s, flush) -+local block_state deflate_stored(s, flush, clas) - deflate_state *s; - int flush; -+ int clas; - { - /* Stored blocks are limited to 0xffff bytes, pending_buf is limited - * to pending_buf_size, and each stored block has a 5 byte header: -@@ -1517,13 +1679,19 @@ local block_state deflate_stored(s, flush) - * new strings in the dictionary only for unmatched strings or for short - * matches. It is used only for the fast compression options. - */ --local block_state deflate_fast(s, flush) -+local block_state deflate_fast(s, flush, clas) - deflate_state *s; - int flush; -+ int clas; - { - IPos hash_head; /* head of the hash chain */ - int bflush; /* set if current block must be flushed */ - -+ if (clas != 0) { -+ /* We haven't patched this code for alternative class data. */ -+ return Z_BUF_ERROR; -+ } -+ - for (;;) { - /* Make sure that we always have enough lookahead, except - * at the end of the input file. We need MAX_MATCH bytes -@@ -1554,7 +1722,7 @@ local block_state deflate_fast(s, flush) - * of window index 0 (in particular we have to avoid a match - * of the string with itself at the start of the input file). - */ -- s->match_length = longest_match (s, hash_head); -+ s->match_length = longest_match (s, hash_head, clas); - /* longest_match() sets match_start */ - } - if (s->match_length >= MIN_MATCH) { -@@ -1613,12 +1781,25 @@ local block_state deflate_fast(s, flush) - * evaluation for matches: a match is finally adopted only if there is - * no better match at the next window position. - */ --local block_state deflate_slow(s, flush) -+local block_state deflate_slow(s, flush, clas) - deflate_state *s; - int flush; -+ int clas; - { - IPos hash_head; /* head of hash chain */ - int bflush; /* set if current block must be flushed */ -+ uInt input_length ; -+ int first = 1; /* first says whether this is the first iteration -+ of the loop, below. */ -+ -+ if (clas == Z_CLASS_COOKIE) { -+ if (s->lookahead) { -+ /* Alternative class data must always be presented at the beginning -+ * of a block. */ -+ return Z_BUF_ERROR; -+ } -+ input_length = s->strm->avail_in; -+ } - - /* Process the input block. */ - for (;;) { -@@ -1648,13 +1829,18 @@ local block_state deflate_slow(s, flush) - s->prev_length = s->match_length, s->prev_match = s->match_start; - s->match_length = MIN_MATCH-1; - -- if (hash_head != NIL && s->prev_length < s->max_lazy_match && -- s->strstart - hash_head <= MAX_DIST(s)) { -+ if (clas == Z_CLASS_COOKIE && first) { -+ s->match_length = cookie_match(s, s->strstart, input_length); -+ } else if (clas == Z_CLASS_STANDARD && -+ hash_head != NIL && -+ s->prev_length < s->max_lazy_match && -+ s->strstart - hash_head <= MAX_DIST(s)) { - /* To simplify the code, we prevent matches with the string - * of window index 0 (in particular we have to avoid a match - * of the string with itself at the start of the input file). - */ -- s->match_length = longest_match (s, hash_head); -+ s->match_length = longest_match (s, hash_head, clas); -+ - /* longest_match() sets match_start */ - - if (s->match_length <= 5 && (s->strategy == Z_FILTERED -@@ -1673,7 +1859,20 @@ local block_state deflate_slow(s, flush) - /* If there was a match at the previous step and the current - * match is not better, output the previous match: - */ -- if (s->prev_length >= MIN_MATCH && s->match_length <= s->prev_length) { -+ first = 0; -+ if (s->prev_length >= MIN_MATCH && s->match_length <= s->prev_length && -+ /* We will only accept an exact match for Z_CLASS_COOKIE data and -+ * we won't match Z_CLASS_HUFFMAN_ONLY data at all. */ -+ (clas == Z_CLASS_STANDARD || (clas == Z_CLASS_COOKIE && -+ s->prev_length == input_length && -+ s->prev_match > 0 && -+ /* We require that a Z_CLASS_COOKIE match be -+ * preceded by either a semicolon (which cannot be -+ * part of a cookie), or non-cookie data. This is -+ * to prevent a cookie from being a suffix of -+ * another. */ -+ (class_at(s, s->prev_match-1) == Z_CLASS_STANDARD || -+ *(s->window + s->prev_match-1) == ';')))) { - uInt max_insert = s->strstart + s->lookahead - MIN_MATCH; - /* Do not insert strings in hash table beyond this. */ - -diff --git a/third_party/zlib/deflate.h b/third_party/zlib/deflate.h -index cbf0d1e..2fe6fd6 100644 ---- a/third_party/zlib/deflate.h -+++ b/third_party/zlib/deflate.h -@@ -91,6 +91,9 @@ typedef unsigned IPos; - * save space in the various tables. IPos is used only for parameter passing. - */ - -+#define Z_COOKIE_HASH_SIZE 256 -+#define Z_COOKIE_HASH_MASK (Z_COOKIE_HASH_SIZE-1) -+ - typedef struct internal_state { - z_streamp strm; /* pointer back to this zlib stream */ - int status; /* as the name implies */ -@@ -139,6 +142,8 @@ typedef struct internal_state { - uInt hash_mask; /* hash_size-1 */ - - uInt hash_shift; -+ Bytef *class_bitmap; /* bitmap of class for each byte in window */ -+ IPos cookie_locations[Z_COOKIE_HASH_SIZE]; - /* Number of bits by which ins_h must be shifted at each input - * step. It must be such that after MIN_MATCH steps, the oldest - * byte no longer takes part in the hash key, that is: -diff --git a/third_party/zlib/zlib.h b/third_party/zlib/zlib.h -index 4d54af9..da7e971 100644 ---- a/third_party/zlib/zlib.h -+++ b/third_party/zlib/zlib.h -@@ -101,6 +101,7 @@ typedef struct z_stream_s { - int data_type; /* best guess about the data type: binary or text */ - uLong adler; /* adler32 value of the uncompressed data */ - uLong reserved; /* reserved for future use */ -+ int clas; - } z_stream; - - typedef z_stream FAR *z_streamp; -@@ -207,6 +208,10 @@ typedef gz_header FAR *gz_headerp; - - #define Z_NULL 0 /* for initializing zalloc, zfree, opaque */ - -+#define Z_CLASS_STANDARD 0 -+#define Z_CLASS_COOKIE 1 -+#define Z_CLASS_HUFFMAN_ONLY 2 -+ - #define zlib_version zlibVersion() - /* for compatibility with versions < 1.0.2 */ - -@@ -1587,6 +1592,13 @@ ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits, - ZEXTERN z_off_t ZEXPORT gzoffset64 OF((gzFile)); - ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t)); - ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t)); -+# else -+ ZEXTERN gzFile ZEXPORT gzopen OF((const char *, const char *)); -+ ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile, z_off_t, int)); -+ ZEXTERN z_off_t ZEXPORT gztell OF((gzFile)); -+ ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile)); -+ ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t)); -+ ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t)); - # endif - #else - ZEXTERN gzFile ZEXPORT gzopen OF((const char *, const char *)); diff --git a/third_party/zlib/mozzconf.h b/third_party/zlib/mozzconf.h deleted file mode 100644 index 25c1d827a8..0000000000 --- a/third_party/zlib/mozzconf.h +++ /dev/null @@ -1,168 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is the mozilla zlib configuration. - * - * The Initial Developer of the Original Code is IBM Corporation. - * Portions created by the Initial Developer are Copyright (C) 2004 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef MOZZCONF_H -#define MOZZCONF_H - -#if defined(XP_WIN) && defined(ZLIB_DLL) && !defined(MOZ_ENABLE_LIBXUL) -#undef ZLIB_DLL -#endif - -#ifdef HAVE_VISIBILITY_ATTRIBUTE -#define ZEXTERN __attribute__((visibility ("default"))) extern -#endif - -/* Exported Symbols */ -#define zlibVersion MOZ_Z_zlibVersion -#define deflate MOZ_Z_deflate -#define deflateEnd MOZ_Z_deflateEnd -#define inflate MOZ_Z_inflate -#define inflateEnd MOZ_Z_inflateEnd -#define deflateSetDictionary MOZ_Z_deflateSetDictionary -#define deflateCopy MOZ_Z_deflateCopy -#define deflateReset MOZ_Z_deflateReset -#define deflateParams MOZ_Z_deflateParams -#define deflateBound MOZ_Z_deflateBound -#define deflatePrime MOZ_Z_deflatePrime -#define inflateSetDictionary MOZ_Z_inflateSetDictionary -#define inflateSync MOZ_Z_inflateSync -#define inflateCopy MOZ_Z_inflateCopy -#define inflateReset MOZ_Z_inflateReset -#define inflateBack MOZ_Z_inflateBack -#define inflateBackEnd MOZ_Z_inflateBackEnd -#define zlibCompileFlags MOZ_Z_zlibCompileFlags -#define compress MOZ_Z_compress -#define compress2 MOZ_Z_compress2 -#define compressBound MOZ_Z_compressBound -#define uncompress MOZ_Z_uncompress -#define gzopen MOZ_Z_gzopen -#define gzdopen MOZ_Z_gzdopen -#define gzsetparams MOZ_Z_gzsetparams -#define gzread MOZ_Z_gzread -#define gzwrite MOZ_Z_gzwrite -#define gzprintf MOZ_Z_gzprintf -#define gzputs MOZ_Z_gzputs -#define gzgets MOZ_Z_gzgets -#define gzputc MOZ_Z_gzputc -#define gzgetc MOZ_Z_gzgetc -#define gzungetc MOZ_Z_gzungetc -#define gzflush MOZ_Z_gzflush -#define gzseek MOZ_Z_gzseek -#define gzrewind MOZ_Z_gzrewind -#define gztell MOZ_Z_gztell -#define gzeof MOZ_Z_gzeof -#define gzclose MOZ_Z_gzclose -#define gzerror MOZ_Z_gzerror -#define gzclearerr MOZ_Z_gzclearerr -#define adler32 MOZ_Z_adler32 -#define crc32 MOZ_Z_crc32 -#define deflateInit_ MOZ_Z_deflateInit_ -#define deflateInit2_ MOZ_Z_deflateInit2_ -#define inflateInit_ MOZ_Z_inflateInit_ -#define inflateInit2_ MOZ_Z_inflateInit2_ -#define inflateBackInit_ MOZ_Z_inflateBackInit_ -#define inflateSyncPoint MOZ_Z_inflateSyncPoint -#define get_crc_table MOZ_Z_get_crc_table -#define zError MOZ_Z_zError - -/* Extra global symbols */ -#define _dist_code MOZ_Z__dist_code -#define _length_code MOZ_Z__length_code -#define _tr_align MOZ_Z__tr_align -#define _tr_flush_block MOZ_Z__tr_flush_block -#define _tr_init MOZ_Z__tr_init -#define _tr_stored_block MOZ_Z__tr_stored_block -#define _tr_tally MOZ_Z__tr_tally -#define deflate_copyright MOZ_Z_deflate_copyright -#define inflate_copyright MOZ_Z_inflate_copyright -#define inflate_fast MOZ_Z_inflate_fast -#define inflate_table MOZ_Z_inflate_table -#define z_errmsg MOZ_Z_z_errmsg -#define zcalloc MOZ_Z_zcalloc -#define zcfree MOZ_Z_zcfree -#define alloc_func MOZ_Z_alloc_func -#define free_func MOZ_Z_free_func -#define in_func MOZ_Z_in_func -#define out_func MOZ_Z_out_func - -/* New as of libpng-1.2.3 */ -#define adler32_combine MOZ_Z_adler32_combine -#define crc32_combine MOZ_Z_crc32_combine -#define deflateSetHeader MOZ_Z_deflateSetHeader -#define deflateTune MOZ_Z_deflateTune -#define gzdirect MOZ_Z_gzdirect -#define inflatePrime MOZ_Z_inflatePrime -#define inflateGetHeader MOZ_Z_inflateGetHeader - -/* New as of zlib 1.2.5 */ -#define gzoffset MOZ_Z_gzoffset -#define gzopen64 MOZ_Z_gzopen64 -#define gzseek64 MOZ_Z_gzseek64 -#define gztell64 MOZ_Z_gztell64 -#define gzoffset64 MOZ_Z_gzoffset64 -#define adler32_combine64 MOZ_Z_adler32_combine64 -#define crc32_combine64 MOZ_Z_crc32_combine64 -#define gz_error MOZ_Z_gz_error -#define gz_intmax MOZ_Z_gz_intmax -#define gz_strwinerror MOZ_Z_gz_strwinerror -#define gzbuffer MOZ_Z_gzbuffer -#define gzclose_r MOZ_Z_gzclose_r -#define gzclose_w MOZ_Z_gzclose_w -#define inflateMark MOZ_Z_inflateMark -#define inflateReset2 MOZ_Z_inflateReset2 -#define inflateUndermine MOZ_Z_inflateUndermine -#define charf MOZ_Z_charf -#define gzFile MOZ_Z_gzFile -#define gz_header MOZ_Z_gz_header -#define gz_headerp MOZ_Z_gz_headerp -#define intf MOZ_Z_intf -#define uInt MOZ_Z_uInt -#define uIntf MOZ_Z_uIntf -#define uLong MOZ_Z_uLong -#define uLongf MOZ_Z_uLongf -#define voidp MOZ_Z_voidp -#define voidpc MOZ_Z_voidpc -#define voidpf MOZ_Z_voidpf -#define gz_header_s MOZ_Z_gz_header_s -#define internal_state MOZ_Z_internal_state - -/* Mangle Byte types except on Mac. */ -#if !defined(__MACTYPES__) -#define Byte MOZ_Z_Byte -#define Bytef MOZ_Z_Bytef -#endif - -#endif diff --git a/third_party/zlib/patches/0000-build.patch b/third_party/zlib/patches/0000-build.patch new file mode 100644 index 0000000000..b7c5996482 --- /dev/null +++ b/third_party/zlib/patches/0000-build.patch @@ -0,0 +1,247 @@ +diff --git a/contrib/minizip/ioapi.h b/contrib/minizip/ioapi.h +index 8dcbdb06e35a..c1b7a54847f5 100644 +--- a/contrib/minizip/ioapi.h ++++ b/contrib/minizip/ioapi.h +@@ -43,7 +43,7 @@ + + #include + #include +-#include "zlib.h" ++#include "third_party/zlib/zlib.h" + + #if defined(USE_FILE32API) + #define fopen64 fopen +diff --git a/contrib/minizip/iowin32.c b/contrib/minizip/iowin32.c +index 274f39eb1dd2..246ceb91a139 100644 +--- a/contrib/minizip/iowin32.c ++++ b/contrib/minizip/iowin32.c +@@ -26,12 +26,19 @@ + #endif + + ++#ifdef _WIN32_WINNT ++#undef _WIN32_WINNT ++#define _WIN32_WINNT 0x601 ++#endif ++ ++#if _WIN32_WINNT >= _WIN32_WINNT_WIN8 + // see Include/shared/winapifamily.h in the Windows Kit + #if defined(WINAPI_FAMILY_PARTITION) && (!(defined(IOWIN32_USING_WINRT_API))) + #if WINAPI_FAMILY_ONE_PARTITION(WINAPI_FAMILY, WINAPI_PARTITION_APP) + #define IOWIN32_USING_WINRT_API 1 + #endif + #endif ++#endif + + voidpf ZCALLBACK win32_open_file_func OF((voidpf opaque, const char* filename, int mode)); + uLong ZCALLBACK win32_read_file_func OF((voidpf opaque, voidpf stream, void* buf, uLong size)); +diff --git a/contrib/minizip/mztools.c b/contrib/minizip/mztools.c +index 96891c2e0b71..8bf9cca32633 100644 +--- a/contrib/minizip/mztools.c ++++ b/contrib/minizip/mztools.c +@@ -8,7 +8,7 @@ + #include + #include + #include +-#include "zlib.h" ++#include "third_party/zlib/zlib.h" + #include "unzip.h" + + #define READ_8(adr) ((unsigned char)*(adr)) +diff --git a/contrib/minizip/mztools.h b/contrib/minizip/mztools.h +index a49a426ec2fc..f295ffeda6af 100644 +--- a/contrib/minizip/mztools.h ++++ b/contrib/minizip/mztools.h +@@ -12,7 +12,7 @@ extern "C" { + #endif + + #ifndef _ZLIB_H +-#include "zlib.h" ++#include "third_party/zlib/zlib.h" + #endif + + #include "unzip.h" +diff --git a/contrib/minizip/unzip.c b/contrib/minizip/unzip.c +index bcfb9416ec35..199b4723fcfc 100644 +--- a/contrib/minizip/unzip.c ++++ b/contrib/minizip/unzip.c +@@ -72,7 +72,7 @@ + #define NOUNCRYPT + #endif + +-#include "zlib.h" ++#include "third_party/zlib/zlib.h" + #include "unzip.h" + + #ifdef STDC +@@ -1705,11 +1705,6 @@ extern int ZEXPORT unzReadCurrentFile (unzFile file, voidp buf, unsigned len) + + pfile_in_zip_read_info->stream.avail_out = (uInt)len; + +- if ((len>pfile_in_zip_read_info->rest_read_uncompressed) && +- (!(pfile_in_zip_read_info->raw))) +- pfile_in_zip_read_info->stream.avail_out = +- (uInt)pfile_in_zip_read_info->rest_read_uncompressed; +- + if ((len>pfile_in_zip_read_info->rest_read_compressed+ + pfile_in_zip_read_info->stream.avail_in) && + (pfile_in_zip_read_info->raw)) +diff --git a/contrib/minizip/unzip.h b/contrib/minizip/unzip.h +index 2104e3915074..3c0143529b91 100644 +--- a/contrib/minizip/unzip.h ++++ b/contrib/minizip/unzip.h +@@ -48,7 +48,7 @@ extern "C" { + #endif + + #ifndef _ZLIB_H +-#include "zlib.h" ++#include "third_party/zlib/zlib.h" + #endif + + #ifndef _ZLIBIOAPI_H +diff --git a/contrib/minizip/zip.c b/contrib/minizip/zip.c +index 44e88a9cb989..65c0c7251843 100644 +--- a/contrib/minizip/zip.c ++++ b/contrib/minizip/zip.c +@@ -26,7 +26,7 @@ + #include + #include + #include +-#include "zlib.h" ++#include "third_party/zlib/zlib.h" + #include "zip.h" + + #ifdef STDC +diff --git a/contrib/minizip/zip.h b/contrib/minizip/zip.h +index 8aaebb623430..8c06c0aa7bb0 100644 +--- a/contrib/minizip/zip.h ++++ b/contrib/minizip/zip.h +@@ -47,7 +47,7 @@ extern "C" { + //#define HAVE_BZIP2 + + #ifndef _ZLIB_H +-#include "zlib.h" ++#include "third_party/zlib/zlib.h" + #endif + + #ifndef _ZLIBIOAPI_H +diff --git a/gzread.c b/gzread.c +index 956b91ea7d9e..832d3ef98c59 100644 +--- a/gzread.c ++++ b/gzread.c +@@ -443,7 +443,11 @@ z_size_t ZEXPORT gzfread(buf, size, nitems, file) + # undef z_gzgetc + #else + # undef gzgetc ++# ifdef Z_CR_PREFIX_SET ++# define gzgetc Cr_z_gzgetc ++# endif + #endif ++ + int ZEXPORT gzgetc(file) + gzFile file; + { +diff --git a/zconf.h b/zconf.h +index 5e1d68a004e9..a7a815f575a7 100644 +--- a/zconf.h ++++ b/zconf.h +@@ -8,6 +8,10 @@ + #ifndef ZCONF_H + #define ZCONF_H + ++/* ++ * This library is also built as a part of AOSP, which does not need to include ++ * chromeconf.h. This config does not want chromeconf.h, so it can set this ++ * macro to opt out. While this works today, there's no guarantee that building ++ * zlib outside of Chromium keeps working in the future. ++ */ ++#if !defined(CHROMIUM_ZLIB_NO_CHROMECONF) ++/* This include does prefixing as below, but with an updated set of names. Also ++ * sets up export macros in component builds. */ ++#include "chromeconf.h" ++#endif ++ + /* + * If you *really* need a unique prefix for all types and library functions, + * compile with -DZ_PREFIX. The "standard" zlib should be compiled without it. +@@ -431,7 +434,7 @@ typedef uLong FAR uLongf; + typedef unsigned long z_crc_t; + #endif + +-#ifdef HAVE_UNISTD_H /* may be set to #if 1 by ./configure */ ++#if !defined(_WIN32) + # define Z_HAVE_UNISTD_H + #endif + +diff --git a/zlib.h b/zlib.h +index f09cdaf1e054..99fd467f6b1a 100644 +--- a/zlib.h ++++ b/zlib.h +@@ -1824,6 +1824,11 @@ ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file)); /* backward compatibility */ + # undef z_gzgetc + # define z_gzgetc(g) \ + ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g)) ++#elif defined(Z_CR_PREFIX_SET) ++# undef gzgetc ++# define gzgetc(g) \ ++ ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) \ ++ : (Cr_z_gzgetc)(g)) + #else + # define gzgetc(g) \ + ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g)) +@@ -1853,11 +1858,29 @@ ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file)); /* backward compatibility */ + # define z_adler32_combine z_adler32_combine64 + # define z_crc32_combine z_crc32_combine64 + # else ++# ifdef gzopen ++# undef gzopen ++# endif + # define gzopen gzopen64 ++# ifdef gzseek ++# undef gzseek ++# endif + # define gzseek gzseek64 ++# ifdef gztell ++# undef gztell ++# endif + # define gztell gztell64 ++# ifdef gzoffset ++# undef gzoffset ++# endif + # define gzoffset gzoffset64 ++# ifdef adler32_combine ++# undef adler32_combine ++# endif + # define adler32_combine adler32_combine64 ++# ifdef crc32_combine ++# undef crc32_combine ++# endif + # define crc32_combine crc32_combine64 + # endif + # ifndef Z_LARGE64 +diff --git a/zutil.h b/zutil.h +index b079ea6a80f5..80375b8b6109 100644 +--- a/zutil.h ++++ b/zutil.h +@@ -28,6 +28,21 @@ + # include + # include + #endif ++#ifdef NO_ERRNO_H ++# ifdef _WIN32_WCE ++ /* The Microsoft C Run-Time Library for Windows CE doesn't have ++ * errno. We define it as a global variable to simplify porting. ++ * Its value is always 0 and should not be used. We rename it to ++ * avoid conflict with other libraries that use the same workaround. ++ */ ++# define errno z_errno ++# endif ++ extern int errno; ++#else ++# ifndef _WIN32_WCE ++# include ++# endif ++#endif + + #ifdef Z_SOLO + typedef long ptrdiff_t; /* guess -- will be caught if guess is wrong */ diff --git a/third_party/zlib/patches/0001-simd.patch b/third_party/zlib/patches/0001-simd.patch new file mode 100644 index 0000000000..9434ca0cc4 --- /dev/null +++ b/third_party/zlib/patches/0001-simd.patch @@ -0,0 +1,1214 @@ +diff --git a/crc32.c b/crc32.c +index 9580440c0e6b..9162429cc7b4 100644 +--- a/crc32.c ++++ b/crc32.c +@@ -28,6 +28,8 @@ + # endif /* !DYNAMIC_CRC_TABLE */ + #endif /* MAKECRCH */ + ++#include "deflate.h" ++#include "x86.h" + #include "zutil.h" /* for STDC and FAR definitions */ + + /* Definitions for doing the crc four data bytes at a time. */ +@@ -440,3 +442,28 @@ uLong ZEXPORT crc32_combine64(crc1, crc2, len2) + { + return crc32_combine_(crc1, crc2, len2); + } ++ ++ZLIB_INTERNAL void crc_reset(deflate_state *const s) ++{ ++ if (x86_cpu_enable_simd) { ++ crc_fold_init(s); ++ return; ++ } ++ s->strm->adler = crc32(0L, Z_NULL, 0); ++} ++ ++ZLIB_INTERNAL void crc_finalize(deflate_state *const s) ++{ ++ if (x86_cpu_enable_simd) ++ s->strm->adler = crc_fold_512to32(s); ++} ++ ++ZLIB_INTERNAL void copy_with_crc(z_streamp strm, Bytef *dst, long size) ++{ ++ if (x86_cpu_enable_simd) { ++ crc_fold_copy(strm->state, dst, strm->next_in, size); ++ return; ++ } ++ zmemcpy(dst, strm->next_in, size); ++ strm->adler = crc32(strm->adler, dst, size); ++} +diff --git a/crc_folding.c b/crc_folding.c +new file mode 100644 +index 000000000000..48d77744aaf4 +--- /dev/null ++++ b/crc_folding.c +@@ -0,0 +1,493 @@ ++/* ++ * Compute the CRC32 using a parallelized folding approach with the PCLMULQDQ ++ * instruction. ++ * ++ * A white paper describing this algorithm can be found at: ++ * http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf ++ * ++ * Copyright (C) 2013 Intel Corporation. All rights reserved. ++ * Authors: ++ * Wajdi Feghali ++ * Jim Guilford ++ * Vinodh Gopal ++ * Erdinc Ozturk ++ * Jim Kukunas ++ * ++ * For conditions of distribution and use, see copyright notice in zlib.h ++ */ ++ ++#include "deflate.h" ++ ++#include ++#include ++#include ++#include ++ ++#define CRC_LOAD(s) \ ++ do { \ ++ __m128i xmm_crc0 = _mm_loadu_si128((__m128i *)s->crc0 + 0);\ ++ __m128i xmm_crc1 = _mm_loadu_si128((__m128i *)s->crc0 + 1);\ ++ __m128i xmm_crc2 = _mm_loadu_si128((__m128i *)s->crc0 + 2);\ ++ __m128i xmm_crc3 = _mm_loadu_si128((__m128i *)s->crc0 + 3);\ ++ __m128i xmm_crc_part = _mm_loadu_si128((__m128i *)s->crc0 + 4); ++ ++#define CRC_SAVE(s) \ ++ _mm_storeu_si128((__m128i *)s->crc0 + 0, xmm_crc0);\ ++ _mm_storeu_si128((__m128i *)s->crc0 + 1, xmm_crc1);\ ++ _mm_storeu_si128((__m128i *)s->crc0 + 2, xmm_crc2);\ ++ _mm_storeu_si128((__m128i *)s->crc0 + 3, xmm_crc3);\ ++ _mm_storeu_si128((__m128i *)s->crc0 + 4, xmm_crc_part);\ ++ } while (0); ++ ++ZLIB_INTERNAL void crc_fold_init(deflate_state *const s) ++{ ++ CRC_LOAD(s) ++ ++ xmm_crc0 = _mm_cvtsi32_si128(0x9db42487); ++ xmm_crc1 = _mm_setzero_si128(); ++ xmm_crc2 = _mm_setzero_si128(); ++ xmm_crc3 = _mm_setzero_si128(); ++ ++ CRC_SAVE(s) ++ ++ s->strm->adler = 0; ++} ++ ++local void fold_1(deflate_state *const s, ++ __m128i *xmm_crc0, __m128i *xmm_crc1, ++ __m128i *xmm_crc2, __m128i *xmm_crc3) ++{ ++ const __m128i xmm_fold4 = _mm_set_epi32( ++ 0x00000001, 0x54442bd4, ++ 0x00000001, 0xc6e41596); ++ ++ __m128i x_tmp3; ++ __m128 ps_crc0, ps_crc3, ps_res; ++ ++ x_tmp3 = *xmm_crc3; ++ ++ *xmm_crc3 = *xmm_crc0; ++ *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); ++ *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); ++ ps_crc0 = _mm_castsi128_ps(*xmm_crc0); ++ ps_crc3 = _mm_castsi128_ps(*xmm_crc3); ++ ps_res = _mm_xor_ps(ps_crc0, ps_crc3); ++ ++ *xmm_crc0 = *xmm_crc1; ++ *xmm_crc1 = *xmm_crc2; ++ *xmm_crc2 = x_tmp3; ++ *xmm_crc3 = _mm_castps_si128(ps_res); ++} ++ ++local void fold_2(deflate_state *const s, ++ __m128i *xmm_crc0, __m128i *xmm_crc1, ++ __m128i *xmm_crc2, __m128i *xmm_crc3) ++{ ++ const __m128i xmm_fold4 = _mm_set_epi32( ++ 0x00000001, 0x54442bd4, ++ 0x00000001, 0xc6e41596); ++ ++ __m128i x_tmp3, x_tmp2; ++ __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3, ps_res31, ps_res20; ++ ++ x_tmp3 = *xmm_crc3; ++ x_tmp2 = *xmm_crc2; ++ ++ *xmm_crc3 = *xmm_crc1; ++ *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); ++ *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); ++ ps_crc3 = _mm_castsi128_ps(*xmm_crc3); ++ ps_crc1 = _mm_castsi128_ps(*xmm_crc1); ++ ps_res31= _mm_xor_ps(ps_crc3, ps_crc1); ++ ++ *xmm_crc2 = *xmm_crc0; ++ *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); ++ *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10); ++ ps_crc0 = _mm_castsi128_ps(*xmm_crc0); ++ ps_crc2 = _mm_castsi128_ps(*xmm_crc2); ++ ps_res20= _mm_xor_ps(ps_crc0, ps_crc2); ++ ++ *xmm_crc0 = x_tmp2; ++ *xmm_crc1 = x_tmp3; ++ *xmm_crc2 = _mm_castps_si128(ps_res20); ++ *xmm_crc3 = _mm_castps_si128(ps_res31); ++} ++ ++local void fold_3(deflate_state *const s, ++ __m128i *xmm_crc0, __m128i *xmm_crc1, ++ __m128i *xmm_crc2, __m128i *xmm_crc3) ++{ ++ const __m128i xmm_fold4 = _mm_set_epi32( ++ 0x00000001, 0x54442bd4, ++ 0x00000001, 0xc6e41596); ++ ++ __m128i x_tmp3; ++ __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3, ps_res32, ps_res21, ps_res10; ++ ++ x_tmp3 = *xmm_crc3; ++ ++ *xmm_crc3 = *xmm_crc2; ++ *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01); ++ *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); ++ ps_crc2 = _mm_castsi128_ps(*xmm_crc2); ++ ps_crc3 = _mm_castsi128_ps(*xmm_crc3); ++ ps_res32 = _mm_xor_ps(ps_crc2, ps_crc3); ++ ++ *xmm_crc2 = *xmm_crc1; ++ *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); ++ *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10); ++ ps_crc1 = _mm_castsi128_ps(*xmm_crc1); ++ ps_crc2 = _mm_castsi128_ps(*xmm_crc2); ++ ps_res21= _mm_xor_ps(ps_crc1, ps_crc2); ++ ++ *xmm_crc1 = *xmm_crc0; ++ *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); ++ *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x10); ++ ps_crc0 = _mm_castsi128_ps(*xmm_crc0); ++ ps_crc1 = _mm_castsi128_ps(*xmm_crc1); ++ ps_res10= _mm_xor_ps(ps_crc0, ps_crc1); ++ ++ *xmm_crc0 = x_tmp3; ++ *xmm_crc1 = _mm_castps_si128(ps_res10); ++ *xmm_crc2 = _mm_castps_si128(ps_res21); ++ *xmm_crc3 = _mm_castps_si128(ps_res32); ++} ++ ++local void fold_4(deflate_state *const s, ++ __m128i *xmm_crc0, __m128i *xmm_crc1, ++ __m128i *xmm_crc2, __m128i *xmm_crc3) ++{ ++ const __m128i xmm_fold4 = _mm_set_epi32( ++ 0x00000001, 0x54442bd4, ++ 0x00000001, 0xc6e41596); ++ ++ __m128i x_tmp0, x_tmp1, x_tmp2, x_tmp3; ++ __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3; ++ __m128 ps_t0, ps_t1, ps_t2, ps_t3; ++ __m128 ps_res0, ps_res1, ps_res2, ps_res3; ++ ++ x_tmp0 = *xmm_crc0; ++ x_tmp1 = *xmm_crc1; ++ x_tmp2 = *xmm_crc2; ++ x_tmp3 = *xmm_crc3; ++ ++ *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); ++ x_tmp0 = _mm_clmulepi64_si128(x_tmp0, xmm_fold4, 0x10); ++ ps_crc0 = _mm_castsi128_ps(*xmm_crc0); ++ ps_t0 = _mm_castsi128_ps(x_tmp0); ++ ps_res0 = _mm_xor_ps(ps_crc0, ps_t0); ++ ++ *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); ++ x_tmp1 = _mm_clmulepi64_si128(x_tmp1, xmm_fold4, 0x10); ++ ps_crc1 = _mm_castsi128_ps(*xmm_crc1); ++ ps_t1 = _mm_castsi128_ps(x_tmp1); ++ ps_res1 = _mm_xor_ps(ps_crc1, ps_t1); ++ ++ *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01); ++ x_tmp2 = _mm_clmulepi64_si128(x_tmp2, xmm_fold4, 0x10); ++ ps_crc2 = _mm_castsi128_ps(*xmm_crc2); ++ ps_t2 = _mm_castsi128_ps(x_tmp2); ++ ps_res2 = _mm_xor_ps(ps_crc2, ps_t2); ++ ++ *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x01); ++ x_tmp3 = _mm_clmulepi64_si128(x_tmp3, xmm_fold4, 0x10); ++ ps_crc3 = _mm_castsi128_ps(*xmm_crc3); ++ ps_t3 = _mm_castsi128_ps(x_tmp3); ++ ps_res3 = _mm_xor_ps(ps_crc3, ps_t3); ++ ++ *xmm_crc0 = _mm_castps_si128(ps_res0); ++ *xmm_crc1 = _mm_castps_si128(ps_res1); ++ *xmm_crc2 = _mm_castps_si128(ps_res2); ++ *xmm_crc3 = _mm_castps_si128(ps_res3); ++} ++ ++local const unsigned zalign(32) pshufb_shf_table[60] = { ++ 0x84838281,0x88878685,0x8c8b8a89,0x008f8e8d, /* shl 15 (16 - 1)/shr1 */ ++ 0x85848382,0x89888786,0x8d8c8b8a,0x01008f8e, /* shl 14 (16 - 3)/shr2 */ ++ 0x86858483,0x8a898887,0x8e8d8c8b,0x0201008f, /* shl 13 (16 - 4)/shr3 */ ++ 0x87868584,0x8b8a8988,0x8f8e8d8c,0x03020100, /* shl 12 (16 - 4)/shr4 */ ++ 0x88878685,0x8c8b8a89,0x008f8e8d,0x04030201, /* shl 11 (16 - 5)/shr5 */ ++ 0x89888786,0x8d8c8b8a,0x01008f8e,0x05040302, /* shl 10 (16 - 6)/shr6 */ ++ 0x8a898887,0x8e8d8c8b,0x0201008f,0x06050403, /* shl 9 (16 - 7)/shr7 */ ++ 0x8b8a8988,0x8f8e8d8c,0x03020100,0x07060504, /* shl 8 (16 - 8)/shr8 */ ++ 0x8c8b8a89,0x008f8e8d,0x04030201,0x08070605, /* shl 7 (16 - 9)/shr9 */ ++ 0x8d8c8b8a,0x01008f8e,0x05040302,0x09080706, /* shl 6 (16 -10)/shr10*/ ++ 0x8e8d8c8b,0x0201008f,0x06050403,0x0a090807, /* shl 5 (16 -11)/shr11*/ ++ 0x8f8e8d8c,0x03020100,0x07060504,0x0b0a0908, /* shl 4 (16 -12)/shr12*/ ++ 0x008f8e8d,0x04030201,0x08070605,0x0c0b0a09, /* shl 3 (16 -13)/shr13*/ ++ 0x01008f8e,0x05040302,0x09080706,0x0d0c0b0a, /* shl 2 (16 -14)/shr14*/ ++ 0x0201008f,0x06050403,0x0a090807,0x0e0d0c0b /* shl 1 (16 -15)/shr15*/ ++}; ++ ++local void partial_fold(deflate_state *const s, const size_t len, ++ __m128i *xmm_crc0, __m128i *xmm_crc1, ++ __m128i *xmm_crc2, __m128i *xmm_crc3, ++ __m128i *xmm_crc_part) ++{ ++ ++ const __m128i xmm_fold4 = _mm_set_epi32( ++ 0x00000001, 0x54442bd4, ++ 0x00000001, 0xc6e41596); ++ const __m128i xmm_mask3 = _mm_set1_epi32(0x80808080); ++ ++ __m128i xmm_shl, xmm_shr, xmm_tmp1, xmm_tmp2, xmm_tmp3; ++ __m128i xmm_a0_0, xmm_a0_1; ++ __m128 ps_crc3, psa0_0, psa0_1, ps_res; ++ ++ xmm_shl = _mm_load_si128((__m128i *)pshufb_shf_table + (len - 1)); ++ xmm_shr = xmm_shl; ++ xmm_shr = _mm_xor_si128(xmm_shr, xmm_mask3); ++ ++ xmm_a0_0 = _mm_shuffle_epi8(*xmm_crc0, xmm_shl); ++ ++ *xmm_crc0 = _mm_shuffle_epi8(*xmm_crc0, xmm_shr); ++ xmm_tmp1 = _mm_shuffle_epi8(*xmm_crc1, xmm_shl); ++ *xmm_crc0 = _mm_or_si128(*xmm_crc0, xmm_tmp1); ++ ++ *xmm_crc1 = _mm_shuffle_epi8(*xmm_crc1, xmm_shr); ++ xmm_tmp2 = _mm_shuffle_epi8(*xmm_crc2, xmm_shl); ++ *xmm_crc1 = _mm_or_si128(*xmm_crc1, xmm_tmp2); ++ ++ *xmm_crc2 = _mm_shuffle_epi8(*xmm_crc2, xmm_shr); ++ xmm_tmp3 = _mm_shuffle_epi8(*xmm_crc3, xmm_shl); ++ *xmm_crc2 = _mm_or_si128(*xmm_crc2, xmm_tmp3); ++ ++ *xmm_crc3 = _mm_shuffle_epi8(*xmm_crc3, xmm_shr); ++ *xmm_crc_part = _mm_shuffle_epi8(*xmm_crc_part, xmm_shl); ++ *xmm_crc3 = _mm_or_si128(*xmm_crc3, *xmm_crc_part); ++ ++ xmm_a0_1 = _mm_clmulepi64_si128(xmm_a0_0, xmm_fold4, 0x10); ++ xmm_a0_0 = _mm_clmulepi64_si128(xmm_a0_0, xmm_fold4, 0x01); ++ ++ ps_crc3 = _mm_castsi128_ps(*xmm_crc3); ++ psa0_0 = _mm_castsi128_ps(xmm_a0_0); ++ psa0_1 = _mm_castsi128_ps(xmm_a0_1); ++ ++ ps_res = _mm_xor_ps(ps_crc3, psa0_0); ++ ps_res = _mm_xor_ps(ps_res, psa0_1); ++ ++ *xmm_crc3 = _mm_castps_si128(ps_res); ++} ++ ++ZLIB_INTERNAL void crc_fold_copy(deflate_state *const s, ++ unsigned char *dst, const unsigned char *src, long len) ++{ ++ unsigned long algn_diff; ++ __m128i xmm_t0, xmm_t1, xmm_t2, xmm_t3; ++ ++ CRC_LOAD(s) ++ ++ if (len < 16) { ++ if (len == 0) ++ return; ++ goto partial; ++ } ++ ++ algn_diff = 0 - (uintptr_t)src & 0xF; ++ if (algn_diff) { ++ xmm_crc_part = _mm_loadu_si128((__m128i *)src); ++ _mm_storeu_si128((__m128i *)dst, xmm_crc_part); ++ ++ dst += algn_diff; ++ src += algn_diff; ++ len -= algn_diff; ++ ++ partial_fold(s, algn_diff, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, ++ &xmm_crc_part); ++ } ++ ++ while ((len -= 64) >= 0) { ++ xmm_t0 = _mm_load_si128((__m128i *)src); ++ xmm_t1 = _mm_load_si128((__m128i *)src + 1); ++ xmm_t2 = _mm_load_si128((__m128i *)src + 2); ++ xmm_t3 = _mm_load_si128((__m128i *)src + 3); ++ ++ fold_4(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); ++ ++ _mm_storeu_si128((__m128i *)dst, xmm_t0); ++ _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); ++ _mm_storeu_si128((__m128i *)dst + 2, xmm_t2); ++ _mm_storeu_si128((__m128i *)dst + 3, xmm_t3); ++ ++ xmm_crc0 = _mm_xor_si128(xmm_crc0, xmm_t0); ++ xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t1); ++ xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t2); ++ xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t3); ++ ++ src += 64; ++ dst += 64; ++ } ++ ++ /* ++ * len = num bytes left - 64 ++ */ ++ if (len + 16 >= 0) { ++ len += 16; ++ ++ xmm_t0 = _mm_load_si128((__m128i *)src); ++ xmm_t1 = _mm_load_si128((__m128i *)src + 1); ++ xmm_t2 = _mm_load_si128((__m128i *)src + 2); ++ ++ fold_3(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); ++ ++ _mm_storeu_si128((__m128i *)dst, xmm_t0); ++ _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); ++ _mm_storeu_si128((__m128i *)dst + 2, xmm_t2); ++ ++ xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t0); ++ xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t1); ++ xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t2); ++ ++ if (len == 0) ++ goto done; ++ ++ dst += 48; ++ src += 48; ++ } else if (len + 32 >= 0) { ++ len += 32; ++ ++ xmm_t0 = _mm_load_si128((__m128i *)src); ++ xmm_t1 = _mm_load_si128((__m128i *)src + 1); ++ ++ fold_2(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); ++ ++ _mm_storeu_si128((__m128i *)dst, xmm_t0); ++ _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); ++ ++ xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t0); ++ xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t1); ++ ++ if (len == 0) ++ goto done; ++ ++ dst += 32; ++ src += 32; ++ } else if (len + 48 >= 0) { ++ len += 48; ++ ++ xmm_t0 = _mm_load_si128((__m128i *)src); ++ ++ fold_1(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); ++ ++ _mm_storeu_si128((__m128i *)dst, xmm_t0); ++ ++ xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t0); ++ ++ if (len == 0) ++ goto done; ++ ++ dst += 16; ++ src += 16; ++ } else { ++ len += 64; ++ if (len == 0) ++ goto done; ++ } ++ ++partial: ++ ++#if defined(_MSC_VER) ++ /* VS does not permit the use of _mm_set_epi64x in 32-bit builds */ ++ { ++ int32_t parts[4] = {0, 0, 0, 0}; ++ memcpy(&parts, src, len); ++ xmm_crc_part = _mm_set_epi32(parts[3], parts[2], parts[1], parts[0]); ++ } ++#else ++ { ++ int64_t parts[2] = {0, 0}; ++ memcpy(&parts, src, len); ++ xmm_crc_part = _mm_set_epi64x(parts[1], parts[0]); ++ } ++#endif ++ ++ _mm_storeu_si128((__m128i *)dst, xmm_crc_part); ++ partial_fold(s, len, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, ++ &xmm_crc_part); ++done: ++ CRC_SAVE(s) ++} ++ ++local const unsigned zalign(16) crc_k[] = { ++ 0xccaa009e, 0x00000000, /* rk1 */ ++ 0x751997d0, 0x00000001, /* rk2 */ ++ 0xccaa009e, 0x00000000, /* rk5 */ ++ 0x63cd6124, 0x00000001, /* rk6 */ ++ 0xf7011640, 0x00000001, /* rk7 */ ++ 0xdb710640, 0x00000001 /* rk8 */ ++}; ++ ++local const unsigned zalign(16) crc_mask[4] = { ++ 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000 ++}; ++ ++local const unsigned zalign(16) crc_mask2[4] = { ++ 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF ++}; ++ ++unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state *const s) ++{ ++ const __m128i xmm_mask = _mm_load_si128((__m128i *)crc_mask); ++ const __m128i xmm_mask2 = _mm_load_si128((__m128i *)crc_mask2); ++ ++ unsigned crc; ++ __m128i x_tmp0, x_tmp1, x_tmp2, crc_fold; ++ ++ CRC_LOAD(s) ++ ++ /* ++ * k1 ++ */ ++ crc_fold = _mm_load_si128((__m128i *)crc_k); ++ ++ x_tmp0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x10); ++ xmm_crc0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x01); ++ xmm_crc1 = _mm_xor_si128(xmm_crc1, x_tmp0); ++ xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_crc0); ++ ++ x_tmp1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x10); ++ xmm_crc1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x01); ++ xmm_crc2 = _mm_xor_si128(xmm_crc2, x_tmp1); ++ xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_crc1); ++ ++ x_tmp2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x10); ++ xmm_crc2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x01); ++ xmm_crc3 = _mm_xor_si128(xmm_crc3, x_tmp2); ++ xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); ++ ++ /* ++ * k5 ++ */ ++ crc_fold = _mm_load_si128((__m128i *)crc_k + 1); ++ ++ xmm_crc0 = xmm_crc3; ++ xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0); ++ xmm_crc0 = _mm_srli_si128(xmm_crc0, 8); ++ xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0); ++ ++ xmm_crc0 = xmm_crc3; ++ xmm_crc3 = _mm_slli_si128(xmm_crc3, 4); ++ xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10); ++ xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0); ++ xmm_crc3 = _mm_and_si128(xmm_crc3, xmm_mask2); ++ ++ /* ++ * k7 ++ */ ++ xmm_crc1 = xmm_crc3; ++ xmm_crc2 = xmm_crc3; ++ crc_fold = _mm_load_si128((__m128i *)crc_k + 2); ++ ++ xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0); ++ xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); ++ xmm_crc3 = _mm_and_si128(xmm_crc3, xmm_mask); ++ ++ xmm_crc2 = xmm_crc3; ++ xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10); ++ xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); ++ xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc1); ++ ++ crc = _mm_extract_epi32(xmm_crc3, 2); ++ return ~crc; ++ CRC_SAVE(s) ++} +diff --git a/deflate.c b/deflate.c +index 1ec761448de9..aa0c9c67a6dc 100644 +--- a/deflate.c ++++ b/deflate.c +@@ -48,8 +48,9 @@ + */ + + /* @(#) $Id$ */ +- ++#include + #include "deflate.h" ++#include "x86.h" + + const char deflate_copyright[] = + " deflate 1.2.11 Copyright 1995-2017 Jean-loup Gailly and Mark Adler "; +@@ -86,7 +87,7 @@ local block_state deflate_huff OF((deflate_state *s, int flush)); + local void lm_init OF((deflate_state *s)); + local void putShortMSB OF((deflate_state *s, uInt b)); + local void flush_pending OF((z_streamp strm)); +-local unsigned read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); ++unsigned ZLIB_INTERNAL deflate_read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); + #ifdef ASMV + # pragma message("Assembler code may have bugs -- use at your own risk") + void match_init OF((void)); /* asm code initialization */ +@@ -100,6 +101,20 @@ local void check_match OF((deflate_state *s, IPos start, IPos match, + int length)); + #endif + ++/* From crc32.c */ ++extern void ZLIB_INTERNAL crc_reset(deflate_state *const s); ++extern void ZLIB_INTERNAL crc_finalize(deflate_state *const s); ++extern void ZLIB_INTERNAL copy_with_crc(z_streamp strm, Bytef *dst, long size); ++ ++#ifdef _MSC_VER ++#define INLINE __inline ++#else ++#define INLINE inline ++#endif ++ ++/* Inline optimisation */ ++local INLINE Pos insert_string_sse(deflate_state *const s, const Pos str); ++ + /* =========================================================================== + * Local data + */ +@@ -162,7 +177,6 @@ local const config configuration_table[10] = { + */ + #define UPDATE_HASH(s,h,c) (h = (((h)<hash_shift) ^ (c)) & s->hash_mask) + +- + /* =========================================================================== + * Insert string str in the dictionary and set match_head to the previous head + * of the hash chain (the most recent string with same hash key). Return +@@ -173,17 +187,28 @@ local const config configuration_table[10] = { + * characters and the first MIN_MATCH bytes of str are valid (except for + * the last MIN_MATCH-1 bytes of the input file). + */ ++local INLINE Pos insert_string_c(deflate_state *const s, const Pos str) ++{ ++ Pos ret; ++ ++ UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]); + #ifdef FASTEST +-#define INSERT_STRING(s, str, match_head) \ +- (UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]), \ +- match_head = s->head[s->ins_h], \ +- s->head[s->ins_h] = (Pos)(str)) ++ ret = s->head[s->ins_h]; + #else +-#define INSERT_STRING(s, str, match_head) \ +- (UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]), \ +- match_head = s->prev[(str) & s->w_mask] = s->head[s->ins_h], \ +- s->head[s->ins_h] = (Pos)(str)) ++ ret = s->prev[str & s->w_mask] = s->head[s->ins_h]; + #endif ++ s->head[s->ins_h] = str; ++ ++ return ret; ++} ++ ++local INLINE Pos insert_string(deflate_state *const s, const Pos str) ++{ ++ if (x86_cpu_enable_simd) ++ return insert_string_sse(s, str); ++ return insert_string_c(s, str); ++} ++ + + /* =========================================================================== + * Initialize the hash table (avoiding 64K overflow for 16 bit systems). +@@ -248,6 +273,7 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, + const char *version; + int stream_size; + { ++ unsigned window_padding = 8; + deflate_state *s; + int wrap = 1; + static const char my_version[] = ZLIB_VERSION; +@@ -257,6 +283,8 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, + * output size for (length,distance) codes is <= 24 bits. + */ + ++ x86_check_features(); ++ + if (version == Z_NULL || version[0] != my_version[0] || + stream_size != sizeof(z_stream)) { + return Z_VERSION_ERROR; +@@ -313,12 +341,19 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, + s->w_size = 1 << s->w_bits; + s->w_mask = s->w_size - 1; + +- s->hash_bits = (uInt)memLevel + 7; ++ if (x86_cpu_enable_simd) { ++ s->hash_bits = 15; ++ } else { ++ s->hash_bits = memLevel + 7; ++ } ++ + s->hash_size = 1 << s->hash_bits; + s->hash_mask = s->hash_size - 1; + s->hash_shift = ((s->hash_bits+MIN_MATCH-1)/MIN_MATCH); + +- s->window = (Bytef *) ZALLOC(strm, s->w_size, 2*sizeof(Byte)); ++ s->window = (Bytef *) ZALLOC(strm, ++ s->w_size + window_padding, ++ 2*sizeof(Byte)); + s->prev = (Posf *) ZALLOC(strm, s->w_size, sizeof(Pos)); + s->head = (Posf *) ZALLOC(strm, s->hash_size, sizeof(Pos)); + +@@ -418,11 +453,7 @@ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength) + str = s->strstart; + n = s->lookahead - (MIN_MATCH-1); + do { +- UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]); +-#ifndef FASTEST +- s->prev[str & s->w_mask] = s->head[s->ins_h]; +-#endif +- s->head[s->ins_h] = (Pos)str; ++ insert_string(s, str); + str++; + } while (--n); + s->strstart = str; +@@ -848,7 +879,7 @@ int ZEXPORT deflate (strm, flush) + #ifdef GZIP + if (s->status == GZIP_STATE) { + /* gzip header */ +- strm->adler = crc32(0L, Z_NULL, 0); ++ crc_reset(s); + put_byte(s, 31); + put_byte(s, 139); + put_byte(s, 8); +@@ -1049,6 +1080,7 @@ int ZEXPORT deflate (strm, flush) + /* Write the trailer */ + #ifdef GZIP + if (s->wrap == 2) { ++ crc_finalize(s); + put_byte(s, (Byte)(strm->adler & 0xff)); + put_byte(s, (Byte)((strm->adler >> 8) & 0xff)); + put_byte(s, (Byte)((strm->adler >> 16) & 0xff)); +@@ -1161,7 +1193,7 @@ int ZEXPORT deflateCopy (dest, source) + * allocating a large strm->next_in buffer and copying from it. + * (See also flush_pending()). + */ +-local unsigned read_buf(strm, buf, size) ++ZLIB_INTERNAL unsigned deflate_read_buf(strm, buf, size) + z_streamp strm; + Bytef *buf; + unsigned size; +@@ -1173,15 +1205,16 @@ local unsigned read_buf(strm, buf, size) + + strm->avail_in -= len; + +- zmemcpy(buf, strm->next_in, len); +- if (strm->state->wrap == 1) { +- strm->adler = adler32(strm->adler, buf, len); +- } + #ifdef GZIP +- else if (strm->state->wrap == 2) { +- strm->adler = crc32(strm->adler, buf, len); +- } ++ if (strm->state->wrap == 2) ++ copy_with_crc(strm, buf, len); ++ else + #endif ++ { ++ zmemcpy(buf, strm->next_in, len); ++ if (strm->state->wrap == 1) ++ strm->adler = adler32(strm->adler, buf, len); ++ } + strm->next_in += len; + strm->total_in += len; + +@@ -1479,7 +1512,19 @@ local void check_match(s, start, match, length) + * performed for at least two bytes (required for the zip translate_eol + * option -- not supported here). + */ +-local void fill_window(s) ++local void fill_window_c(deflate_state *s); ++ ++local void fill_window(deflate_state *s) ++{ ++ if (x86_cpu_enable_simd) { ++ fill_window_sse(s); ++ return; ++ } ++ ++ fill_window_c(s); ++} ++ ++local void fill_window_c(s) + deflate_state *s; + { + unsigned n; +@@ -1847,7 +1892,7 @@ local block_state deflate_fast(s, flush) + */ + hash_head = NIL; + if (s->lookahead >= MIN_MATCH) { +- INSERT_STRING(s, s->strstart, hash_head); ++ hash_head = insert_string(s, s->strstart); + } + + /* Find the longest match, discarding those <= prev_length. +@@ -1878,7 +1923,7 @@ local block_state deflate_fast(s, flush) + s->match_length--; /* string at strstart already in table */ + do { + s->strstart++; +- INSERT_STRING(s, s->strstart, hash_head); ++ hash_head = insert_string(s, s->strstart); + /* strstart never exceeds WSIZE-MAX_MATCH, so there are + * always MIN_MATCH bytes ahead. + */ +@@ -1950,7 +1995,7 @@ local block_state deflate_slow(s, flush) + */ + hash_head = NIL; + if (s->lookahead >= MIN_MATCH) { +- INSERT_STRING(s, s->strstart, hash_head); ++ hash_head = insert_string(s, s->strstart); + } + + /* Find the longest match, discarding those <= prev_length. +@@ -2001,7 +2046,7 @@ local block_state deflate_slow(s, flush) + s->prev_length -= 2; + do { + if (++s->strstart <= max_insert) { +- INSERT_STRING(s, s->strstart, hash_head); ++ hash_head = insert_string(s, s->strstart); + } + } while (--s->prev_length != 0); + s->match_available = 0; +@@ -2161,3 +2206,37 @@ local block_state deflate_huff(s, flush) + FLUSH_BLOCK(s, 0); + return block_done; + } ++ ++/* Safe to inline this as GCC/clang will use inline asm and Visual Studio will ++ * use intrinsic without extra params ++ */ ++local INLINE Pos insert_string_sse(deflate_state *const s, const Pos str) ++{ ++ Pos ret; ++ unsigned *ip, val, h = 0; ++ ++ ip = (unsigned *)&s->window[str]; ++ val = *ip; ++ ++ if (s->level >= 6) ++ val &= 0xFFFFFF; ++ ++/* Windows clang should use inline asm */ ++#if defined(_MSC_VER) && !defined(__clang__) ++ h = _mm_crc32_u32(h, val); ++#elif defined(__i386__) || defined(__amd64__) ++ __asm__ __volatile__ ( ++ "crc32 %1,%0\n\t" ++ : "+r" (h) ++ : "r" (val) ++ ); ++#else ++ /* This should never happen */ ++ assert(0); ++#endif ++ ++ ret = s->head[h & s->hash_mask]; ++ s->head[h & s->hash_mask] = str; ++ s->prev[str & s->w_mask] = ret; ++ return ret; ++} +diff --git a/deflate.h b/deflate.h +index 23ecdd312bc0..ab56df7663b6 100644 +--- a/deflate.h ++++ b/deflate.h +@@ -109,7 +109,7 @@ typedef struct internal_state { + ulg gzindex; /* where in extra, name, or comment */ + Byte method; /* can only be DEFLATED */ + int last_flush; /* value of flush param for previous deflate call */ +- ++ unsigned zalign(16) crc0[4 * 5]; + /* used by deflate.c: */ + + uInt w_size; /* LZ77 window size (32K by default) */ +@@ -346,4 +346,14 @@ void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf, + flush = _tr_tally(s, distance, length) + #endif + ++/* Functions that are SIMD optimised on x86 */ ++void ZLIB_INTERNAL crc_fold_init(deflate_state* const s); ++void ZLIB_INTERNAL crc_fold_copy(deflate_state* const s, ++ unsigned char* dst, ++ const unsigned char* src, ++ long len); ++unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state* const s); ++ ++void ZLIB_INTERNAL fill_window_sse(deflate_state* s); ++ + #endif /* DEFLATE_H */ +diff --git a/fill_window_sse.c b/fill_window_sse.c +new file mode 100644 +index 000000000000..949ccce1ba9c +--- /dev/null ++++ b/fill_window_sse.c +@@ -0,0 +1,177 @@ ++/* ++ * Fill Window with SSE2-optimized hash shifting ++ * ++ * Copyright (C) 2013 Intel Corporation ++ * Authors: ++ * Arjan van de Ven ++ * Jim Kukunas ++ * ++ * For conditions of distribution and use, see copyright notice in zlib.h ++ */ ++ ++#include ++#include "deflate.h" ++ ++#define UPDATE_HASH(s,h,i) \ ++ {\ ++ if (s->level < 6) { \ ++ h = (3483 * (s->window[i]) +\ ++ 23081* (s->window[i+1]) +\ ++ 6954 * (s->window[i+2]) +\ ++ 20947* (s->window[i+3])) & s->hash_mask;\ ++ } else {\ ++ h = (25881* (s->window[i]) +\ ++ 24674* (s->window[i+1]) +\ ++ 25811* (s->window[i+2])) & s->hash_mask;\ ++ }\ ++ }\ ++ ++extern int deflate_read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); ++ ++void fill_window_sse(deflate_state *s) ++{ ++ const __m128i xmm_wsize = _mm_set1_epi16(s->w_size); ++ ++ register unsigned n; ++ register Posf *p; ++ unsigned more; /* Amount of free space at the end of the window. */ ++ uInt wsize = s->w_size; ++ ++ Assert(s->lookahead < MIN_LOOKAHEAD, "already enough lookahead"); ++ ++ do { ++ more = (unsigned)(s->window_size -(ulg)s->lookahead -(ulg)s->strstart); ++ ++ /* Deal with !@#$% 64K limit: */ ++ if (sizeof(int) <= 2) { ++ if (more == 0 && s->strstart == 0 && s->lookahead == 0) { ++ more = wsize; ++ ++ } else if (more == (unsigned)(-1)) { ++ /* Very unlikely, but possible on 16 bit machine if ++ * strstart == 0 && lookahead == 1 (input done a byte at time) ++ */ ++ more--; ++ } ++ } ++ ++ /* If the window is almost full and there is insufficient lookahead, ++ * move the upper half to the lower one to make room in the upper half. ++ */ ++ if (s->strstart >= wsize+MAX_DIST(s)) { ++ ++ zmemcpy(s->window, s->window+wsize, (unsigned)wsize); ++ s->match_start -= wsize; ++ s->strstart -= wsize; /* we now have strstart >= MAX_DIST */ ++ s->block_start -= (long) wsize; ++ ++ /* Slide the hash table (could be avoided with 32 bit values ++ at the expense of memory usage). We slide even when level == 0 ++ to keep the hash table consistent if we switch back to level > 0 ++ later. (Using level 0 permanently is not an optimal usage of ++ zlib, so we don't care about this pathological case.) ++ */ ++ n = s->hash_size; ++ p = &s->head[n]; ++ p -= 8; ++ do { ++ __m128i value, result; ++ ++ value = _mm_loadu_si128((__m128i *)p); ++ result = _mm_subs_epu16(value, xmm_wsize); ++ _mm_storeu_si128((__m128i *)p, result); ++ ++ p -= 8; ++ n -= 8; ++ } while (n > 0); ++ ++ n = wsize; ++#ifndef FASTEST ++ p = &s->prev[n]; ++ p -= 8; ++ do { ++ __m128i value, result; ++ ++ value = _mm_loadu_si128((__m128i *)p); ++ result = _mm_subs_epu16(value, xmm_wsize); ++ _mm_storeu_si128((__m128i *)p, result); ++ ++ p -= 8; ++ n -= 8; ++ } while (n > 0); ++#endif ++ more += wsize; ++ } ++ if (s->strm->avail_in == 0) break; ++ ++ /* If there was no sliding: ++ * strstart <= WSIZE+MAX_DIST-1 && lookahead <= MIN_LOOKAHEAD - 1 && ++ * more == window_size - lookahead - strstart ++ * => more >= window_size - (MIN_LOOKAHEAD-1 + WSIZE + MAX_DIST-1) ++ * => more >= window_size - 2*WSIZE + 2 ++ * In the BIG_MEM or MMAP case (not yet supported), ++ * window_size == input_size + MIN_LOOKAHEAD && ++ * strstart + s->lookahead <= input_size => more >= MIN_LOOKAHEAD. ++ * Otherwise, window_size == 2*WSIZE so more >= 2. ++ * If there was sliding, more >= WSIZE. So in all cases, more >= 2. ++ */ ++ Assert(more >= 2, "more < 2"); ++ ++ n = deflate_read_buf(s->strm, ++ s->window + s->strstart + s->lookahead, ++ more); ++ s->lookahead += n; ++ ++ /* Initialize the hash value now that we have some input: */ ++ if (s->lookahead >= MIN_MATCH) { ++ uInt str = s->strstart; ++ s->ins_h = s->window[str]; ++ if (str >= 1) ++ UPDATE_HASH(s, s->ins_h, str + 1 - (MIN_MATCH-1)); ++#if MIN_MATCH != 3 ++ Call UPDATE_HASH() MIN_MATCH-3 more times ++#endif ++ } ++ /* If the whole input has less than MIN_MATCH bytes, ins_h is garbage, ++ * but this is not important since only literal bytes will be emitted. ++ */ ++ ++ } while (s->lookahead < MIN_LOOKAHEAD && s->strm->avail_in != 0); ++ ++ /* If the WIN_INIT bytes after the end of the current data have never been ++ * written, then zero those bytes in order to avoid memory check reports of ++ * the use of uninitialized (or uninitialised as Julian writes) bytes by ++ * the longest match routines. Update the high water mark for the next ++ * time through here. WIN_INIT is set to MAX_MATCH since the longest match ++ * routines allow scanning to strstart + MAX_MATCH, ignoring lookahead. ++ */ ++ if (s->high_water < s->window_size) { ++ ulg curr = s->strstart + (ulg)(s->lookahead); ++ ulg init; ++ ++ if (s->high_water < curr) { ++ /* Previous high water mark below current data -- zero WIN_INIT ++ * bytes or up to end of window, whichever is less. ++ */ ++ init = s->window_size - curr; ++ if (init > WIN_INIT) ++ init = WIN_INIT; ++ zmemzero(s->window + curr, (unsigned)init); ++ s->high_water = curr + init; ++ } ++ else if (s->high_water < (ulg)curr + WIN_INIT) { ++ /* High water mark at or above current data, but below current data ++ * plus WIN_INIT -- zero out to current data plus WIN_INIT, or up ++ * to end of window, whichever is less. ++ */ ++ init = (ulg)curr + WIN_INIT - s->high_water; ++ if (init > s->window_size - s->high_water) ++ init = s->window_size - s->high_water; ++ zmemzero(s->window + s->high_water, (unsigned)init); ++ s->high_water += init; ++ } ++ } ++ ++ Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD, ++ "not enough room for search"); ++} +diff --git a/simd_stub.c b/simd_stub.c +new file mode 100644 +index 000000000000..c6d46051498f +--- /dev/null ++++ b/simd_stub.c +@@ -0,0 +1,35 @@ ++/* simd_stub.c -- stub implementations ++* Copyright (C) 2014 Intel Corporation ++* For conditions of distribution and use, see copyright notice in zlib.h ++*/ ++#include ++ ++#include "deflate.h" ++#include "x86.h" ++ ++int ZLIB_INTERNAL x86_cpu_enable_simd = 0; ++ ++void ZLIB_INTERNAL crc_fold_init(deflate_state *const s) { ++ assert(0); ++} ++ ++void ZLIB_INTERNAL crc_fold_copy(deflate_state *const s, ++ unsigned char *dst, ++ const unsigned char *src, ++ long len) { ++ assert(0); ++} ++ ++unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state *const s) { ++ assert(0); ++ return 0; ++} ++ ++void ZLIB_INTERNAL fill_window_sse(deflate_state *s) ++{ ++ assert(0); ++} ++ ++void x86_check_features(void) ++{ ++} +diff --git a/x86.c b/x86.c +new file mode 100644 +index 000000000000..e56fe8b85a39 +--- /dev/null ++++ b/x86.c +@@ -0,0 +1,92 @@ ++/* ++ * x86 feature check ++ * ++ * Copyright (C) 2013 Intel Corporation. All rights reserved. ++ * Author: ++ * Jim Kukunas ++ * ++ * For conditions of distribution and use, see copyright notice in zlib.h ++ */ ++ ++#include "x86.h" ++#include "zutil.h" ++ ++int ZLIB_INTERNAL x86_cpu_enable_simd = 0; ++ ++#ifndef _MSC_VER ++#include ++ ++pthread_once_t cpu_check_inited_once = PTHREAD_ONCE_INIT; ++static void _x86_check_features(void); ++ ++void x86_check_features(void) ++{ ++ pthread_once(&cpu_check_inited_once, _x86_check_features); ++} ++ ++static void _x86_check_features(void) ++{ ++ int x86_cpu_has_sse2; ++ int x86_cpu_has_sse42; ++ int x86_cpu_has_pclmulqdq; ++ unsigned eax, ebx, ecx, edx; ++ ++ eax = 1; ++#ifdef __i386__ ++ __asm__ __volatile__ ( ++ "xchg %%ebx, %1\n\t" ++ "cpuid\n\t" ++ "xchg %1, %%ebx\n\t" ++ : "+a" (eax), "=S" (ebx), "=c" (ecx), "=d" (edx) ++ ); ++#else ++ __asm__ __volatile__ ( ++ "cpuid\n\t" ++ : "+a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx) ++ ); ++#endif /* (__i386__) */ ++ ++ x86_cpu_has_sse2 = edx & 0x4000000; ++ x86_cpu_has_sse42 = ecx & 0x100000; ++ x86_cpu_has_pclmulqdq = ecx & 0x2; ++ ++ x86_cpu_enable_simd = x86_cpu_has_sse2 && ++ x86_cpu_has_sse42 && ++ x86_cpu_has_pclmulqdq; ++} ++#else ++#include ++#include ++ ++static BOOL CALLBACK _x86_check_features(PINIT_ONCE once, ++ PVOID param, ++ PVOID *context); ++static INIT_ONCE cpu_check_inited_once = INIT_ONCE_STATIC_INIT; ++ ++void x86_check_features(void) ++{ ++ InitOnceExecuteOnce(&cpu_check_inited_once, _x86_check_features, ++ NULL, NULL); ++} ++ ++static BOOL CALLBACK _x86_check_features(PINIT_ONCE once, ++ PVOID param, ++ PVOID *context) ++{ ++ int x86_cpu_has_sse2; ++ int x86_cpu_has_sse42; ++ int x86_cpu_has_pclmulqdq; ++ int regs[4]; ++ ++ __cpuid(regs, 1); ++ ++ x86_cpu_has_sse2 = regs[3] & 0x4000000; ++ x86_cpu_has_sse42= regs[2] & 0x100000; ++ x86_cpu_has_pclmulqdq = regs[2] & 0x2; ++ ++ x86_cpu_enable_simd = x86_cpu_has_sse2 && ++ x86_cpu_has_sse42 && ++ x86_cpu_has_pclmulqdq; ++ return TRUE; ++} ++#endif /* _MSC_VER */ +diff --git a/x86.h b/x86.h +new file mode 100644 +index 000000000000..ebcf10ab09d2 +--- /dev/null ++++ b/x86.h +@@ -0,0 +1,15 @@ ++/* x86.h -- check for x86 CPU features ++* Copyright (C) 2013 Intel Corporation Jim Kukunas ++* For conditions of distribution and use, see copyright notice in zlib.h ++*/ ++ ++#ifndef X86_H ++#define X86_H ++ ++#include "zlib.h" ++ ++extern int x86_cpu_enable_simd; ++ ++void x86_check_features(void); ++ ++#endif /* X86_H */ +diff --git a/zutil.h b/zutil.h +index 80375b8b6109..4425bcf75eb3 100644 +--- a/zutil.h ++++ b/zutil.h +@@ -283,4 +283,10 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ + #define ZSWAP32(q) ((((q) >> 24) & 0xff) + (((q) >> 8) & 0xff00) + \ + (((q) & 0xff00) << 8) + (((q) & 0xff) << 24)) + ++#ifdef _MSC_VER ++#define zalign(x) __declspec(align(x)) ++#else ++#define zalign(x) __attribute__((aligned((x)))) ++#endif ++ + #endif /* ZUTIL_H */ diff --git a/third_party/zlib/patches/0002-uninitializedcheck.patch b/third_party/zlib/patches/0002-uninitializedcheck.patch new file mode 100644 index 0000000000..7c497839d3 --- /dev/null +++ b/third_party/zlib/patches/0002-uninitializedcheck.patch @@ -0,0 +1,27 @@ +From 1a03e7aec95d89c659fd91f195b99893b6458cd7 Mon Sep 17 00:00:00 2001 +From: Adenilson Cavalcanti +Date: Wed, 27 Sep 2017 13:59:04 -0700 +Subject: [PATCH] Zlib patch: prevent uninitialized use of state->check + +No need to call the Adler32 checksum function, just set +the struct field to the expected value. + +Upstream bug: madler/zlib#245 +--- + third_party/zlib/inflate.c | 2 +- + 1 files changed, 1 insertion(+), 1 deletions(-) + delete mode 100644 third_party/zlib/patches/0002-uninitializedcheck.patch + +diff --git a/third_party/zlib/inflate.c b/third_party/zlib/inflate.c +index bec9497..5c40cf1 100644 +--- a/third_party/zlib/inflate.c ++++ b/third_party/zlib/inflate.c +@@ -228,7 +228,7 @@ int stream_size; + state->strm = strm; + state->window = Z_NULL; + state->mode = HEAD; /* to pass state test in inflateReset2() */ +- state->check = adler32(0L, Z_NULL, 0); ++ state->check = 1L; /* 1L is the result of adler32() zero length data */ + ret = inflateReset2(strm, windowBits); + if (ret != Z_OK) { + ZFREE(strm, state); diff --git a/third_party/zlib/patches/0003-uninitializedjump.patch b/third_party/zlib/patches/0003-uninitializedjump.patch new file mode 100644 index 0000000000..7aae3238a5 --- /dev/null +++ b/third_party/zlib/patches/0003-uninitializedjump.patch @@ -0,0 +1,15 @@ +diff --git a/third_party/zlib/deflate.c b/third_party/zlib/deflate.c +index a39e62787862..c6053fd1c7ea 100644 +--- a/third_party/zlib/deflate.c ++++ b/third_party/zlib/deflate.c +@@ -318,6 +318,10 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, + s->w_size + window_padding, + 2*sizeof(Byte)); + s->prev = (Posf *) ZALLOC(strm, s->w_size, sizeof(Pos)); ++ /* Avoid use of uninitialized value, see: ++ * https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=11360 ++ */ ++ zmemzero(s->prev, s->w_size * sizeof(Pos)); + s->head = (Posf *) ZALLOC(strm, s->hash_size, sizeof(Pos)); + + s->high_water = 0; /* nothing written to s->window yet */ diff --git a/third_party/zlib/patches/0004-fix-uwp.patch b/third_party/zlib/patches/0004-fix-uwp.patch new file mode 100644 index 0000000000..23145a7ae5 --- /dev/null +++ b/third_party/zlib/patches/0004-fix-uwp.patch @@ -0,0 +1,22 @@ +diff --git a/third_party/zlib/contrib/minizip/iowin32.c b/third_party/zlib/contrib/minizip/iowin32.c +index 246ceb91a139..c6bc314b3c28 100644 +--- a/third_party/zlib/contrib/minizip/iowin32.c ++++ b/third_party/zlib/contrib/minizip/iowin32.c +@@ -31,14 +31,12 @@ + #define _WIN32_WINNT 0x601 + #endif + +-#if _WIN32_WINNT >= _WIN32_WINNT_WIN8 +-// see Include/shared/winapifamily.h in the Windows Kit +-#if defined(WINAPI_FAMILY_PARTITION) && (!(defined(IOWIN32_USING_WINRT_API))) +-#if WINAPI_FAMILY_ONE_PARTITION(WINAPI_FAMILY, WINAPI_PARTITION_APP) ++#if !defined(IOWIN32_USING_WINRT_API) ++#if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_APP) ++// Windows Store or Universal Windows Platform + #define IOWIN32_USING_WINRT_API 1 + #endif + #endif +-#endif + + voidpf ZCALLBACK win32_open_file_func OF((voidpf opaque, const char* filename, int mode)); + uLong ZCALLBACK win32_read_file_func OF((voidpf opaque, voidpf stream, void* buf, uLong size)); diff --git a/third_party/zlib/patches/0005-infcover-gtest.patch b/third_party/zlib/patches/0005-infcover-gtest.patch new file mode 100644 index 0000000000..f5443bd7d8 --- /dev/null +++ b/third_party/zlib/patches/0005-infcover-gtest.patch @@ -0,0 +1,405 @@ +From 409594639f15d825202971db7a275023e05772ff Mon Sep 17 00:00:00 2001 +From: Adenilson Cavalcanti +Date: Tue, 28 Apr 2020 10:48:01 -0700 +Subject: [PATCH] Local Changes: - make C tests build as C++ code so we can + use gtest. - use gtest EXPECT_TRUE instead of C assert. - replace C + streams for C++ (portability issues). + +--- + test/infcover.c | 167 ++++++++++++++++++++++++++---------------------- + 1 file changed, 90 insertions(+), 77 deletions(-) + +diff --git a/test/infcover.c b/test/infcover.c +index 2be0164..a8c51c7 100644 +--- a/test/infcover.c ++++ b/test/infcover.c +@@ -4,11 +4,12 @@ + */ + + /* to use, do: ./configure --cover && make cover */ +- ++// clang-format off ++#include "infcover.h" + #include + #include + #include +-#include ++ + #include "zlib.h" + + /* get definition of internal structure so we can mess with it (see pull()), +@@ -17,8 +18,22 @@ + #include "inftrees.h" + #include "inflate.h" + ++/* XXX: use C++ streams instead of printf/fputs/etc due to portability ++ * as type sizes can vary between platforms. ++ */ ++#include + #define local static + ++/* XXX: hacking C assert and plugging into GTest. */ ++#include "gtest.h" ++#if defined(assert) ++#undef assert ++#define assert EXPECT_TRUE ++#endif ++ ++/* XXX: handle what is a reserved word in C++. */ ++#define try try_f ++ + /* -- memory tracking routines -- */ + + /* +@@ -72,7 +87,7 @@ local void *mem_alloc(void *mem, unsigned count, unsigned size) + { + void *ptr; + struct mem_item *item; +- struct mem_zone *zone = mem; ++ struct mem_zone *zone = static_cast(mem); + size_t len = count * (size_t)size; + + /* induced allocation failure */ +@@ -87,7 +102,7 @@ local void *mem_alloc(void *mem, unsigned count, unsigned size) + memset(ptr, 0xa5, len); + + /* create a new item for the list */ +- item = malloc(sizeof(struct mem_item)); ++ item = static_cast(malloc(sizeof(struct mem_item))); + if (item == NULL) { + free(ptr); + return NULL; +@@ -112,7 +127,7 @@ local void *mem_alloc(void *mem, unsigned count, unsigned size) + local void mem_free(void *mem, void *ptr) + { + struct mem_item *item, *next; +- struct mem_zone *zone = mem; ++ struct mem_zone *zone = static_cast(mem); + + /* if no zone, just do a free */ + if (zone == NULL) { +@@ -159,7 +174,7 @@ local void mem_setup(z_stream *strm) + { + struct mem_zone *zone; + +- zone = malloc(sizeof(struct mem_zone)); ++ zone = static_cast(malloc(sizeof(struct mem_zone))); + assert(zone != NULL); + zone->first = NULL; + zone->total = 0; +@@ -175,33 +190,33 @@ local void mem_setup(z_stream *strm) + /* set a limit on the total memory allocation, or 0 to remove the limit */ + local void mem_limit(z_stream *strm, size_t limit) + { +- struct mem_zone *zone = strm->opaque; ++ struct mem_zone *zone = static_cast(strm->opaque); + + zone->limit = limit; + } + + /* show the current total requested allocations in bytes */ +-local void mem_used(z_stream *strm, char *prefix) ++local void mem_used(z_stream *strm, const char *prefix) + { +- struct mem_zone *zone = strm->opaque; ++ struct mem_zone *zone = static_cast(strm->opaque); + +- fprintf(stderr, "%s: %lu allocated\n", prefix, zone->total); ++ std::cout << prefix << ": " << zone->total << " allocated" << std::endl; + } + + /* show the high water allocation in bytes */ +-local void mem_high(z_stream *strm, char *prefix) ++local void mem_high(z_stream *strm, const char *prefix) + { +- struct mem_zone *zone = strm->opaque; ++ struct mem_zone *zone = static_cast(strm->opaque); + +- fprintf(stderr, "%s: %lu high water mark\n", prefix, zone->highwater); ++ std::cout << prefix << ": " << zone->highwater << " high water mark" << std::endl; + } + + /* release the memory allocation zone -- if there are any surprises, notify */ +-local void mem_done(z_stream *strm, char *prefix) ++local void mem_done(z_stream *strm, const char *prefix) + { + int count = 0; + struct mem_item *item, *next; +- struct mem_zone *zone = strm->opaque; ++ struct mem_zone *zone = static_cast(strm->opaque); + + /* show high water mark */ + mem_high(strm, prefix); +@@ -218,13 +233,20 @@ local void mem_done(z_stream *strm, char *prefix) + + /* issue alerts about anything unexpected */ + if (count || zone->total) +- fprintf(stderr, "** %s: %lu bytes in %d blocks not freed\n", +- prefix, zone->total, count); ++ std::cout << "** " << prefix << ": " ++ << zone->total << " bytes in " ++ << count << " blocks not freed" ++ << std::endl; ++ + if (zone->notlifo) +- fprintf(stderr, "** %s: %d frees not LIFO\n", prefix, zone->notlifo); ++ std::cout << "** " << prefix << ": " ++ << zone->notlifo << " frees not LIFO" ++ << std::endl; ++ + if (zone->rogue) +- fprintf(stderr, "** %s: %d frees not recognized\n", +- prefix, zone->rogue); ++ std::cout << "** " << prefix << ": " ++ << zone->rogue << " frees not recognized" ++ << std::endl; + + /* free the zone and delete from the stream */ + free(zone); +@@ -247,7 +269,7 @@ local unsigned char *h2b(const char *hex, unsigned *len) + unsigned char *in, *re; + unsigned next, val; + +- in = malloc((strlen(hex) + 1) >> 1); ++ in = static_cast(malloc((strlen(hex) + 1) >> 1)); + if (in == NULL) + return NULL; + next = 0; +@@ -268,7 +290,7 @@ local unsigned char *h2b(const char *hex, unsigned *len) + } while (*hex++); /* go through the loop with the terminating null */ + if (len != NULL) + *len = next; +- re = realloc(in, next); ++ re = static_cast(realloc(in, next)); + return re == NULL ? in : re; + } + +@@ -281,7 +303,7 @@ local unsigned char *h2b(const char *hex, unsigned *len) + header information is collected with inflateGetHeader(). If a zlib stream + is looking for a dictionary, then an empty dictionary is provided. + inflate() is run until all of the input data is consumed. */ +-local void inf(char *hex, char *what, unsigned step, int win, unsigned len, ++local void inf(const char *hex, const char *what, unsigned step, int win, unsigned len, + int err) + { + int ret; +@@ -298,7 +320,7 @@ local void inf(char *hex, char *what, unsigned step, int win, unsigned len, + mem_done(&strm, what); + return; + } +- out = malloc(len); assert(out != NULL); ++ out = static_cast(malloc(len)); assert(out != NULL); + if (win == 47) { + head.extra = out; + head.extra_max = len; +@@ -347,7 +369,7 @@ local void inf(char *hex, char *what, unsigned step, int win, unsigned len, + } + + /* cover all of the lines in inflate.c up to inflate() */ +-local void cover_support(void) ++void cover_support(void) + { + int ret; + z_stream strm; +@@ -381,11 +403,11 @@ local void cover_support(void) + strm.next_in = Z_NULL; + ret = inflateInit(&strm); assert(ret == Z_OK); + ret = inflateEnd(&strm); assert(ret == Z_OK); +- fputs("inflate built-in memory routines\n", stderr); ++ std::cout << "inflate built-in memory routines" << std::endl;; + } + + /* cover all inflate() header and trailer cases and code after inflate() */ +-local void cover_wrap(void) ++void cover_wrap(void) + { + int ret; + z_stream strm, copy; +@@ -394,7 +416,7 @@ local void cover_wrap(void) + ret = inflate(Z_NULL, 0); assert(ret == Z_STREAM_ERROR); + ret = inflateEnd(Z_NULL); assert(ret == Z_STREAM_ERROR); + ret = inflateCopy(Z_NULL, Z_NULL); assert(ret == Z_STREAM_ERROR); +- fputs("inflate bad parameters\n", stderr); ++ std::cout << "inflate bad parameters" << std::endl; + + inf("1f 8b 0 0", "bad gzip method", 0, 31, 0, Z_DATA_ERROR); + inf("1f 8b 8 80", "bad gzip flags", 0, 31, 0, Z_DATA_ERROR); +@@ -415,9 +437,9 @@ local void cover_wrap(void) + strm.next_in = Z_NULL; + ret = inflateInit2(&strm, -8); + strm.avail_in = 2; +- strm.next_in = (void *)"\x63"; ++ strm.next_in = (Bytef *)"\x63"; + strm.avail_out = 1; +- strm.next_out = (void *)&ret; ++ strm.next_out = (Bytef *)&ret; + mem_limit(&strm, 1); + ret = inflate(&strm, Z_NO_FLUSH); assert(ret == Z_MEM_ERROR); + ret = inflate(&strm, Z_NO_FLUSH); assert(ret == Z_MEM_ERROR); +@@ -428,11 +450,11 @@ local void cover_wrap(void) + mem_limit(&strm, (sizeof(struct inflate_state) << 1) + 256); + ret = inflatePrime(&strm, 16, 0); assert(ret == Z_OK); + strm.avail_in = 2; +- strm.next_in = (void *)"\x80"; ++ strm.next_in = (Bytef *)"\x80"; + ret = inflateSync(&strm); assert(ret == Z_DATA_ERROR); + ret = inflate(&strm, Z_NO_FLUSH); assert(ret == Z_STREAM_ERROR); + strm.avail_in = 4; +- strm.next_in = (void *)"\0\0\xff\xff"; ++ strm.next_in = (Bytef *)"\0\0\xff\xff"; + ret = inflateSync(&strm); assert(ret == Z_OK); + (void)inflateSyncPoint(&strm); + ret = inflateCopy(©, &strm); assert(ret == Z_MEM_ERROR); +@@ -454,7 +476,7 @@ local unsigned pull(void *desc, unsigned char **buf) + next = 0; + return 0; /* no input (already provided at next_in) */ + } +- state = (void *)((z_stream *)desc)->state; ++ state = reinterpret_cast(((z_stream *)desc)->state); + if (state != Z_NULL) + state->mode = SYNC; /* force an otherwise impossible situation */ + return next < sizeof(dat) ? (*buf = dat + next++, 1) : 0; +@@ -467,7 +489,7 @@ local int push(void *desc, unsigned char *buf, unsigned len) + } + + /* cover inflateBack() up to common deflate data cases and after those */ +-local void cover_back(void) ++void cover_back(void) + { + int ret; + z_stream strm; +@@ -479,17 +501,17 @@ local void cover_back(void) + ret = inflateBack(Z_NULL, Z_NULL, Z_NULL, Z_NULL, Z_NULL); + assert(ret == Z_STREAM_ERROR); + ret = inflateBackEnd(Z_NULL); assert(ret == Z_STREAM_ERROR); +- fputs("inflateBack bad parameters\n", stderr); ++ std::cout << "inflateBack bad parameters" << std::endl;; + + mem_setup(&strm); + ret = inflateBackInit(&strm, 15, win); assert(ret == Z_OK); + strm.avail_in = 2; +- strm.next_in = (void *)"\x03"; ++ strm.next_in = (Bytef *)"\x03"; + ret = inflateBack(&strm, pull, Z_NULL, push, Z_NULL); + assert(ret == Z_STREAM_END); + /* force output error */ + strm.avail_in = 3; +- strm.next_in = (void *)"\x63\x00"; ++ strm.next_in = (Bytef *)"\x63\x00"; + ret = inflateBack(&strm, pull, Z_NULL, push, &strm); + assert(ret == Z_BUF_ERROR); + /* force mode error by mucking with state */ +@@ -500,11 +522,11 @@ local void cover_back(void) + + ret = inflateBackInit(&strm, 15, win); assert(ret == Z_OK); + ret = inflateBackEnd(&strm); assert(ret == Z_OK); +- fputs("inflateBack built-in memory routines\n", stderr); ++ std::cout << "inflateBack built-in memory routines" << std::endl;; + } + + /* do a raw inflate of data in hexadecimal with both inflate and inflateBack */ +-local int try(char *hex, char *id, int err) ++local int try(const char *hex, const char *id, int err) + { + int ret; + unsigned len, size; +@@ -518,11 +540,11 @@ local int try(char *hex, char *id, int err) + + /* allocate work areas */ + size = len << 3; +- out = malloc(size); ++ out = static_cast(malloc(size)); + assert(out != NULL); +- win = malloc(32768); ++ win = static_cast(malloc(32768)); + assert(win != NULL); +- prefix = malloc(strlen(id) + 6); ++ prefix = static_cast(malloc(strlen(id) + 6)); + assert(prefix != NULL); + + /* first with inflate */ +@@ -578,7 +600,7 @@ local int try(char *hex, char *id, int err) + } + + /* cover deflate data cases in both inflate() and inflateBack() */ +-local void cover_inflate(void) ++void cover_inflate(void) + { + try("0 0 0 0 0", "invalid stored block lengths", 1); + try("3 0", "fixed", 0); +@@ -613,32 +635,33 @@ local void cover_inflate(void) + inf("63 18 5 40 c 0", "window wrap", 3, -8, 300, Z_OK); + } + ++/* XXX(cavalcantii): fix linking error due inflate_table. */ + /* cover remaining lines in inftrees.c */ +-local void cover_trees(void) +-{ +- int ret; +- unsigned bits; +- unsigned short lens[16], work[16]; +- code *next, table[ENOUGH_DISTS]; +- +- /* we need to call inflate_table() directly in order to manifest not- +- enough errors, since zlib insures that enough is always enough */ +- for (bits = 0; bits < 15; bits++) +- lens[bits] = (unsigned short)(bits + 1); +- lens[15] = 15; +- next = table; +- bits = 15; +- ret = inflate_table(DISTS, lens, 16, &next, &bits, work); +- assert(ret == 1); +- next = table; +- bits = 1; +- ret = inflate_table(DISTS, lens, 16, &next, &bits, work); +- assert(ret == 1); +- fputs("inflate_table not enough errors\n", stderr); +-} ++/* void cover_trees(void) */ ++/* { */ ++/* int ret; */ ++/* unsigned bits; */ ++/* unsigned short lens[16], work[16]; */ ++/* code *next, table[ENOUGH_DISTS]; */ ++ ++/* /\* we need to call inflate_table() directly in order to manifest not- */ ++/* enough errors, since zlib insures that enough is always enough *\/ */ ++/* for (bits = 0; bits < 15; bits++) */ ++/* lens[bits] = (unsigned short)(bits + 1); */ ++/* lens[15] = 15; */ ++/* next = table; */ ++/* bits = 15; */ ++/* ret = inflate_table(DISTS, lens, 16, &next, &bits, work); */ ++/* assert(ret == 1); */ ++/* next = table; */ ++/* bits = 1; */ ++/* ret = inflate_table(DISTS, lens, 16, &next, &bits, work); */ ++/* assert(ret == 1); */ ++/* fputs("inflate_table not enough errors\n", stderr); */ ++/* } */ + + /* cover remaining inffast.c decoding and window copying */ +-local void cover_fast(void) ++void cover_fast(void) + { + inf("e5 e0 81 ad 6d cb b2 2c c9 01 1e 59 63 ae 7d ee fb 4d fd b5 35 41 68" + " ff 7f 0f 0 0 0", "fast length extra bits", 0, -8, 258, Z_DATA_ERROR); +@@ -658,14 +681,4 @@ local void cover_fast(void) + Z_STREAM_END); + } + +-int main(void) +-{ +- fprintf(stderr, "%s\n", zlibVersion()); +- cover_support(); +- cover_wrap(); +- cover_back(); +- cover_inflate(); +- cover_trees(); +- cover_fast(); +- return 0; +-} ++// clang-format on +-- +2.21.1 (Apple Git-122.3) + diff --git a/third_party/zlib/patches/README b/third_party/zlib/patches/README new file mode 100644 index 0000000000..b21d309ff9 --- /dev/null +++ b/third_party/zlib/patches/README @@ -0,0 +1,43 @@ +== Patches applied on top of zlib == + + - 0000-build.patch: changes from the upstream version, mostly related to the + build. + - 0001-simd.patch: integrate Intel SIMD optimizations from + https://github.com/jtkukunas/zlib/ + - 0002-uninitializedcheck.patch: prevent uninitialized use of state->check + +== Procedure to create a patch file == + + Assuming you are working in a new feature branch: + - git format-patch master --stdout > foo.patch # where naming follows a growing + # number plus patch description. + - git add foo.patch + - git commit -a -m "Local patch." + - git rebase -i HEAD~2 # Squashing the second commit + + As patches created in this way will feature a ChangeLog, there is no longer +the need to append this file with a description of what the patch does. This +should help to solve frequent conflicts in pending new patches on +Chromium's zlib. + + The plan for the near future is to better insulate the platform specific +changes to ease update adoption with new releases of zlib. This insulation +happens by making changes inside contrib/ rather than the root directory +(where conflicts can happen). + + If a change modifies enough things inside the root directory that the +intention is not immediately clear, generate a .patch file to go with your +change. If the change's modifications in the root directory are small, like: + +#ifdef FEATURE_FLAG +use_special_feature(); +#elif +use_default_behavior(); +#endif + + then the intent is clear and a .patch file doesn't need to be generated (since +it would not provide much value). + + Ideally local changes should have a merge request featured in either: + - canonical zlib: https://github.com/madler/zlib/ + - zlib-ng: https://github.com/Dead2/zlib-ng diff --git a/third_party/zlib/simd_stub.c b/third_party/zlib/simd_stub.c deleted file mode 100644 index 796f1f63f6..0000000000 --- a/third_party/zlib/simd_stub.c +++ /dev/null @@ -1,35 +0,0 @@ -/* simd_stub.c -- stub implementations -* Copyright (C) 2014 Intel Corporation -* For conditions of distribution and use, see copyright notice in zlib.h -*/ -#include - -#include "deflate.h" -#include "x86.h" - -int x86_cpu_enable_simd = 0; - -void ZLIB_INTERNAL crc_fold_init(deflate_state *const s) { - assert(0); -} - -void ZLIB_INTERNAL crc_fold_copy(deflate_state *const s, - unsigned char *dst, - const unsigned char *src, - long len) { - assert(0); -} - -unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state *const s) { - assert(0); - return 0; -} - -void ZLIB_INTERNAL fill_window_sse(deflate_state *s) -{ - assert(0); -} - -void x86_check_features(void) -{ -} diff --git a/third_party/zlib/trees.c b/third_party/zlib/trees.c index 56e9bb1c11..5f89d056ef 100644 --- a/third_party/zlib/trees.c +++ b/third_party/zlib/trees.c @@ -1,5 +1,5 @@ /* trees.c -- output deflated data using Huffman coding - * Copyright (C) 1995-2010 Jean-loup Gailly + * Copyright (C) 1995-2017 Jean-loup Gailly * detect_data_type() function provided freely by Cosmin Truta, 2006 * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -36,7 +36,7 @@ #include "deflate.h" -#ifdef DEBUG +#ifdef ZLIB_DEBUG # include #endif @@ -74,11 +74,6 @@ local const uch bl_order[BL_CODES] * probability, to avoid transmitting the lengths for unused bit length codes. */ -#define Buf_size (8 * 2*sizeof(char)) -/* Number of bits used within bi_buf. (bi_buf might be implemented on - * more than 16 bits on some systems.) - */ - /* =========================================================================== * Local data. These are initialized only once. */ @@ -127,13 +122,13 @@ struct static_tree_desc_s { int max_length; /* max bit length for the codes */ }; -local static_tree_desc static_l_desc = +local const static_tree_desc static_l_desc = {static_ltree, extra_lbits, LITERALS+1, L_CODES, MAX_BITS}; -local static_tree_desc static_d_desc = +local const static_tree_desc static_d_desc = {static_dtree, extra_dbits, 0, D_CODES, MAX_BITS}; -local static_tree_desc static_bl_desc = +local const static_tree_desc static_bl_desc = {(const ct_data *)0, extra_blbits, 0, BL_CODES, MAX_BL_BITS}; /* =========================================================================== @@ -151,24 +146,22 @@ local void send_tree OF((deflate_state *s, ct_data *tree, int max_code)); local int build_bl_tree OF((deflate_state *s)); local void send_all_trees OF((deflate_state *s, int lcodes, int dcodes, int blcodes)); -local void compress_block OF((deflate_state *s, ct_data *ltree, - ct_data *dtree)); +local void compress_block OF((deflate_state *s, const ct_data *ltree, + const ct_data *dtree)); local int detect_data_type OF((deflate_state *s)); local unsigned bi_reverse OF((unsigned value, int length)); local void bi_windup OF((deflate_state *s)); local void bi_flush OF((deflate_state *s)); -local void copy_block OF((deflate_state *s, charf *buf, unsigned len, - int header)); #ifdef GEN_TREES_H local void gen_trees_header OF((void)); #endif -#ifndef DEBUG +#ifndef ZLIB_DEBUG # define send_code(s, c, tree) send_bits(s, tree[c].Code, tree[c].Len) /* Send a code of the given tree. c and tree must not have side effects */ -#else /* DEBUG */ +#else /* !ZLIB_DEBUG */ # define send_code(s, c, tree) \ { if (z_verbose>2) fprintf(stderr,"\ncd %3d ",(c)); \ send_bits(s, tree[c].Code, tree[c].Len); } @@ -187,7 +180,7 @@ local void gen_trees_header OF((void)); * Send a value on a given number of bits. * IN assertion: length <= 16 and value fits in length bits. */ -#ifdef DEBUG +#ifdef ZLIB_DEBUG local void send_bits OF((deflate_state *s, int value, int length)); local void send_bits(s, value, length) @@ -213,12 +206,12 @@ local void send_bits(s, value, length) s->bi_valid += length; } } -#else /* !DEBUG */ +#else /* !ZLIB_DEBUG */ #define send_bits(s, value, length) \ { int len = length;\ if (s->bi_valid > (int)Buf_size - len) {\ - int val = value;\ + int val = (int)value;\ s->bi_buf |= (ush)val << s->bi_valid;\ put_short(s, s->bi_buf);\ s->bi_buf = (ush)val >> (Buf_size - s->bi_valid);\ @@ -228,7 +221,7 @@ local void send_bits(s, value, length) s->bi_valid += len;\ }\ } -#endif /* DEBUG */ +#endif /* ZLIB_DEBUG */ /* the arguments must not have side effects */ @@ -322,7 +315,7 @@ local void tr_static_init() * Genererate the file trees.h describing the static trees. */ #ifdef GEN_TREES_H -# ifndef DEBUG +# ifndef ZLIB_DEBUG # include # endif @@ -399,8 +392,7 @@ void ZLIB_INTERNAL _tr_init(s) s->bi_buf = 0; s->bi_valid = 0; - s->last_eob_len = 8; /* enough lookahead for inflate */ -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->compressed_len = 0L; s->bits_sent = 0L; #endif @@ -424,7 +416,7 @@ local void init_block(s) s->dyn_ltree[END_BLOCK].Freq = 1; s->opt_len = s->static_len = 0L; - s->last_lit = s->matches = 0; + s->sym_next = s->matches = 0; } #define SMALLEST 1 @@ -528,12 +520,12 @@ local void gen_bitlen(s, desc) xbits = 0; if (n >= base) xbits = extra[n-base]; f = tree[n].Freq; - s->opt_len += (ulg)f * (bits + xbits); - if (stree) s->static_len += (ulg)f * (stree[n].Len + xbits); + s->opt_len += (ulg)f * (unsigned)(bits + xbits); + if (stree) s->static_len += (ulg)f * (unsigned)(stree[n].Len + xbits); } if (overflow == 0) return; - Trace((stderr,"\nbit length overflow\n")); + Tracev((stderr,"\nbit length overflow\n")); /* This happens for example on obj2 and pic of the Calgary corpus */ /* Find the first bit length which could increase: */ @@ -560,9 +552,8 @@ local void gen_bitlen(s, desc) m = s->heap[--h]; if (m > max_code) continue; if ((unsigned) tree[m].Len != (unsigned) bits) { - Trace((stderr,"code %d bits %d->%d\n", m, tree[m].Len, bits)); - s->opt_len += ((long)bits - (long)tree[m].Len) - *(long)tree[m].Freq; + Tracev((stderr,"code %d bits %d->%d\n", m, tree[m].Len, bits)); + s->opt_len += ((ulg)bits - tree[m].Len) * tree[m].Freq; tree[m].Len = (ush)bits; } n--; @@ -584,7 +575,7 @@ local void gen_codes (tree, max_code, bl_count) ushf *bl_count; /* number of codes at each bit length */ { ush next_code[MAX_BITS+1]; /* next code value for each bit length */ - ush code = 0; /* running code value */ + unsigned code = 0; /* running code value */ int bits; /* bit index */ int n; /* code index */ @@ -592,7 +583,8 @@ local void gen_codes (tree, max_code, bl_count) * without bit reversal. */ for (bits = 1; bits <= MAX_BITS; bits++) { - next_code[bits] = code = (code + bl_count[bits-1]) << 1; + code = (code + bl_count[bits-1]) << 1; + next_code[bits] = (ush)code; } /* Check that the bit counts in bl_count are consistent. The last code * must be all ones. @@ -605,7 +597,7 @@ local void gen_codes (tree, max_code, bl_count) int len = tree[n].Len; if (len == 0) continue; /* Now reverse the bits */ - tree[n].Code = bi_reverse(next_code[len]++, len); + tree[n].Code = (ush)bi_reverse(next_code[len]++, len); Tracecv(tree != static_ltree, (stderr,"\nn %3d %c l %2d c %4x (%x) ", n, (isgraph(n) ? n : ' '), len, tree[n].Code, next_code[len]-1)); @@ -827,7 +819,7 @@ local int build_bl_tree(s) if (s->bl_tree[bl_order[max_blindex]].Len != 0) break; } /* Update opt_len to include the bit length tree and counts */ - s->opt_len += 3*(max_blindex+1) + 5+5+4; + s->opt_len += 3*((ulg)max_blindex+1) + 5+5+4; Tracev((stderr, "\ndyn trees: dyn %ld, stat %ld", s->opt_len, s->static_len)); @@ -875,52 +867,46 @@ void ZLIB_INTERNAL _tr_stored_block(s, buf, stored_len, last) int last; /* one if this is the last block for a file */ { send_bits(s, (STORED_BLOCK<<1)+last, 3); /* send block type */ -#ifdef DEBUG + bi_windup(s); /* align on byte boundary */ + put_short(s, (ush)stored_len); + put_short(s, (ush)~stored_len); + zmemcpy(s->pending_buf + s->pending, (Bytef *)buf, stored_len); + s->pending += stored_len; +#ifdef ZLIB_DEBUG s->compressed_len = (s->compressed_len + 3 + 7) & (ulg)~7L; s->compressed_len += (stored_len + 4) << 3; + s->bits_sent += 2*16; + s->bits_sent += stored_len<<3; #endif - copy_block(s, buf, (unsigned)stored_len, 1); /* with header */ +} + +/* =========================================================================== + * Flush the bits in the bit buffer to pending output (leaves at most 7 bits) + */ +void ZLIB_INTERNAL _tr_flush_bits(s) + deflate_state *s; +{ + bi_flush(s); } /* =========================================================================== * Send one empty static block to give enough lookahead for inflate. * This takes 10 bits, of which 7 may remain in the bit buffer. - * The current inflate code requires 9 bits of lookahead. If the - * last two codes for the previous block (real code plus EOB) were coded - * on 5 bits or less, inflate may have only 5+3 bits of lookahead to decode - * the last real code. In this case we send two empty static blocks instead - * of one. (There are no problems if the previous block is stored or fixed.) - * To simplify the code, we assume the worst case of last real code encoded - * on one bit only. */ void ZLIB_INTERNAL _tr_align(s) deflate_state *s; { send_bits(s, STATIC_TREES<<1, 3); send_code(s, END_BLOCK, static_ltree); -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->compressed_len += 10L; /* 3 for block type, 7 for EOB */ #endif bi_flush(s); - /* Of the 10 bits for the empty block, we have already sent - * (10 - bi_valid) bits. The lookahead for the last real code (before - * the EOB of the previous block) was thus at least one plus the length - * of the EOB plus what we have just sent of the empty static block. - */ - if (1 + s->last_eob_len + 10 - s->bi_valid < 9) { - send_bits(s, STATIC_TREES<<1, 3); - send_code(s, END_BLOCK, static_ltree); -#ifdef DEBUG - s->compressed_len += 10L; -#endif - bi_flush(s); - } - s->last_eob_len = 7; } /* =========================================================================== * Determine the best encoding for the current block: dynamic trees, static - * trees or store, and output the encoded block to the zip file. + * trees or store, and write out the encoded block. */ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) deflate_state *s; @@ -961,7 +947,7 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) Tracev((stderr, "\nopt %lu(%lu) stat %lu(%lu) stored %lu lit %u ", opt_lenb, s->opt_len, static_lenb, s->static_len, stored_len, - s->last_lit)); + s->sym_next / 3)); if (static_lenb <= opt_lenb) opt_lenb = static_lenb; @@ -990,16 +976,18 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) } else if (s->strategy == Z_FIXED || static_lenb == opt_lenb) { #endif send_bits(s, (STATIC_TREES<<1)+last, 3); - compress_block(s, (ct_data *)static_ltree, (ct_data *)static_dtree); -#ifdef DEBUG + compress_block(s, (const ct_data *)static_ltree, + (const ct_data *)static_dtree); +#ifdef ZLIB_DEBUG s->compressed_len += 3 + s->static_len; #endif } else { send_bits(s, (DYN_TREES<<1)+last, 3); send_all_trees(s, s->l_desc.max_code+1, s->d_desc.max_code+1, max_blindex+1); - compress_block(s, (ct_data *)s->dyn_ltree, (ct_data *)s->dyn_dtree); -#ifdef DEBUG + compress_block(s, (const ct_data *)s->dyn_ltree, + (const ct_data *)s->dyn_dtree); +#ifdef ZLIB_DEBUG s->compressed_len += 3 + s->opt_len; #endif } @@ -1011,7 +999,7 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) if (last) { bi_windup(s); -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->compressed_len += 7; /* align on byte boundary */ #endif } @@ -1028,8 +1016,9 @@ int ZLIB_INTERNAL _tr_tally (s, dist, lc) unsigned dist; /* distance of matched string */ unsigned lc; /* match length-MIN_MATCH or unmatched char (if dist==0) */ { - s->d_buf[s->last_lit] = (ush)dist; - s->l_buf[s->last_lit++] = (uch)lc; + s->sym_buf[s->sym_next++] = dist; + s->sym_buf[s->sym_next++] = dist >> 8; + s->sym_buf[s->sym_next++] = lc; if (dist == 0) { /* lc is the unmatched char */ s->dyn_ltree[lc].Freq++; @@ -1044,30 +1033,7 @@ int ZLIB_INTERNAL _tr_tally (s, dist, lc) s->dyn_ltree[_length_code[lc]+LITERALS+1].Freq++; s->dyn_dtree[d_code(dist)].Freq++; } - -#ifdef TRUNCATE_BLOCK - /* Try to guess if it is profitable to stop the current block here */ - if ((s->last_lit & 0x1fff) == 0 && s->level > 2) { - /* Compute an upper bound for the compressed length */ - ulg out_length = (ulg)s->last_lit*8L; - ulg in_length = (ulg)((long)s->strstart - s->block_start); - int dcode; - for (dcode = 0; dcode < D_CODES; dcode++) { - out_length += (ulg)s->dyn_dtree[dcode].Freq * - (5L+extra_dbits[dcode]); - } - out_length >>= 3; - Tracev((stderr,"\nlast_lit %u, in %ld, out ~%ld(%ld%%) ", - s->last_lit, in_length, out_length, - 100L - out_length*100L/in_length)); - if (s->matches < s->last_lit/2 && out_length < in_length/2) return 1; - } -#endif - return (s->last_lit == s->lit_bufsize-1); - /* We avoid equality with lit_bufsize because of wraparound at 64K - * on 16 bit machines and because stored blocks are restricted to - * 64K-1 bytes. - */ + return (s->sym_next == s->sym_end); } /* =========================================================================== @@ -1075,18 +1041,19 @@ int ZLIB_INTERNAL _tr_tally (s, dist, lc) */ local void compress_block(s, ltree, dtree) deflate_state *s; - ct_data *ltree; /* literal tree */ - ct_data *dtree; /* distance tree */ + const ct_data *ltree; /* literal tree */ + const ct_data *dtree; /* distance tree */ { unsigned dist; /* distance of matched string */ int lc; /* match length or unmatched char (if dist == 0) */ - unsigned lx = 0; /* running index in l_buf */ + unsigned sx = 0; /* running index in sym_buf */ unsigned code; /* the code to send */ int extra; /* number of extra bits to send */ - if (s->last_lit != 0) do { - dist = s->d_buf[lx]; - lc = s->l_buf[lx++]; + if (s->sym_next != 0) do { + dist = s->sym_buf[sx++] & 0xff; + dist += (unsigned)(s->sym_buf[sx++] & 0xff) << 8; + lc = s->sym_buf[sx++]; if (dist == 0) { send_code(s, lc, ltree); /* send a literal byte */ Tracecv(isgraph(lc), (stderr," '%c' ", lc)); @@ -1106,19 +1073,17 @@ local void compress_block(s, ltree, dtree) send_code(s, code, dtree); /* send the distance code */ extra = extra_dbits[code]; if (extra != 0) { - dist -= base_dist[code]; + dist -= (unsigned)base_dist[code]; send_bits(s, dist, extra); /* send the extra distance bits */ } } /* literal or match pair ? */ - /* Check that the overlay between pending_buf and d_buf+l_buf is ok: */ - Assert((uInt)(s->pending) < s->lit_bufsize + 2*lx, - "pendingBuf overflow"); + /* Check that the overlay between pending_buf and sym_buf is ok: */ + Assert(s->pending < s->lit_bufsize + sx, "pendingBuf overflow"); - } while (lx < s->last_lit); + } while (sx < s->sym_next); send_code(s, END_BLOCK, ltree); - s->last_eob_len = ltree[END_BLOCK].Len; } /* =========================================================================== @@ -1210,35 +1175,7 @@ local void bi_windup(s) } s->bi_buf = 0; s->bi_valid = 0; -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->bits_sent = (s->bits_sent+7) & ~7; #endif } - -/* =========================================================================== - * Copy a stored block, storing first the length and its - * one's complement if requested. - */ -local void copy_block(s, buf, len, header) - deflate_state *s; - charf *buf; /* the input data */ - unsigned len; /* its length */ - int header; /* true if block header must be written */ -{ - bi_windup(s); /* align on byte boundary */ - s->last_eob_len = 8; /* enough lookahead for inflate */ - - if (header) { - put_short(s, (ush)len); - put_short(s, (ush)~len); -#ifdef DEBUG - s->bits_sent += 2*16; -#endif - } -#ifdef DEBUG - s->bits_sent += (ulg)len<<3; -#endif - while (len--) { - put_byte(s, *buf++); - } -} diff --git a/third_party/zlib/uncompr.c b/third_party/zlib/uncompr.c index ad98be3a5d..f03a1a865e 100644 --- a/third_party/zlib/uncompr.c +++ b/third_party/zlib/uncompr.c @@ -1,5 +1,5 @@ /* uncompr.c -- decompress a memory buffer - * Copyright (C) 1995-2003, 2010 Jean-loup Gailly. + * Copyright (C) 1995-2003, 2010, 2014, 2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -9,51 +9,85 @@ #include "zlib.h" /* =========================================================================== - Decompresses the source buffer into the destination buffer. sourceLen is - the byte length of the source buffer. Upon entry, destLen is the total - size of the destination buffer, which must be large enough to hold the - entire uncompressed data. (The size of the uncompressed data must have - been saved previously by the compressor and transmitted to the decompressor - by some mechanism outside the scope of this compression library.) - Upon exit, destLen is the actual size of the compressed buffer. - - uncompress returns Z_OK if success, Z_MEM_ERROR if there was not - enough memory, Z_BUF_ERROR if there was not enough room in the output - buffer, or Z_DATA_ERROR if the input data was corrupted. + Decompresses the source buffer into the destination buffer. *sourceLen is + the byte length of the source buffer. Upon entry, *destLen is the total size + of the destination buffer, which must be large enough to hold the entire + uncompressed data. (The size of the uncompressed data must have been saved + previously by the compressor and transmitted to the decompressor by some + mechanism outside the scope of this compression library.) Upon exit, + *destLen is the size of the decompressed data and *sourceLen is the number + of source bytes consumed. Upon return, source + *sourceLen points to the + first unused input byte. + + uncompress returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_BUF_ERROR if there was not enough room in the output buffer, or + Z_DATA_ERROR if the input data was corrupted, including if the input data is + an incomplete zlib stream. */ -int ZEXPORT uncompress (dest, destLen, source, sourceLen) +int ZEXPORT uncompress2 (dest, destLen, source, sourceLen) Bytef *dest; uLongf *destLen; const Bytef *source; - uLong sourceLen; + uLong *sourceLen; { z_stream stream; int err; + const uInt max = (uInt)-1; + uLong len, left; + Byte buf[1]; /* for detection of incomplete stream when *destLen == 0 */ - stream.next_in = (Bytef*)source; - stream.avail_in = (uInt)sourceLen; - /* Check for source > 64K on 16-bit machine: */ - if ((uLong)stream.avail_in != sourceLen) return Z_BUF_ERROR; - - stream.next_out = dest; - stream.avail_out = (uInt)*destLen; - if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR; + len = *sourceLen; + if (*destLen) { + left = *destLen; + *destLen = 0; + } + else { + left = 1; + dest = buf; + } + stream.next_in = (z_const Bytef *)source; + stream.avail_in = 0; stream.zalloc = (alloc_func)0; stream.zfree = (free_func)0; + stream.opaque = (voidpf)0; err = inflateInit(&stream); if (err != Z_OK) return err; - err = inflate(&stream, Z_FINISH); - if (err != Z_STREAM_END) { - inflateEnd(&stream); - if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0)) - return Z_DATA_ERROR; - return err; - } - *destLen = stream.total_out; + stream.next_out = dest; + stream.avail_out = 0; + + do { + if (stream.avail_out == 0) { + stream.avail_out = left > (uLong)max ? max : (uInt)left; + left -= stream.avail_out; + } + if (stream.avail_in == 0) { + stream.avail_in = len > (uLong)max ? max : (uInt)len; + len -= stream.avail_in; + } + err = inflate(&stream, Z_NO_FLUSH); + } while (err == Z_OK); - err = inflateEnd(&stream); - return err; + *sourceLen -= len + stream.avail_in; + if (dest != buf) + *destLen = stream.total_out; + else if (stream.total_out && err == Z_BUF_ERROR) + left = 1; + + inflateEnd(&stream); + return err == Z_STREAM_END ? Z_OK : + err == Z_NEED_DICT ? Z_DATA_ERROR : + err == Z_BUF_ERROR && left + stream.avail_out ? Z_DATA_ERROR : + err; +} + +int ZEXPORT uncompress (dest, destLen, source, sourceLen) + Bytef *dest; + uLongf *destLen; + const Bytef *source; + uLong sourceLen; +{ + return uncompress2(dest, destLen, source, &sourceLen); } diff --git a/third_party/zlib/x86.c b/third_party/zlib/x86.c deleted file mode 100644 index 0649306f2f..0000000000 --- a/third_party/zlib/x86.c +++ /dev/null @@ -1,112 +0,0 @@ -/* - * x86 feature check - * - * Copyright (C) 2013 Intel Corporation. All rights reserved. - * Author: - * Jim Kukunas - * - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -#include "x86.h" - -int x86_cpu_enable_simd = 0; - -#ifndef _MSC_VER -#include - -pthread_once_t cpu_check_inited_once = PTHREAD_ONCE_INIT; -static void _x86_check_features(void); - -void x86_check_features(void) -{ - pthread_once(&cpu_check_inited_once, _x86_check_features); -} - -static void _x86_check_features(void) -{ - int x86_cpu_has_sse2; - int x86_cpu_has_sse42; - int x86_cpu_has_pclmulqdq; - unsigned eax, ebx, ecx, edx; - - eax = 1; -#ifdef __i386__ - __asm__ __volatile__ ( - "xchg %%ebx, %1\n\t" - "cpuid\n\t" - "xchg %1, %%ebx\n\t" - : "+a" (eax), "=S" (ebx), "=c" (ecx), "=d" (edx) - ); -#else - __asm__ __volatile__ ( - "cpuid\n\t" - : "+a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx) - ); -#endif /* (__i386__) */ - - x86_cpu_has_sse2 = edx & 0x4000000; - x86_cpu_has_sse42 = ecx & 0x100000; - x86_cpu_has_pclmulqdq = ecx & 0x2; - - x86_cpu_enable_simd = x86_cpu_has_sse2 && - x86_cpu_has_sse42 && - x86_cpu_has_pclmulqdq; -} -#else -#include -#include -#include - -static volatile int32_t once_control = 0; -static void _x86_check_features(void); -static int fake_pthread_once(volatile int32_t *once_control, - void (*init_routine)(void)); - -void x86_check_features(void) -{ - fake_pthread_once(&once_control, _x86_check_features); -} - -/* Copied from "perftools_pthread_once" in tcmalloc */ -static int fake_pthread_once(volatile int32_t *once_control, - void (*init_routine)(void)) { - // Try for a fast path first. Note: this should be an acquire semantics read - // It is on x86 and x64, where Windows runs. - if (*once_control != 1) { - while (1) { - switch (InterlockedCompareExchange(once_control, 2, 0)) { - case 0: - init_routine(); - InterlockedExchange(once_control, 1); - return 0; - case 1: - // The initializer has already been executed - return 0; - default: - // The initializer is being processed by another thread - SwitchToThread(); - } - } - } - return 0; -} - -static void _x86_check_features(void) -{ - int x86_cpu_has_sse2; - int x86_cpu_has_sse42; - int x86_cpu_has_pclmulqdq; - int regs[4]; - - __cpuid(regs, 1); - - x86_cpu_has_sse2 = regs[3] & 0x4000000; - x86_cpu_has_sse42= regs[2] & 0x100000; - x86_cpu_has_pclmulqdq = regs[2] & 0x2; - - x86_cpu_enable_simd = x86_cpu_has_sse2 && - x86_cpu_has_sse42 && - x86_cpu_has_pclmulqdq; -} -#endif /* _MSC_VER */ diff --git a/third_party/zlib/x86.h b/third_party/zlib/x86.h deleted file mode 100644 index ac3d1808db..0000000000 --- a/third_party/zlib/x86.h +++ /dev/null @@ -1,13 +0,0 @@ -/* x86.h -- check for x86 CPU features -* Copyright (C) 2013 Intel Corporation Jim Kukunas -* For conditions of distribution and use, see copyright notice in zlib.h -*/ - -#ifndef X86_H -#define X86_H - -extern int x86_cpu_enable_simd; - -void x86_check_features(void); - -#endif /* X86_H */ diff --git a/third_party/zlib/zconf.h b/third_party/zlib/zconf.h index 8cae8a91d5..2d1d03ff95 100644 --- a/third_party/zlib/zconf.h +++ b/third_party/zlib/zconf.h @@ -1,5 +1,5 @@ /* zconf.h -- configuration of the zlib compression library - * Copyright (C) 1995-2010 Jean-loup Gailly. + * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -8,8 +8,17 @@ #ifndef ZCONF_H #define ZCONF_H -/* This include does prefixing as below, but with an updated set of names */ -#include "mozzconf.h" +/* + * This library is also built as a part of AOSP, which does not need to include + * chromeconf.h. This config does not want chromeconf.h, so it can set this + * macro to opt out. While this works today, there's no guarantee that building + * zlib outside of Chromium keeps working in the future. + */ +#if !defined(CHROMIUM_ZLIB_NO_CHROMECONF) +/* This include does prefixing as below, but with an updated set of names. Also + * sets up export macros in component builds. */ +#include "chromeconf.h" +#endif /* * If you *really* need a unique prefix for all types and library functions, @@ -18,11 +27,13 @@ * this permanently in zconf.h using "./configure --zprefix". */ #ifdef Z_PREFIX /* may be set to #if 1 by ./configure */ +# define Z_PREFIX_SET -/* all linked symbols */ +/* all linked symbols and init macros */ # define _dist_code z__dist_code # define _length_code z__length_code # define _tr_align z__tr_align +# define _tr_flush_bits z__tr_flush_bits # define _tr_flush_block z__tr_flush_block # define _tr_init z__tr_init # define _tr_stored_block z__tr_stored_block @@ -30,81 +41,111 @@ # define adler32 z_adler32 # define adler32_combine z_adler32_combine # define adler32_combine64 z_adler32_combine64 -# define compress z_compress -# define compress2 z_compress2 -# define compressBound z_compressBound +# define adler32_z z_adler32_z +# ifndef Z_SOLO +# define compress z_compress +# define compress2 z_compress2 +# define compressBound z_compressBound +# endif # define crc32 z_crc32 # define crc32_combine z_crc32_combine # define crc32_combine64 z_crc32_combine64 +# define crc32_z z_crc32_z # define deflate z_deflate # define deflateBound z_deflateBound # define deflateCopy z_deflateCopy # define deflateEnd z_deflateEnd +# define deflateGetDictionary z_deflateGetDictionary +# define deflateInit z_deflateInit +# define deflateInit2 z_deflateInit2 # define deflateInit2_ z_deflateInit2_ # define deflateInit_ z_deflateInit_ # define deflateParams z_deflateParams +# define deflatePending z_deflatePending # define deflatePrime z_deflatePrime # define deflateReset z_deflateReset +# define deflateResetKeep z_deflateResetKeep # define deflateSetDictionary z_deflateSetDictionary # define deflateSetHeader z_deflateSetHeader # define deflateTune z_deflateTune # define deflate_copyright z_deflate_copyright # define get_crc_table z_get_crc_table -# define gz_error z_gz_error -# define gz_intmax z_gz_intmax -# define gz_strwinerror z_gz_strwinerror -# define gzbuffer z_gzbuffer -# define gzclearerr z_gzclearerr -# define gzclose z_gzclose -# define gzclose_r z_gzclose_r -# define gzclose_w z_gzclose_w -# define gzdirect z_gzdirect -# define gzdopen z_gzdopen -# define gzeof z_gzeof -# define gzerror z_gzerror -# define gzflush z_gzflush -# define gzgetc z_gzgetc -# define gzgets z_gzgets -# define gzoffset z_gzoffset -# define gzoffset64 z_gzoffset64 -# define gzopen z_gzopen -# define gzopen64 z_gzopen64 -# define gzprintf z_gzprintf -# define gzputc z_gzputc -# define gzputs z_gzputs -# define gzread z_gzread -# define gzrewind z_gzrewind -# define gzseek z_gzseek -# define gzseek64 z_gzseek64 -# define gzsetparams z_gzsetparams -# define gztell z_gztell -# define gztell64 z_gztell64 -# define gzungetc z_gzungetc -# define gzwrite z_gzwrite +# ifndef Z_SOLO +# define gz_error z_gz_error +# define gz_intmax z_gz_intmax +# define gz_strwinerror z_gz_strwinerror +# define gzbuffer z_gzbuffer +# define gzclearerr z_gzclearerr +# define gzclose z_gzclose +# define gzclose_r z_gzclose_r +# define gzclose_w z_gzclose_w +# define gzdirect z_gzdirect +# define gzdopen z_gzdopen +# define gzeof z_gzeof +# define gzerror z_gzerror +# define gzflush z_gzflush +# define gzfread z_gzfread +# define gzfwrite z_gzfwrite +# define gzgetc z_gzgetc +# define gzgetc_ z_gzgetc_ +# define gzgets z_gzgets +# define gzoffset z_gzoffset +# define gzoffset64 z_gzoffset64 +# define gzopen z_gzopen +# define gzopen64 z_gzopen64 +# ifdef _WIN32 +# define gzopen_w z_gzopen_w +# endif +# define gzprintf z_gzprintf +# define gzputc z_gzputc +# define gzputs z_gzputs +# define gzread z_gzread +# define gzrewind z_gzrewind +# define gzseek z_gzseek +# define gzseek64 z_gzseek64 +# define gzsetparams z_gzsetparams +# define gztell z_gztell +# define gztell64 z_gztell64 +# define gzungetc z_gzungetc +# define gzvprintf z_gzvprintf +# define gzwrite z_gzwrite +# endif # define inflate z_inflate # define inflateBack z_inflateBack # define inflateBackEnd z_inflateBackEnd +# define inflateBackInit z_inflateBackInit # define inflateBackInit_ z_inflateBackInit_ +# define inflateCodesUsed z_inflateCodesUsed # define inflateCopy z_inflateCopy # define inflateEnd z_inflateEnd +# define inflateGetDictionary z_inflateGetDictionary # define inflateGetHeader z_inflateGetHeader +# define inflateInit z_inflateInit +# define inflateInit2 z_inflateInit2 # define inflateInit2_ z_inflateInit2_ # define inflateInit_ z_inflateInit_ # define inflateMark z_inflateMark # define inflatePrime z_inflatePrime # define inflateReset z_inflateReset # define inflateReset2 z_inflateReset2 +# define inflateResetKeep z_inflateResetKeep # define inflateSetDictionary z_inflateSetDictionary # define inflateSync z_inflateSync # define inflateSyncPoint z_inflateSyncPoint # define inflateUndermine z_inflateUndermine +# define inflateValidate z_inflateValidate # define inflate_copyright z_inflate_copyright # define inflate_fast z_inflate_fast # define inflate_table z_inflate_table -# define uncompress z_uncompress +# ifndef Z_SOLO +# define uncompress z_uncompress +# define uncompress2 z_uncompress2 +# endif # define zError z_zError -# define zcalloc z_zcalloc -# define zcfree z_zcfree +# ifndef Z_SOLO +# define zcalloc z_zcalloc +# define zcfree z_zcfree +# endif # define zlibCompileFlags z_zlibCompileFlags # define zlibVersion z_zlibVersion @@ -114,7 +155,9 @@ # define alloc_func z_alloc_func # define charf z_charf # define free_func z_free_func -# define gzFile z_gzFile +# ifndef Z_SOLO +# define gzFile z_gzFile +# endif # define gz_header z_gz_header # define gz_headerp z_gz_headerp # define in_func z_in_func @@ -200,9 +243,25 @@ # endif #endif -/* Some Mac compilers merge all .h files incorrectly: */ -#if defined(__MWERKS__)||defined(applec)||defined(THINK_C)||defined(__SC__) -# define NO_DUMMY_DECL +#if defined(ZLIB_CONST) && !defined(z_const) +# define z_const const +#else +# define z_const +#endif + +#ifdef Z_SOLO + typedef unsigned long z_size_t; +#else +# define z_longlong long long +# if defined(NO_SIZE_T) + typedef unsigned NO_SIZE_T z_size_t; +# elif defined(STDC) +# include + typedef size_t z_size_t; +# else + typedef unsigned long z_size_t; +# endif +# undef z_longlong #endif /* Maximum value for memLevel in deflateInit2 */ @@ -232,7 +291,7 @@ Of course this will generally degrade compression (there's no free lunch). The memory requirements for inflate are (in bytes) 1 << windowBits - that is, 32K for windowBits=15 (default value) plus a few kilobytes + that is, 32K for windowBits=15 (default value) plus about 7 kilobytes for small objects. */ @@ -246,6 +305,14 @@ # endif #endif +#ifndef Z_ARG /* function prototypes for stdarg */ +# if defined(STDC) || defined(Z_HAVE_STDARG_H) +# define Z_ARG(args) args +# else +# define Z_ARG(args) () +# endif +#endif + /* The following definitions for FAR are needed only for MSDOS mixed * model programming (small or medium model with some far allocations). * This was tested only with MSC; for other MSDOS compilers you may have @@ -362,12 +429,47 @@ typedef uLong FAR uLongf; typedef Byte *voidp; #endif +#if !defined(Z_U4) && !defined(Z_SOLO) && defined(STDC) +# include +# if (UINT_MAX == 0xffffffffUL) +# define Z_U4 unsigned +# elif (ULONG_MAX == 0xffffffffUL) +# define Z_U4 unsigned long +# elif (USHRT_MAX == 0xffffffffUL) +# define Z_U4 unsigned short +# endif +#endif + +#ifdef Z_U4 + typedef Z_U4 z_crc_t; +#else + typedef unsigned long z_crc_t; +#endif + #if !defined(_WIN32) # define Z_HAVE_UNISTD_H #endif +#ifdef HAVE_STDARG_H /* may be set to #if 1 by ./configure */ +# define Z_HAVE_STDARG_H +#endif + #ifdef STDC -# include /* for off_t */ +# ifndef Z_SOLO +# include /* for off_t */ +# endif +#endif + +#if defined(STDC) || defined(Z_HAVE_STDARG_H) +# ifndef Z_SOLO +# include /* for va_list */ +# endif +#endif + +#ifdef _WIN32 +# ifndef Z_SOLO +# include /* for wchar_t */ +# endif #endif /* a little trick to accommodate both "#define _LARGEFILE64_SOURCE" and @@ -376,21 +478,38 @@ typedef uLong FAR uLongf; * both "#undef _LARGEFILE64_SOURCE" and "#define _LARGEFILE64_SOURCE 0" as * equivalently requesting no 64-bit operations */ -#if -_LARGEFILE64_SOURCE - -1 == 1 +#if defined(_LARGEFILE64_SOURCE) && -_LARGEFILE64_SOURCE - -1 == 1 # undef _LARGEFILE64_SOURCE #endif -#if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE) -# include /* for SEEK_* and off_t */ -# ifdef VMS -# include /* for off_t */ -# endif -# ifndef z_off_t -# define z_off_t off_t +#if defined(__WATCOMC__) && !defined(Z_HAVE_UNISTD_H) +# define Z_HAVE_UNISTD_H +#endif +#ifndef Z_SOLO +# if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE) +# include /* for SEEK_*, off_t, and _LFS64_LARGEFILE */ +# ifdef VMS +# include /* for off_t */ +# endif +# ifndef z_off_t +# define z_off_t off_t +# endif # endif #endif -#ifndef SEEK_SET +#if defined(_LFS64_LARGEFILE) && _LFS64_LARGEFILE-0 +# define Z_LFS64 +#endif + +#if defined(_LARGEFILE64_SOURCE) && defined(Z_LFS64) +# define Z_LARGE64 +#endif + +#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS-0 == 64 && defined(Z_LFS64) +# define Z_WANT64 +#endif + +#if !defined(SEEK_SET) && !defined(Z_SOLO) # define SEEK_SET 0 /* Seek from beginning of file. */ # define SEEK_CUR 1 /* Seek from current position. */ # define SEEK_END 2 /* Set file pointer to EOF plus "offset" */ @@ -400,18 +519,14 @@ typedef uLong FAR uLongf; # define z_off_t long #endif -#if defined(_LARGEFILE64_SOURCE) && _LFS64_LARGEFILE-0 +#if !defined(_WIN32) && defined(Z_LARGE64) # define z_off64_t off64_t #else -# define z_off64_t z_off_t -#endif - -#if defined(__OS400__) -# define NO_vsnprintf -#endif - -#if defined(__MVS__) -# define NO_vsnprintf +# if defined(_WIN32) && !defined(__GNUC__) && !defined(Z_SOLO) +# define z_off64_t __int64 +# else +# define z_off64_t z_off_t +# endif #endif /* MVS linker does not support external names larger than 8 bytes */ diff --git a/third_party/zlib/zlib.gyp b/third_party/zlib/zlib.gyp deleted file mode 100644 index 42e4e95084..0000000000 --- a/third_party/zlib/zlib.gyp +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -{ - 'targets': [ - { - 'target_name' : 'zlib_x86_simd', - 'type': 'static_library', - 'conditions': [ - ['OS!="ios" and (target_arch=="ia32" or target_arch=="x64")', { - 'cflags' : ['-msse4.2', '-mpclmul'], - 'xcode_settings' : { - 'OTHER_CFLAGS' : ['-msse4.2', '-mpclmul'], - }, - 'sources' : [ - 'crc_folding.c', - 'fill_window_sse.c', - ], - 'conditions': [ - ['OS=="win" and clang==1', { - 'msvs_settings': { - 'VCCLCompilerTool': { - 'AdditionalOptions': [ '-msse4.2', '-mpclmul' ], - }, - }, - }], - ], - }, { - 'sources' : [ 'simd_stub.c' ], - }], - ['OS=="android"', { - 'toolsets': ['target', 'host'], - }], - ], - }, - { - 'target_name': 'zlib', - 'type': 'static_library', - 'sources': [ - 'adler32.c', - 'compress.c', - 'crc32.c', - 'crc32.h', - 'deflate.c', - 'deflate.h', - 'gzclose.c', - 'gzguts.h', - 'gzlib.c', - 'gzread.c', - 'gzwrite.c', - 'infback.c', - 'inffast.c', - 'inffast.h', - 'inffixed.h', - 'inflate.c', - 'inflate.h', - 'inftrees.c', - 'inftrees.h', - 'mozzconf.h', - 'trees.c', - 'trees.h', - 'uncompr.c', - 'x86.h', - 'zconf.h', - 'zlib.h', - 'zutil.c', - 'zutil.h', - ], - 'dependencies' : [ - 'zlib_x86_simd' - ], - 'include_dirs': [ - '.', - ], - 'direct_dependent_settings': { - 'include_dirs': [ - '.', - ], - }, - 'conditions': [ - ['OS!="ios" and (target_arch=="ia32" or target_arch=="x64")', { - 'sources' : [ 'x86.c', ], - 'variables': { - 'clang_warning_flags': [ - '-Wno-incompatible-pointer-types', - ], - }, - }], - ['OS!="win"', { - 'product_name': 'chrome_zlib', - }], ['OS=="android"', { - 'toolsets': ['target', 'host'], - }], - ], - }, - { - 'target_name': 'minizip', - 'type': 'static_library', - 'sources': [ - 'contrib/minizip/ioapi.c', - 'contrib/minizip/ioapi.h', - 'contrib/minizip/iowin32.c', - 'contrib/minizip/iowin32.h', - 'contrib/minizip/unzip.c', - 'contrib/minizip/unzip.h', - 'contrib/minizip/zip.c', - 'contrib/minizip/zip.h', - ], - 'dependencies': [ - 'zlib', - ], - 'include_dirs': [ - '.', - '../..', - ], - 'direct_dependent_settings': { - 'include_dirs': [ - '.', - ], - }, - 'variables': { - 'clang_warning_flags': [ - # zlib uses `if ((a == b))` for some reason. - '-Wno-parentheses-equality', - ], - }, - 'conditions': [ - ['OS!="win"', { - 'sources!': [ - 'contrib/minizip/iowin32.c' - ], - }], - ['OS=="android"', { - 'toolsets': ['target', 'host'], - }], - ['OS=="mac" or OS=="ios" or os_bsd==1 or OS=="android"', { - # Mac, Android and the BSDs don't have fopen64, ftello64, or - # fseeko64. We use fopen, ftell, and fseek instead on these - # systems. - 'defines': [ - 'USE_FILE32API' - ], - }], - ], - }, - ], -} diff --git a/third_party/zlib/zlib.h b/third_party/zlib/zlib.h index e88e5a5158..99fd467f6b 100644 --- a/third_party/zlib/zlib.h +++ b/third_party/zlib/zlib.h @@ -1,7 +1,7 @@ /* zlib.h -- interface of the 'zlib' general purpose compression library - version 1.2.5, April 19th, 2010 + version 1.2.11, January 15th, 2017 - Copyright (C) 1995-2010 Jean-loup Gailly and Mark Adler + Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages @@ -24,8 +24,8 @@ The data format used by the zlib library is described by RFCs (Request for - Comments) 1950 to 1952 in the files http://www.ietf.org/rfc/rfc1950.txt - (zlib format), rfc1951.txt (deflate format) and rfc1952.txt (gzip format). + Comments) 1950 to 1952 in the files http://tools.ietf.org/html/rfc1950 + (zlib format), rfc1951 (deflate format) and rfc1952 (gzip format). */ #ifndef ZLIB_H @@ -37,11 +37,11 @@ extern "C" { #endif -#define ZLIB_VERSION "1.2.5" -#define ZLIB_VERNUM 0x1250 +#define ZLIB_VERSION "1.2.11" +#define ZLIB_VERNUM 0x12b0 #define ZLIB_VER_MAJOR 1 #define ZLIB_VER_MINOR 2 -#define ZLIB_VER_REVISION 5 +#define ZLIB_VER_REVISION 11 #define ZLIB_VER_SUBREVISION 0 /* @@ -65,7 +65,8 @@ extern "C" { with "gz". The gzip format is different from the zlib format. gzip is a gzip wrapper, documented in RFC 1952, wrapped around a deflate stream. - This library can optionally read and write gzip streams in memory as well. + This library can optionally read and write gzip and raw deflate streams in + memory as well. The zlib format was designed to be compact and fast for use in memory and on communications channels. The gzip format was designed for single- @@ -74,7 +75,7 @@ extern "C" { The library does not install any signal handler. The decoder checks the consistency of the compressed data, so the library should never crash - even in case of corrupted input. + even in the case of corrupted input. */ typedef voidpf (*alloc_func) OF((voidpf opaque, uInt items, uInt size)); @@ -83,25 +84,25 @@ typedef void (*free_func) OF((voidpf opaque, voidpf address)); struct internal_state; typedef struct z_stream_s { - Bytef *next_in; /* next input byte */ + z_const Bytef *next_in; /* next input byte */ uInt avail_in; /* number of bytes available at next_in */ - uLong total_in; /* total nb of input bytes read so far */ + uLong total_in; /* total number of input bytes read so far */ - Bytef *next_out; /* next output byte should be put there */ + Bytef *next_out; /* next output byte will go here */ uInt avail_out; /* remaining free space at next_out */ - uLong total_out; /* total nb of bytes output so far */ + uLong total_out; /* total number of bytes output so far */ - char *msg; /* last error message, NULL if no error */ + z_const char *msg; /* last error message, NULL if no error */ struct internal_state FAR *state; /* not visible by applications */ alloc_func zalloc; /* used to allocate the internal state */ free_func zfree; /* used to free the internal state */ voidpf opaque; /* private data object passed to zalloc and zfree */ - int data_type; /* best guess about the data type: binary or text */ - uLong adler; /* adler32 value of the uncompressed data */ + int data_type; /* best guess about the data type: binary or text + for deflate, or the decoding state for inflate */ + uLong adler; /* Adler-32 or CRC-32 value of the uncompressed data */ uLong reserved; /* reserved for future use */ - int clas; } z_stream; typedef z_stream FAR *z_streamp; @@ -143,7 +144,9 @@ typedef gz_header FAR *gz_headerp; zalloc must return Z_NULL if there is not enough memory for the object. If zlib is used in a multi-threaded application, zalloc and zfree must be - thread safe. + thread safe. In that case, zlib is thread-safe. When zalloc and zfree are + Z_NULL on entry to the initialization function, they are set to internal + routines that use the standard library functions malloc() and free(). On 16-bit systems, the functions zalloc and zfree must be able to allocate exactly 65536 bytes, but will not be required to allocate more than this if @@ -156,7 +159,7 @@ typedef gz_header FAR *gz_headerp; The fields total_in and total_out can be used for statistics or progress reports. After compression, total_in holds the total size of the - uncompressed data and may be saved for use in the decompressor (particularly + uncompressed data and may be saved for use by the decompressor (particularly if the decompressor wants to decompress everything in a single step). */ @@ -201,17 +204,13 @@ typedef gz_header FAR *gz_headerp; #define Z_TEXT 1 #define Z_ASCII Z_TEXT /* for compatibility with 1.2.2 and earlier */ #define Z_UNKNOWN 2 -/* Possible values of the data_type field (though see inflate()) */ +/* Possible values of the data_type field for deflate() */ #define Z_DEFLATED 8 /* The deflate compression method (the only one supported in this version) */ #define Z_NULL 0 /* for initializing zalloc, zfree, opaque */ -#define Z_CLASS_STANDARD 0 -#define Z_CLASS_COOKIE 1 -#define Z_CLASS_HUFFMAN_ONLY 2 - #define zlib_version zlibVersion() /* for compatibility with versions < 1.0.2 */ @@ -263,11 +262,11 @@ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); enough room in the output buffer), next_in and avail_in are updated and processing will resume at this point for the next call of deflate(). - - Provide more output starting at next_out and update next_out and avail_out + - Generate more output starting at next_out and update next_out and avail_out accordingly. This action is forced if the parameter flush is non zero. Forcing flush frequently degrades the compression ratio, so this parameter - should be set only when necessary (in interactive applications). Some - output may be provided even if flush is not set. + should be set only when necessary. Some output may be provided even if + flush is zero. Before the call of deflate(), the application should ensure that at least one of the actions is possible, by providing more input and/or consuming more @@ -276,7 +275,9 @@ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); output when it wants, for example when the output buffer is full (avail_out == 0), or after each call of deflate(). If deflate returns Z_OK and with zero avail_out, it must be called again after making room in the output - buffer because there might be more output pending. + buffer because there might be more output pending. See deflatePending(), + which can be used if desired to determine whether or not there is more ouput + in that case. Normally the parameter flush is set to Z_NO_FLUSH, which allows deflate to decide how much data to accumulate before producing output, in order to @@ -297,8 +298,8 @@ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); input data so far will be available to the decompressor, as for Z_SYNC_FLUSH. This completes the current deflate block and follows it with an empty fixed codes block that is 10 bits long. This assures that enough bytes are output - in order for the decompressor to finish the block before the empty fixed code - block. + in order for the decompressor to finish the block before the empty fixed + codes block. If flush is set to Z_BLOCK, a deflate block is completed and emitted, as for Z_SYNC_FLUSH, but the output is not aligned on a byte boundary, and up to @@ -324,33 +325,38 @@ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); If the parameter flush is set to Z_FINISH, pending input is processed, pending output is flushed and deflate returns with Z_STREAM_END if there was - enough output space; if deflate returns with Z_OK, this function must be - called again with Z_FINISH and more output space (updated avail_out) but no - more input data, until it returns with Z_STREAM_END or an error. After - deflate has returned Z_STREAM_END, the only possible operations on the stream - are deflateReset or deflateEnd. - - Z_FINISH can be used immediately after deflateInit if all the compression - is to be done in a single step. In this case, avail_out must be at least the - value returned by deflateBound (see below). If deflate does not return - Z_STREAM_END, then it must be called again as described above. - - deflate() sets strm->adler to the adler32 checksum of all input read - so far (that is, total_in bytes). + enough output space. If deflate returns with Z_OK or Z_BUF_ERROR, this + function must be called again with Z_FINISH and more output space (updated + avail_out) but no more input data, until it returns with Z_STREAM_END or an + error. After deflate has returned Z_STREAM_END, the only possible operations + on the stream are deflateReset or deflateEnd. + + Z_FINISH can be used in the first deflate call after deflateInit if all the + compression is to be done in a single step. In order to complete in one + call, avail_out must be at least the value returned by deflateBound (see + below). Then deflate is guaranteed to return Z_STREAM_END. If not enough + output space is provided, deflate will not return Z_STREAM_END, and it must + be called again as described above. + + deflate() sets strm->adler to the Adler-32 checksum of all input read + so far (that is, total_in bytes). If a gzip stream is being generated, then + strm->adler will be the CRC-32 checksum of the input read so far. (See + deflateInit2 below.) deflate() may update strm->data_type if it can make a good guess about - the input data type (Z_BINARY or Z_TEXT). In doubt, the data is considered - binary. This field is only for information purposes and does not affect the - compression algorithm in any manner. + the input data type (Z_BINARY or Z_TEXT). If in doubt, the data is + considered binary. This field is only for information purposes and does not + affect the compression algorithm in any manner. deflate() returns Z_OK if some progress has been made (more input processed or more output produced), Z_STREAM_END if all input has been consumed and all output has been produced (only when flush is set to Z_FINISH), Z_STREAM_ERROR if the stream state was inconsistent (for example - if next_in or next_out was Z_NULL), Z_BUF_ERROR if no progress is possible - (for example avail_in or avail_out was zero). Note that Z_BUF_ERROR is not - fatal, and deflate() can be called again with more input and more output - space to continue compressing. + if next_in or next_out was Z_NULL or the state was inadvertently written over + by the application), or Z_BUF_ERROR if no progress is possible (for example + avail_in or avail_out was zero). Note that Z_BUF_ERROR is not fatal, and + deflate() can be called again with more input and more output space to + continue compressing. */ @@ -373,23 +379,21 @@ ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm)); Initializes the internal stream state for decompression. The fields next_in, avail_in, zalloc, zfree and opaque must be initialized before by - the caller. If next_in is not Z_NULL and avail_in is large enough (the - exact value depends on the compression method), inflateInit determines the - compression method from the zlib header and allocates all data structures - accordingly; otherwise the allocation will be deferred to the first call of - inflate. If zalloc and zfree are set to Z_NULL, inflateInit updates them to - use default allocation functions. + the caller. In the current version of inflate, the provided input is not + read or consumed. The allocation of a sliding window will be deferred to + the first call of inflate (if the decompression does not complete on the + first call). If zalloc and zfree are set to Z_NULL, inflateInit updates + them to use default allocation functions. inflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_VERSION_ERROR if the zlib library version is incompatible with the version assumed by the caller, or Z_STREAM_ERROR if the parameters are invalid, such as a null pointer to the structure. msg is set to null if - there is no error message. inflateInit does not perform any decompression - apart from possibly reading the zlib header if present: actual decompression - will be done by inflate(). (So next_in and avail_in may be modified, but - next_out and avail_out are unused and unchanged.) The current implementation - of inflateInit() does not process any header information -- that is deferred - until inflate() is called. + there is no error message. inflateInit does not perform any decompression. + Actual decompression will be done by inflate(). So next_in, and avail_in, + next_out, and avail_out are unused and unchanged. The current + implementation of inflateInit() does not process any header information -- + that is deferred until inflate() is called. */ @@ -405,17 +409,20 @@ ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); - Decompress more input starting at next_in and update next_in and avail_in accordingly. If not all input can be processed (because there is not - enough room in the output buffer), next_in is updated and processing will - resume at this point for the next call of inflate(). + enough room in the output buffer), then next_in and avail_in are updated + accordingly, and processing will resume at this point for the next call of + inflate(). - - Provide more output starting at next_out and update next_out and avail_out + - Generate more output starting at next_out and update next_out and avail_out accordingly. inflate() provides as much output as possible, until there is no more input data or no more space in the output buffer (see below about the flush parameter). Before the call of inflate(), the application should ensure that at least one of the actions is possible, by providing more input and/or consuming more - output, and updating the next_* and avail_* values accordingly. The + output, and updating the next_* and avail_* values accordingly. If the + caller of inflate() does not provide both available input and available + output space, it is possible that there will be no progress made. The application can consume the uncompressed output when it wants, for example when the output buffer is full (avail_out == 0), or after each call of inflate(). If inflate returns Z_OK and with zero avail_out, it must be @@ -432,7 +439,7 @@ ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); gets to the end of that block, or when it runs out of data. The Z_BLOCK option assists in appending to or combining deflate streams. - Also to assist in this, on return inflate() will set strm->data_type to the + To assist in this, on return inflate() always sets strm->data_type to the number of unused bits in the last byte taken from strm->next_in, plus 64 if inflate() is currently decoding the last block in the deflate stream, plus 128 if inflate() returned immediately after decoding an end-of-block code or @@ -456,48 +463,57 @@ ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); error. However if all decompression is to be performed in a single step (a single call of inflate), the parameter flush should be set to Z_FINISH. In this case all pending input is processed and all pending output is flushed; - avail_out must be large enough to hold all the uncompressed data. (The size - of the uncompressed data may have been saved by the compressor for this - purpose.) The next operation on this stream must be inflateEnd to deallocate - the decompression state. The use of Z_FINISH is never required, but can be - used to inform inflate that a faster approach may be used for the single - inflate() call. + avail_out must be large enough to hold all of the uncompressed data for the + operation to complete. (The size of the uncompressed data may have been + saved by the compressor for this purpose.) The use of Z_FINISH is not + required to perform an inflation in one step. However it may be used to + inform inflate that a faster approach can be used for the single inflate() + call. Z_FINISH also informs inflate to not maintain a sliding window if the + stream completes, which reduces inflate's memory footprint. If the stream + does not complete, either because not all of the stream is provided or not + enough output space is provided, then a sliding window will be allocated and + inflate() can be called again to continue the operation as if Z_NO_FLUSH had + been used. In this implementation, inflate() always flushes as much output as possible to the output buffer, and always uses the faster approach on the - first call. So the only effect of the flush parameter in this implementation - is on the return value of inflate(), as noted below, or when it returns early - because Z_BLOCK or Z_TREES is used. + first call. So the effects of the flush parameter in this implementation are + on the return value of inflate() as noted below, when inflate() returns early + when Z_BLOCK or Z_TREES is used, and when inflate() avoids the allocation of + memory for a sliding window when Z_FINISH is used. If a preset dictionary is needed after this call (see inflateSetDictionary - below), inflate sets strm->adler to the adler32 checksum of the dictionary + below), inflate sets strm->adler to the Adler-32 checksum of the dictionary chosen by the compressor and returns Z_NEED_DICT; otherwise it sets - strm->adler to the adler32 checksum of all output produced so far (that is, + strm->adler to the Adler-32 checksum of all output produced so far (that is, total_out bytes) and returns Z_OK, Z_STREAM_END or an error code as described - below. At the end of the stream, inflate() checks that its computed adler32 + below. At the end of the stream, inflate() checks that its computed Adler-32 checksum is equal to that saved by the compressor and returns Z_STREAM_END only if the checksum is correct. inflate() can decompress and check either zlib-wrapped or gzip-wrapped deflate data. The header type is detected automatically, if requested when initializing with inflateInit2(). Any information contained in the gzip - header is not retained, so applications that need that information should - instead use raw inflate, see inflateInit2() below, or inflateBack() and - perform their own processing of the gzip header and trailer. + header is not retained unless inflateGetHeader() is used. When processing + gzip-wrapped deflate data, strm->adler32 is set to the CRC-32 of the output + produced so far. The CRC-32 is checked against the gzip trailer, as is the + uncompressed length, modulo 2^32. inflate() returns Z_OK if some progress has been made (more input processed or more output produced), Z_STREAM_END if the end of the compressed data has been reached and all uncompressed output has been produced, Z_NEED_DICT if a preset dictionary is needed at this point, Z_DATA_ERROR if the input data was corrupted (input stream not conforming to the zlib format or incorrect check - value), Z_STREAM_ERROR if the stream structure was inconsistent (for example - next_in or next_out was Z_NULL), Z_MEM_ERROR if there was not enough memory, - Z_BUF_ERROR if no progress is possible or if there was not enough room in the - output buffer when Z_FINISH is used. Note that Z_BUF_ERROR is not fatal, and + value, in which case strm->msg points to a string with a more specific + error), Z_STREAM_ERROR if the stream structure was inconsistent (for example + next_in or next_out was Z_NULL, or the state was inadvertently written over + by the application), Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR + if no progress was possible or if there was not enough room in the output + buffer when Z_FINISH is used. Note that Z_BUF_ERROR is not fatal, and inflate() can be called again with more input and more output space to continue decompressing. If Z_DATA_ERROR is returned, the application may then call inflateSync() to look for a good compression block if a partial - recovery of the data is desired. + recovery of the data is to be attempted. */ @@ -507,9 +523,8 @@ ZEXTERN int ZEXPORT inflateEnd OF((z_streamp strm)); This function discards any unprocessed input and does not flush any pending output. - inflateEnd returns Z_OK if success, Z_STREAM_ERROR if the stream state - was inconsistent. In the error case, msg may be set but then points to a - static string (which must not be deallocated). + inflateEnd returns Z_OK if success, or Z_STREAM_ERROR if the stream state + was inconsistent. */ @@ -540,16 +555,29 @@ ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm, compression at the expense of memory usage. The default value is 15 if deflateInit is used instead. + For the current implementation of deflate(), a windowBits value of 8 (a + window size of 256 bytes) is not supported. As a result, a request for 8 + will result in 9 (a 512-byte window). In that case, providing 8 to + inflateInit2() will result in an error when the zlib header with 9 is + checked against the initialization of inflate(). The remedy is to not use 8 + with deflateInit2() with this initialization, or at least in that case use 9 + with inflateInit2(). + windowBits can also be -8..-15 for raw deflate. In this case, -windowBits determines the window size. deflate() will then generate raw deflate data - with no zlib header or trailer, and will not compute an adler32 check value. + with no zlib header or trailer, and will not compute a check value. windowBits can also be greater than 15 for optional gzip encoding. Add 16 to windowBits to write a simple gzip header and trailer around the compressed data instead of a zlib wrapper. The gzip header will have no file name, no extra data, no comment, no modification time (set to zero), no - header crc, and the operating system will be set to 255 (unknown). If a - gzip stream is being written, strm->adler is a crc32 instead of an adler32. + header crc, and the operating system will be set to the appropriate value, + if the operating system was determined at compile time. If a gzip stream is + being written, strm->adler is a CRC-32 instead of an Adler-32. + + For raw deflate or gzip encoding, a request for a 256-byte window is + rejected as invalid, since only the zlib header provides a means of + transmitting the window size to the decompressor. The memLevel parameter specifies how much memory should be allocated for the internal compression state. memLevel=1 uses minimum memory but is @@ -585,10 +613,15 @@ ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm, uInt dictLength)); /* Initializes the compression dictionary from the given byte sequence - without producing any compressed output. This function must be called - immediately after deflateInit, deflateInit2 or deflateReset, before any call - of deflate. The compressor and decompressor must use exactly the same - dictionary (see inflateSetDictionary). + without producing any compressed output. When using the zlib format, this + function must be called immediately after deflateInit, deflateInit2 or + deflateReset, and before any call of deflate. When doing raw deflate, this + function must be called either before any call of deflate, or immediately + after the completion of a deflate block, i.e. after all input has been + consumed and all output has been delivered when using any of the flush + options Z_BLOCK, Z_PARTIAL_FLUSH, Z_SYNC_FLUSH, or Z_FULL_FLUSH. The + compressor and decompressor must use exactly the same dictionary (see + inflateSetDictionary). The dictionary should consist of strings (byte sequences) that are likely to be encountered later in the data to be compressed, with the most commonly @@ -605,18 +638,40 @@ ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm, addition, the current implementation of deflate will use at most the window size minus 262 bytes of the provided dictionary. - Upon return of this function, strm->adler is set to the adler32 value + Upon return of this function, strm->adler is set to the Adler-32 value of the dictionary; the decompressor may later use this value to determine - which dictionary has been used by the compressor. (The adler32 value + which dictionary has been used by the compressor. (The Adler-32 value applies to the whole dictionary even if only a subset of the dictionary is actually used by the compressor.) If a raw deflate was requested, then the - adler32 value is not computed and strm->adler is not set. + Adler-32 value is not computed and strm->adler is not set. deflateSetDictionary returns Z_OK if success, or Z_STREAM_ERROR if a parameter is invalid (e.g. dictionary being Z_NULL) or the stream state is inconsistent (for example if deflate has already been called for this stream - or if the compression method is bsort). deflateSetDictionary does not - perform any compression: this will be done by deflate(). + or if not at a block boundary for raw deflate). deflateSetDictionary does + not perform any compression: this will be done by deflate(). +*/ + +ZEXTERN int ZEXPORT deflateGetDictionary OF((z_streamp strm, + Bytef *dictionary, + uInt *dictLength)); +/* + Returns the sliding dictionary being maintained by deflate. dictLength is + set to the number of bytes in the dictionary, and that many bytes are copied + to dictionary. dictionary must have enough space, where 32768 bytes is + always enough. If deflateGetDictionary() is called with dictionary equal to + Z_NULL, then only the dictionary length is returned, and nothing is copied. + Similary, if dictLength is Z_NULL, then it is not set. + + deflateGetDictionary() may return a length less than the window size, even + when more than the window size in input has been provided. It may return up + to 258 bytes less in that case, due to how zlib's implementation of deflate + manages the sliding window and lookahead for matches, where matches can be + up to 258 bytes long. If the application needs the last window-size bytes of + input, then that would need to be saved by the application outside of zlib. + + deflateGetDictionary returns Z_OK on success, or Z_STREAM_ERROR if the + stream state is inconsistent. */ ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest, @@ -639,10 +694,10 @@ ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest, ZEXTERN int ZEXPORT deflateReset OF((z_streamp strm)); /* - This function is equivalent to deflateEnd followed by deflateInit, - but does not free and reallocate all the internal compression state. The - stream will keep the same compression level and any other attributes that - may have been set by deflateInit2. + This function is equivalent to deflateEnd followed by deflateInit, but + does not free and reallocate the internal compression state. The stream + will leave the compression level and any other attributes that may have been + set unchanged. deflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent (such as zalloc or state being Z_NULL). @@ -653,20 +708,36 @@ ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm, int strategy)); /* Dynamically update the compression level and compression strategy. The - interpretation of level and strategy is as in deflateInit2. This can be + interpretation of level and strategy is as in deflateInit2(). This can be used to switch between compression and straight copy of the input data, or to switch to a different kind of input data requiring a different strategy. - If the compression level is changed, the input available so far is - compressed with the old level (and may be flushed); the new level will take - effect only at the next call of deflate(). - - Before the call of deflateParams, the stream state must be set as for - a call of deflate(), since the currently available input may have to be - compressed and flushed. In particular, strm->avail_out must be non-zero. - - deflateParams returns Z_OK if success, Z_STREAM_ERROR if the source - stream state was inconsistent or if a parameter was invalid, Z_BUF_ERROR if - strm->avail_out was zero. + If the compression approach (which is a function of the level) or the + strategy is changed, and if any input has been consumed in a previous + deflate() call, then the input available so far is compressed with the old + level and strategy using deflate(strm, Z_BLOCK). There are three approaches + for the compression levels 0, 1..3, and 4..9 respectively. The new level + and strategy will take effect at the next call of deflate(). + + If a deflate(strm, Z_BLOCK) is performed by deflateParams(), and it does + not have enough output space to complete, then the parameter change will not + take effect. In this case, deflateParams() can be called again with the + same parameters and more output space to try again. + + In order to assure a change in the parameters on the first try, the + deflate stream should be flushed using deflate() with Z_BLOCK or other flush + request until strm.avail_out is not zero, before calling deflateParams(). + Then no more input data should be provided before the deflateParams() call. + If this is done, the old level and strategy will be applied to the data + compressed before deflateParams(), and the new level and strategy will be + applied to the the data compressed after deflateParams(). + + deflateParams returns Z_OK on success, Z_STREAM_ERROR if the source stream + state was inconsistent or if a parameter was invalid, or Z_BUF_ERROR if + there was not enough output space to complete the compression of the + available input data before a change in the strategy or approach. Note that + in the case of a Z_BUF_ERROR, the parameters are not changed. A return + value of Z_BUF_ERROR is not fatal, in which case deflateParams() can be + retried with more output space. */ ZEXTERN int ZEXPORT deflateTune OF((z_streamp strm, @@ -693,8 +764,28 @@ ZEXTERN uLong ZEXPORT deflateBound OF((z_streamp strm, deflation of sourceLen bytes. It must be called after deflateInit() or deflateInit2(), and after deflateSetHeader(), if used. This would be used to allocate an output buffer for deflation in a single pass, and so would be - called before deflate(). -*/ + called before deflate(). If that first deflate() call is provided the + sourceLen input bytes, an output buffer allocated to the size returned by + deflateBound(), and the flush value Z_FINISH, then deflate() is guaranteed + to return Z_STREAM_END. Note that it is possible for the compressed size to + be larger than the value returned by deflateBound() if flush options other + than Z_FINISH or Z_NO_FLUSH are used. +*/ + +ZEXTERN int ZEXPORT deflatePending OF((z_streamp strm, + unsigned *pending, + int *bits)); +/* + deflatePending() returns the number of bytes and bits of output that have + been generated, but not yet provided in the available output. The bytes not + provided would be due to the available output space having being consumed. + The number of bits of output not provided are between 0 and 7, where they + await more bits to join them in order to fill out a full byte. If pending + or bits are Z_NULL, then those values are not set. + + deflatePending returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent. + */ ZEXTERN int ZEXPORT deflatePrime OF((z_streamp strm, int bits, @@ -708,8 +799,9 @@ ZEXTERN int ZEXPORT deflatePrime OF((z_streamp strm, than or equal to 16, and that many of the least significant bits of value will be inserted in the output. - deflatePrime returns Z_OK if success, or Z_STREAM_ERROR if the source - stream state was inconsistent. + deflatePrime returns Z_OK if success, Z_BUF_ERROR if there was not enough + room in the internal buffer to insert the bits, or Z_STREAM_ERROR if the + source stream state was inconsistent. */ ZEXTERN int ZEXPORT deflateSetHeader OF((z_streamp strm, @@ -763,7 +855,7 @@ ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm, is for use with other formats that use the deflate compressed data format such as zip. Those formats provide their own check values. If a custom format is developed using the raw deflate format for compressed data, it is - recommended that a check value such as an adler32 or a crc32 be applied to + recommended that a check value such as an Adler-32 or a CRC-32 be applied to the uncompressed data as is done in the zlib, gzip, and zip formats. For most applications, the zlib format should be used as is. Note that comments above on the use in deflateInit2() applies to the magnitude of windowBits. @@ -772,7 +864,10 @@ ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm, 32 to windowBits to enable zlib and gzip decoding with automatic header detection, or add 16 to decode only the gzip format (the zlib format will return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a - crc32 instead of an adler32. + CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see + below), inflate() will not automatically decode concatenated gzip streams. + inflate() will return Z_STREAM_END at the end of the gzip stream. The state + would need to be reset to continue decoding a subsequent gzip stream. inflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_VERSION_ERROR if the zlib library version is incompatible with the @@ -793,34 +888,54 @@ ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm, Initializes the decompression dictionary from the given uncompressed byte sequence. This function must be called immediately after a call of inflate, if that call returned Z_NEED_DICT. The dictionary chosen by the compressor - can be determined from the adler32 value returned by that call of inflate. + can be determined from the Adler-32 value returned by that call of inflate. The compressor and decompressor must use exactly the same dictionary (see - deflateSetDictionary). For raw inflate, this function can be called - immediately after inflateInit2() or inflateReset() and before any call of - inflate() to set the dictionary. The application must insure that the - dictionary that was used for compression is provided. + deflateSetDictionary). For raw inflate, this function can be called at any + time to set the dictionary. If the provided dictionary is smaller than the + window and there is already data in the window, then the provided dictionary + will amend what's there. The application must insure that the dictionary + that was used for compression is provided. inflateSetDictionary returns Z_OK if success, Z_STREAM_ERROR if a parameter is invalid (e.g. dictionary being Z_NULL) or the stream state is inconsistent, Z_DATA_ERROR if the given dictionary doesn't match the - expected one (incorrect adler32 value). inflateSetDictionary does not + expected one (incorrect Adler-32 value). inflateSetDictionary does not perform any decompression: this will be done by subsequent calls of inflate(). */ +ZEXTERN int ZEXPORT inflateGetDictionary OF((z_streamp strm, + Bytef *dictionary, + uInt *dictLength)); +/* + Returns the sliding dictionary being maintained by inflate. dictLength is + set to the number of bytes in the dictionary, and that many bytes are copied + to dictionary. dictionary must have enough space, where 32768 bytes is + always enough. If inflateGetDictionary() is called with dictionary equal to + Z_NULL, then only the dictionary length is returned, and nothing is copied. + Similary, if dictLength is Z_NULL, then it is not set. + + inflateGetDictionary returns Z_OK on success, or Z_STREAM_ERROR if the + stream state is inconsistent. +*/ + ZEXTERN int ZEXPORT inflateSync OF((z_streamp strm)); /* - Skips invalid compressed data until a full flush point (see above the - description of deflate with Z_FULL_FLUSH) can be found, or until all + Skips invalid compressed data until a possible full flush point (see above + for the description of deflate with Z_FULL_FLUSH) can be found, or until all available input is skipped. No output is provided. - inflateSync returns Z_OK if a full flush point has been found, Z_BUF_ERROR - if no more input was provided, Z_DATA_ERROR if no flush point has been - found, or Z_STREAM_ERROR if the stream structure was inconsistent. In the - success case, the application may save the current current value of total_in - which indicates where valid compressed data was found. In the error case, - the application may repeatedly call inflateSync, providing more input each - time, until success or end of the input data. + inflateSync searches for a 00 00 FF FF pattern in the compressed data. + All full flush points have this pattern, but not all occurrences of this + pattern are full flush points. + + inflateSync returns Z_OK if a possible full flush point has been found, + Z_BUF_ERROR if no more input was provided, Z_DATA_ERROR if no flush point + has been found, or Z_STREAM_ERROR if the stream structure was inconsistent. + In the success case, the application may save the current current value of + total_in which indicates where valid compressed data was found. In the + error case, the application may repeatedly call inflateSync, providing more + input each time, until success or end of the input data. */ ZEXTERN int ZEXPORT inflateCopy OF((z_streamp dest, @@ -842,7 +957,7 @@ ZEXTERN int ZEXPORT inflateCopy OF((z_streamp dest, ZEXTERN int ZEXPORT inflateReset OF((z_streamp strm)); /* This function is equivalent to inflateEnd followed by inflateInit, - but does not free and reallocate all the internal decompression state. The + but does not free and reallocate the internal decompression state. The stream will keep attributes that may have been set by inflateInit2. inflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source @@ -854,7 +969,9 @@ ZEXTERN int ZEXPORT inflateReset2 OF((z_streamp strm, /* This function is the same as inflateReset, but it also permits changing the wrap and window size requests. The windowBits parameter is interpreted - the same as it is for inflateInit2. + the same as it is for inflateInit2. If the window size is changed, then the + memory allocated for the window is freed, and the window will be reallocated + by inflate() if needed. inflateReset2 returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent (such as zalloc or state being Z_NULL), or if @@ -906,7 +1023,7 @@ ZEXTERN long ZEXPORT inflateMark OF((z_streamp strm)); location in the input stream can be determined from avail_in and data_type as noted in the description for the Z_BLOCK flush parameter for inflate. - inflateMark returns the value noted above or -1 << 16 if the provided + inflateMark returns the value noted above, or -65536 if the provided source stream state was inconsistent. */ @@ -967,12 +1084,13 @@ ZEXTERN int ZEXPORT inflateBackInit OF((z_streamp strm, int windowBits, See inflateBack() for the usage of these routines. inflateBackInit will return Z_OK on success, Z_STREAM_ERROR if any of - the paramaters are invalid, Z_MEM_ERROR if the internal state could not be + the parameters are invalid, Z_MEM_ERROR if the internal state could not be allocated, or Z_VERSION_ERROR if the version of the library does not match the version of the header file. */ -typedef unsigned (*in_func) OF((void FAR *, unsigned char FAR * FAR *)); +typedef unsigned (*in_func) OF((void FAR *, + z_const unsigned char FAR * FAR *)); typedef int (*out_func) OF((void FAR *, unsigned char FAR *, unsigned)); ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, @@ -980,11 +1098,12 @@ ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, out_func out, void FAR *out_desc)); /* inflateBack() does a raw inflate with a single call using a call-back - interface for input and output. This is more efficient than inflate() for - file i/o applications in that it avoids copying between the output and the - sliding window by simply making the window itself the output buffer. This - function trusts the application to not change the output buffer passed by - the output function, at least until inflateBack() returns. + interface for input and output. This is potentially more efficient than + inflate() for file i/o applications, in that it avoids copying between the + output and the sliding window by simply making the window itself the output + buffer. inflate() can be faster on modern CPUs when used with large + buffers. inflateBack() trusts the application to not change the output + buffer passed by the output function, at least until inflateBack() returns. inflateBackInit() must be called first to allocate the internal state and to initialize the state with the user-provided window buffer. @@ -996,9 +1115,9 @@ ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, This routine would normally be used in a utility that reads zip or gzip files and writes out uncompressed files. The utility would decode the header and process the trailer on its own, hence this routine expects only - the raw deflate stream to decompress. This is different from the normal - behavior of inflate(), which expects either a zlib or gzip header and - trailer around the deflate stream. + the raw deflate stream to decompress. This is different from the default + behavior of inflate(), which expects a zlib header and trailer around the + deflate stream. inflateBack() uses two subroutines supplied by the caller that are then called by inflateBack() for input and output. inflateBack() calls those @@ -1007,12 +1126,12 @@ ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, parameters and return types are defined above in the in_func and out_func typedefs. inflateBack() will call in(in_desc, &buf) which should return the number of bytes of provided input, and a pointer to that input in buf. If - there is no input available, in() must return zero--buf is ignored in that - case--and inflateBack() will return a buffer error. inflateBack() will call - out(out_desc, buf, len) to write the uncompressed data buf[0..len-1]. out() - should return zero on success, or non-zero on failure. If out() returns - non-zero, inflateBack() will return with an error. Neither in() nor out() - are permitted to change the contents of the window provided to + there is no input available, in() must return zero -- buf is ignored in that + case -- and inflateBack() will return a buffer error. inflateBack() will + call out(out_desc, buf, len) to write the uncompressed data buf[0..len-1]. + out() should return zero on success, or non-zero on failure. If out() + returns non-zero, inflateBack() will return with an error. Neither in() nor + out() are permitted to change the contents of the window provided to inflateBackInit(), which is also the buffer that out() uses to write from. The length written by out() will be at most the window size. Any non-zero amount of input may be provided by in(). @@ -1040,7 +1159,7 @@ ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, using strm->next_in which will be Z_NULL only if in() returned an error. If strm->next_in is not Z_NULL, then the Z_BUF_ERROR was due to out() returning non-zero. (in() will always be called before out(), so strm->next_in is - assured to be defined if out() returns non-zero.) Note that inflateBack() + assured to be defined if out() returns non-zero.) Note that inflateBack() cannot return Z_OK. */ @@ -1062,7 +1181,7 @@ ZEXTERN uLong ZEXPORT zlibCompileFlags OF((void)); 7.6: size of z_off_t Compiler, assembler, and debug options: - 8: DEBUG + 8: ZLIB_DEBUG 9: ASMV or ASMINF -- use ASM code 10: ZLIB_WINAPI -- exported functions use the WINAPI calling convention 11: 0 (reserved) @@ -1093,6 +1212,7 @@ ZEXTERN uLong ZEXPORT zlibCompileFlags OF((void)); 27-31: 0 (reserved) */ +#ifndef Z_SOLO /* utility functions */ @@ -1111,7 +1231,8 @@ ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen, the byte length of the source buffer. Upon entry, destLen is the total size of the destination buffer, which must be at least the value returned by compressBound(sourceLen). Upon exit, destLen is the actual size of the - compressed buffer. + compressed data. compress() is equivalent to compress2() with a level + parameter of Z_DEFAULT_COMPRESSION. compress returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if there was not enough room in the output @@ -1127,7 +1248,7 @@ ZEXTERN int ZEXPORT compress2 OF((Bytef *dest, uLongf *destLen, length of the source buffer. Upon entry, destLen is the total size of the destination buffer, which must be at least the value returned by compressBound(sourceLen). Upon exit, destLen is the actual size of the - compressed buffer. + compressed data. compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if there was not enough room in the output buffer, @@ -1150,13 +1271,22 @@ ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, uncompressed data. (The size of the uncompressed data must have been saved previously by the compressor and transmitted to the decompressor by some mechanism outside the scope of this compression library.) Upon exit, destLen - is the actual size of the uncompressed buffer. + is the actual size of the uncompressed data. uncompress returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if there was not enough room in the output - buffer, or Z_DATA_ERROR if the input data was corrupted or incomplete. + buffer, or Z_DATA_ERROR if the input data was corrupted or incomplete. In + the case where there is not enough room, uncompress() will fill the output + buffer with the uncompressed data up to that point. */ +ZEXTERN int ZEXPORT uncompress2 OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong *sourceLen)); +/* + Same as uncompress, except that sourceLen is a pointer, where the + length of the source is *sourceLen. On return, *sourceLen is the number of + source bytes consumed. +*/ /* gzip file access functions */ @@ -1167,7 +1297,7 @@ ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, wrapper, documented in RFC 1952, wrapped around a deflate stream. */ -typedef voidp gzFile; /* opaque gzip file descriptor */ +typedef struct gzFile_s *gzFile; /* semi-opaque gzip file descriptor */ /* ZEXTERN gzFile ZEXPORT gzopen OF((const char *path, const char *mode)); @@ -1177,13 +1307,28 @@ ZEXTERN gzFile ZEXPORT gzopen OF((const char *path, const char *mode)); a strategy: 'f' for filtered data as in "wb6f", 'h' for Huffman-only compression as in "wb1h", 'R' for run-length encoding as in "wb1R", or 'F' for fixed code compression as in "wb9F". (See the description of - deflateInit2 for more information about the strategy parameter.) Also "a" - can be used instead of "w" to request that the gzip stream that will be - written be appended to the file. "+" will result in an error, since reading - and writing to the same gzip file is not supported. + deflateInit2 for more information about the strategy parameter.) 'T' will + request transparent writing or appending with no compression and not using + the gzip format. + + "a" can be used instead of "w" to request that the gzip stream that will + be written be appended to the file. "+" will result in an error, since + reading and writing to the same gzip file is not supported. The addition of + "x" when writing will create the file exclusively, which fails if the file + already exists. On systems that support it, the addition of "e" when + reading or writing will set the flag to close the file on an execve() call. + + These functions, as well as gzip, will read and decode a sequence of gzip + streams in a file. The append function of gzopen() can be used to create + such a file. (Also see gzflush() for another way to do this.) When + appending, gzopen does not test whether the file begins with a gzip stream, + nor does it look for the end of the gzip streams to begin appending. gzopen + will simply append a gzip stream to the existing file. gzopen can be used to read a file which is not in gzip format; in this - case gzread will directly read from the file without decompression. + case gzread will directly read from the file without decompression. When + reading, this will be detected automatically by looking for the magic two- + byte gzip header. gzopen returns NULL if the file could not be opened, if there was insufficient memory to allocate the gzFile state, or if an invalid mode was @@ -1202,7 +1347,11 @@ ZEXTERN gzFile ZEXPORT gzdopen OF((int fd, const char *mode)); descriptor fd, just like fclose(fdopen(fd, mode)) closes the file descriptor fd. If you want to keep fd open, use fd = dup(fd_keep); gz = gzdopen(fd, mode);. The duplicated descriptor should be saved to avoid a leak, since - gzdopen does not close fd if it fails. + gzdopen does not close fd if it fails. If you are using fileno() to get the + file descriptor from a FILE *, then you will have to use dup() to avoid + double-close()ing the file descriptor. Both gzclose() and fclose() will + close the associated file descriptor, so they need to have different file + descriptors. gzdopen returns NULL if there was insufficient memory to allocate the gzFile state, if an invalid mode was specified (an 'r', 'w', or 'a' was not @@ -1217,10 +1366,9 @@ ZEXTERN int ZEXPORT gzbuffer OF((gzFile file, unsigned size)); default buffer size is 8192 bytes. This function must be called after gzopen() or gzdopen(), and before any other calls that read or write the file. The buffer memory allocation is always deferred to the first read or - write. Two buffers are allocated, either both of the specified size when - writing, or one of the specified size and the other twice that size when - reading. A larger buffer size of, for example, 64K or 128K bytes will - noticeably increase the speed of decompression (reading). + write. Three times that size in buffer space is allocated. A larger buffer + size of, for example, 64K or 128K bytes will noticeably increase the speed + of decompression (reading). The new buffer size also affects the maximum length for gzprintf(). @@ -1231,26 +1379,68 @@ ZEXTERN int ZEXPORT gzbuffer OF((gzFile file, unsigned size)); ZEXTERN int ZEXPORT gzsetparams OF((gzFile file, int level, int strategy)); /* Dynamically update the compression level or strategy. See the description - of deflateInit2 for the meaning of these parameters. + of deflateInit2 for the meaning of these parameters. Previously provided + data is flushed before the parameter change. - gzsetparams returns Z_OK if success, or Z_STREAM_ERROR if the file was not - opened for writing. + gzsetparams returns Z_OK if success, Z_STREAM_ERROR if the file was not + opened for writing, Z_ERRNO if there is an error writing the flushed data, + or Z_MEM_ERROR if there is a memory allocation error. */ ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len)); /* Reads the given number of uncompressed bytes from the compressed file. If - the input file was not in gzip format, gzread copies the given number of - bytes into the buffer. + the input file is not in gzip format, gzread copies the given number of + bytes into the buffer directly from the file. After reaching the end of a gzip stream in the input, gzread will continue - to read, looking for another gzip stream, or failing that, reading the rest - of the input file directly without decompression. The entire input file - will be read if gzread is called until it returns less than the requested - len. + to read, looking for another gzip stream. Any number of gzip streams may be + concatenated in the input file, and will all be decompressed by gzread(). + If something other than a gzip stream is encountered after a gzip stream, + that remaining trailing garbage is ignored (and no error is returned). + + gzread can be used to read a gzip file that is being concurrently written. + Upon reaching the end of the input, gzread will return with the available + data. If the error code returned by gzerror is Z_OK or Z_BUF_ERROR, then + gzclearerr can be used to clear the end of file indicator in order to permit + gzread to be tried again. Z_OK indicates that a gzip stream was completed + on the last gzread. Z_BUF_ERROR indicates that the input file ended in the + middle of a gzip stream. Note that gzread does not return -1 in the event + of an incomplete gzip stream. This error is deferred until gzclose(), which + will return Z_BUF_ERROR if the last gzread ended in the middle of a gzip + stream. Alternatively, gzerror can be used before gzclose to detect this + case. gzread returns the number of uncompressed bytes actually read, less than - len for end of file, or -1 for error. + len for end of file, or -1 for error. If len is too large to fit in an int, + then nothing is read, -1 is returned, and the error state is set to + Z_STREAM_ERROR. +*/ + +ZEXTERN z_size_t ZEXPORT gzfread OF((voidp buf, z_size_t size, z_size_t nitems, + gzFile file)); +/* + Read up to nitems items of size size from file to buf, otherwise operating + as gzread() does. This duplicates the interface of stdio's fread(), with + size_t request and return types. If the library defines size_t, then + z_size_t is identical to size_t. If not, then z_size_t is an unsigned + integer type that can contain a pointer. + + gzfread() returns the number of full items read of size size, or zero if + the end of the file was reached and a full item could not be read, or if + there was an error. gzerror() must be consulted if zero is returned in + order to determine if there was an error. If the multiplication of size and + nitems overflows, i.e. the product does not fit in a z_size_t, then nothing + is read, zero is returned, and the error state is set to Z_STREAM_ERROR. + + In the event that the end of file is reached and only a partial item is + available at the end, i.e. the remaining uncompressed data length is not a + multiple of size, then the final partial item is nevetheless read into buf + and the end-of-file flag is set. The length of the partial item read is not + provided, but could be inferred from the result of gztell(). This behavior + is the same as the behavior of fread() implementations in common libraries, + but it prevents the direct use of gzfread() to read a concurrently written + file, reseting and retrying on end-of-file, when size is not 1. */ ZEXTERN int ZEXPORT gzwrite OF((gzFile file, @@ -1261,19 +1451,33 @@ ZEXTERN int ZEXPORT gzwrite OF((gzFile file, error. */ -ZEXTERN int ZEXPORTVA gzprintf OF((gzFile file, const char *format, ...)); +ZEXTERN z_size_t ZEXPORT gzfwrite OF((voidpc buf, z_size_t size, + z_size_t nitems, gzFile file)); +/* + gzfwrite() writes nitems items of size size from buf to file, duplicating + the interface of stdio's fwrite(), with size_t request and return types. If + the library defines size_t, then z_size_t is identical to size_t. If not, + then z_size_t is an unsigned integer type that can contain a pointer. + + gzfwrite() returns the number of full items written of size size, or zero + if there was an error. If the multiplication of size and nitems overflows, + i.e. the product does not fit in a z_size_t, then nothing is written, zero + is returned, and the error state is set to Z_STREAM_ERROR. +*/ + +ZEXTERN int ZEXPORTVA gzprintf Z_ARG((gzFile file, const char *format, ...)); /* Converts, formats, and writes the arguments to the compressed file under control of the format string, as in fprintf. gzprintf returns the number of - uncompressed bytes actually written, or 0 in case of error. The number of - uncompressed bytes written is limited to 8191, or one less than the buffer - size given to gzbuffer(). The caller should assure that this limit is not - exceeded. If it is exceeded, then gzprintf() will return an error (0) with - nothing written. In this case, there may also be a buffer overflow with - unpredictable consequences, which is possible only if zlib was compiled with - the insecure functions sprintf() or vsprintf() because the secure snprintf() - or vsnprintf() functions were not available. This can be determined using - zlibCompileFlags(). + uncompressed bytes actually written, or a negative zlib error code in case + of error. The number of uncompressed bytes written is limited to 8191, or + one less than the buffer size given to gzbuffer(). The caller should assure + that this limit is not exceeded. If it is exceeded, then gzprintf() will + return an error (0) with nothing written. In this case, there may also be a + buffer overflow with unpredictable consequences, which is possible only if + zlib was compiled with the insecure functions sprintf() or vsprintf() + because the secure snprintf() or vsnprintf() functions were not available. + This can be determined using zlibCompileFlags(). */ ZEXTERN int ZEXPORT gzputs OF((gzFile file, const char *s)); @@ -1306,7 +1510,10 @@ ZEXTERN int ZEXPORT gzputc OF((gzFile file, int c)); ZEXTERN int ZEXPORT gzgetc OF((gzFile file)); /* Reads one byte from the compressed file. gzgetc returns this byte or -1 - in case of end of file or error. + in case of end of file or error. This is implemented as a macro for speed. + As such, it does not do all of the checking the other functions do. I.e. + it does not check to see if file is NULL, nor whether the structure file + points to has been clobbered or not. */ ZEXTERN int ZEXPORT gzungetc OF((int c, gzFile file)); @@ -1330,7 +1537,7 @@ ZEXTERN int ZEXPORT gzflush OF((gzFile file, int flush)); If the flush parameter is Z_FINISH, the remaining data is written and the gzip stream is completed in the output. If gzwrite() is called again, a new gzip stream will be started in the output. gzread() is able to read such - concatented gzip streams. + concatenated gzip streams. gzflush should be called only when strictly necessary because it will degrade compression if called too often. @@ -1402,9 +1609,7 @@ ZEXTERN int ZEXPORT gzeof OF((gzFile file)); ZEXTERN int ZEXPORT gzdirect OF((gzFile file)); /* Returns true (1) if file is being copied directly while reading, or false - (0) if file is a gzip stream being decompressed. This state can change from - false to true while reading the input file if the end of a gzip stream is - reached, but is followed by data that is not another gzip stream. + (0) if file is a gzip stream being decompressed. If the input file is empty, gzdirect() will return true, since the input does not contain a gzip stream. @@ -1413,6 +1618,13 @@ ZEXTERN int ZEXPORT gzdirect OF((gzFile file)); cause buffers to be allocated to allow reading the file to determine if it is a gzip file. Therefore if gzbuffer() is used, it should be called before gzdirect(). + + When writing, gzdirect() returns true (1) if transparent writing was + requested ("wT" for the gzopen() mode), or false (0) otherwise. (Note: + gzdirect() is not needed when writing. Transparent writing must be + explicitly requested, so the application already knows the answer. When + linking statically, using gzdirect() will include all of the zlib code for + gzip file reading and decompression, which may not be desired.) */ ZEXTERN int ZEXPORT gzclose OF((gzFile file)); @@ -1424,7 +1636,8 @@ ZEXTERN int ZEXPORT gzclose OF((gzFile file)); must not be called more than once on the same allocation. gzclose will return Z_STREAM_ERROR if file is not valid, Z_ERRNO on a - file operation error, or Z_OK on success. + file operation error, Z_MEM_ERROR if out of memory, Z_BUF_ERROR if the + last read ended in the middle of a gzip stream, or Z_OK on success. */ ZEXTERN int ZEXPORT gzclose_r OF((gzFile file)); @@ -1462,6 +1675,7 @@ ZEXTERN void ZEXPORT gzclearerr OF((gzFile file)); file that is being written concurrently. */ +#endif /* !Z_SOLO */ /* checksum functions */ @@ -1477,7 +1691,7 @@ ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len)); return the updated checksum. If buf is Z_NULL, this function returns the required initial value for the checksum. - An Adler-32 checksum is almost as reliable as a CRC32 but can be computed + An Adler-32 checksum is almost as reliable as a CRC-32 but can be computed much faster. Usage example: @@ -1490,6 +1704,12 @@ ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len)); if (adler != original_adler) error(); */ +ZEXTERN uLong ZEXPORT adler32_z OF((uLong adler, const Bytef *buf, + z_size_t len)); +/* + Same as adler32(), but with a size_t length. +*/ + /* ZEXTERN uLong ZEXPORT adler32_combine OF((uLong adler1, uLong adler2, z_off_t len2)); @@ -1497,16 +1717,17 @@ ZEXTERN uLong ZEXPORT adler32_combine OF((uLong adler1, uLong adler2, Combine two Adler-32 checksums into one. For two sequences of bytes, seq1 and seq2 with lengths len1 and len2, Adler-32 checksums were calculated for each, adler1 and adler2. adler32_combine() returns the Adler-32 checksum of - seq1 and seq2 concatenated, requiring only adler1, adler2, and len2. + seq1 and seq2 concatenated, requiring only adler1, adler2, and len2. Note + that the z_off_t type (like off_t) is a signed integer. If len2 is + negative, the result has no meaning or utility. */ ZEXTERN uLong ZEXPORT crc32 OF((uLong crc, const Bytef *buf, uInt len)); /* Update a running CRC-32 with the bytes buf[0..len-1] and return the updated CRC-32. If buf is Z_NULL, this function returns the required - initial value for the for the crc. Pre- and post-conditioning (one's - complement) is performed within this function so it shouldn't be done by the - application. + initial value for the crc. Pre- and post-conditioning (one's complement) is + performed within this function so it shouldn't be done by the application. Usage example: @@ -1518,6 +1739,12 @@ ZEXTERN uLong ZEXPORT crc32 OF((uLong crc, const Bytef *buf, uInt len)); if (crc != original_crc) error(); */ +ZEXTERN uLong ZEXPORT crc32_z OF((uLong adler, const Bytef *buf, + z_size_t len)); +/* + Same as crc32(), but with a size_t length. +*/ + /* ZEXTERN uLong ZEXPORT crc32_combine OF((uLong crc1, uLong crc2, z_off_t len2)); @@ -1548,18 +1775,64 @@ ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits, unsigned char FAR *window, const char *version, int stream_size)); -#define deflateInit(strm, level) \ - deflateInit_((strm), (level), ZLIB_VERSION, sizeof(z_stream)) -#define inflateInit(strm) \ - inflateInit_((strm), ZLIB_VERSION, sizeof(z_stream)) -#define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ - deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ - (strategy), ZLIB_VERSION, sizeof(z_stream)) -#define inflateInit2(strm, windowBits) \ - inflateInit2_((strm), (windowBits), ZLIB_VERSION, sizeof(z_stream)) -#define inflateBackInit(strm, windowBits, window) \ - inflateBackInit_((strm), (windowBits), (window), \ - ZLIB_VERSION, sizeof(z_stream)) +#ifdef Z_PREFIX_SET +# define z_deflateInit(strm, level) \ + deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_inflateInit(strm) \ + inflateInit_((strm), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ + deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ + (strategy), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_inflateInit2(strm, windowBits) \ + inflateInit2_((strm), (windowBits), ZLIB_VERSION, \ + (int)sizeof(z_stream)) +# define z_inflateBackInit(strm, windowBits, window) \ + inflateBackInit_((strm), (windowBits), (window), \ + ZLIB_VERSION, (int)sizeof(z_stream)) +#else +# define deflateInit(strm, level) \ + deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) +# define inflateInit(strm) \ + inflateInit_((strm), ZLIB_VERSION, (int)sizeof(z_stream)) +# define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ + deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ + (strategy), ZLIB_VERSION, (int)sizeof(z_stream)) +# define inflateInit2(strm, windowBits) \ + inflateInit2_((strm), (windowBits), ZLIB_VERSION, \ + (int)sizeof(z_stream)) +# define inflateBackInit(strm, windowBits, window) \ + inflateBackInit_((strm), (windowBits), (window), \ + ZLIB_VERSION, (int)sizeof(z_stream)) +#endif + +#ifndef Z_SOLO + +/* gzgetc() macro and its supporting function and exposed data structure. Note + * that the real internal state is much larger than the exposed structure. + * This abbreviated structure exposes just enough for the gzgetc() macro. The + * user should not mess with these exposed elements, since their names or + * behavior could change in the future, perhaps even capriciously. They can + * only be used by the gzgetc() macro. You have been warned. + */ +struct gzFile_s { + unsigned have; + unsigned char *next; + z_off64_t pos; +}; +ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file)); /* backward compatibility */ +#ifdef Z_PREFIX_SET +# undef z_gzgetc +# define z_gzgetc(g) \ + ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g)) +#elif defined(Z_CR_PREFIX_SET) +# undef gzgetc +# define gzgetc(g) \ + ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) \ + : (Cr_z_gzgetc)(g)) +#else +# define gzgetc(g) \ + ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g)) +#endif /* provide 64-bit offset functions if _LARGEFILE64_SOURCE defined, and/or * change the regular functions to 64 bits if _FILE_OFFSET_BITS is 64 (if @@ -1567,7 +1840,7 @@ ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits, * functions are changed to 64 bits) -- in case these are set on systems * without large file support, _LFS64_LARGEFILE must also be true */ -#if defined(_LARGEFILE64_SOURCE) && _LFS64_LARGEFILE-0 +#ifdef Z_LARGE64 ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); ZEXTERN z_off64_t ZEXPORT gzseek64 OF((gzFile, z_off64_t, int)); ZEXTERN z_off64_t ZEXPORT gztell64 OF((gzFile)); @@ -1576,45 +1849,47 @@ ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits, ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off64_t)); #endif -#if !defined(ZLIB_INTERNAL) && _FILE_OFFSET_BITS-0 == 64 && _LFS64_LARGEFILE-0 -# ifdef gzopen -# undef gzopen -# endif -# define gzopen gzopen64 -# ifdef gzseek -# undef gzseek -# endif -# define gzseek gzseek64 -# ifdef gztell -# undef gztell -# endif -# define gztell gztell64 -# ifdef gzoffset -# undef gzoffset -# endif -# define gzoffset gzoffset64 -# ifdef adler32_combine -# undef adler32_combine -# endif -# define adler32_combine adler32_combine64 -# ifdef crc32_combine -# undef crc32_combine +#if !defined(ZLIB_INTERNAL) && defined(Z_WANT64) +# ifdef Z_PREFIX_SET +# define z_gzopen z_gzopen64 +# define z_gzseek z_gzseek64 +# define z_gztell z_gztell64 +# define z_gzoffset z_gzoffset64 +# define z_adler32_combine z_adler32_combine64 +# define z_crc32_combine z_crc32_combine64 +# else +# ifdef gzopen +# undef gzopen +# endif +# define gzopen gzopen64 +# ifdef gzseek +# undef gzseek +# endif +# define gzseek gzseek64 +# ifdef gztell +# undef gztell +# endif +# define gztell gztell64 +# ifdef gzoffset +# undef gzoffset +# endif +# define gzoffset gzoffset64 +# ifdef adler32_combine +# undef adler32_combine +# endif +# define adler32_combine adler32_combine64 +# ifdef crc32_combine +# undef crc32_combine +# endif +# define crc32_combine crc32_combine64 # endif -# define crc32_combine crc32_combine64 -# ifdef _LARGEFILE64_SOURCE +# ifndef Z_LARGE64 ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); ZEXTERN z_off_t ZEXPORT gzseek64 OF((gzFile, z_off_t, int)); ZEXTERN z_off_t ZEXPORT gztell64 OF((gzFile)); ZEXTERN z_off_t ZEXPORT gzoffset64 OF((gzFile)); ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t)); ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t)); -# else - ZEXTERN gzFile ZEXPORT gzopen OF((const char *, const char *)); - ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile, z_off_t, int)); - ZEXTERN z_off_t ZEXPORT gztell OF((gzFile)); - ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile)); - ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t)); - ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t)); # endif #else ZEXTERN gzFile ZEXPORT gzopen OF((const char *, const char *)); @@ -1625,16 +1900,33 @@ ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits, ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t)); #endif -/* hack for buggy compilers */ -#if !defined(ZUTIL_H) && !defined(NO_DUMMY_DECL) - struct internal_state {int dummy;}; -#endif +#else /* Z_SOLO */ + + ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t)); + +#endif /* !Z_SOLO */ /* undocumented functions */ ZEXTERN const char * ZEXPORT zError OF((int)); ZEXTERN int ZEXPORT inflateSyncPoint OF((z_streamp)); -ZEXTERN const uLongf * ZEXPORT get_crc_table OF((void)); +ZEXTERN const z_crc_t FAR * ZEXPORT get_crc_table OF((void)); ZEXTERN int ZEXPORT inflateUndermine OF((z_streamp, int)); +ZEXTERN int ZEXPORT inflateValidate OF((z_streamp, int)); +ZEXTERN unsigned long ZEXPORT inflateCodesUsed OF ((z_streamp)); +ZEXTERN int ZEXPORT inflateResetKeep OF((z_streamp)); +ZEXTERN int ZEXPORT deflateResetKeep OF((z_streamp)); +#if (defined(_WIN32) || defined(__CYGWIN__)) && !defined(Z_SOLO) +ZEXTERN gzFile ZEXPORT gzopen_w OF((const wchar_t *path, + const char *mode)); +#endif +#if defined(STDC) || defined(Z_HAVE_STDARG_H) +# ifndef Z_SOLO +ZEXTERN int ZEXPORTVA gzvprintf Z_ARG((gzFile file, + const char *format, + va_list va)); +# endif +#endif #ifdef __cplusplus } diff --git a/third_party/zlib/zutil.c b/third_party/zlib/zutil.c index 898ed345b0..a76c6b0c7e 100644 --- a/third_party/zlib/zutil.c +++ b/third_party/zlib/zutil.c @@ -1,27 +1,27 @@ /* zutil.c -- target dependent utility functions for the compression library - * Copyright (C) 1995-2005, 2010 Jean-loup Gailly. + * Copyright (C) 1995-2017 Jean-loup Gailly * For conditions of distribution and use, see copyright notice in zlib.h */ /* @(#) $Id$ */ #include "zutil.h" - -#ifndef NO_DUMMY_DECL -struct internal_state {int dummy;}; /* for buggy compilers */ +#ifndef Z_SOLO +# include "gzguts.h" #endif -const char * const z_errmsg[10] = { -"need dictionary", /* Z_NEED_DICT 2 */ -"stream end", /* Z_STREAM_END 1 */ -"", /* Z_OK 0 */ -"file error", /* Z_ERRNO (-1) */ -"stream error", /* Z_STREAM_ERROR (-2) */ -"data error", /* Z_DATA_ERROR (-3) */ -"insufficient memory", /* Z_MEM_ERROR (-4) */ -"buffer error", /* Z_BUF_ERROR (-5) */ -"incompatible version",/* Z_VERSION_ERROR (-6) */ -""}; +z_const char * const z_errmsg[10] = { + (z_const char *)"need dictionary", /* Z_NEED_DICT 2 */ + (z_const char *)"stream end", /* Z_STREAM_END 1 */ + (z_const char *)"", /* Z_OK 0 */ + (z_const char *)"file error", /* Z_ERRNO (-1) */ + (z_const char *)"stream error", /* Z_STREAM_ERROR (-2) */ + (z_const char *)"data error", /* Z_DATA_ERROR (-3) */ + (z_const char *)"insufficient memory", /* Z_MEM_ERROR (-4) */ + (z_const char *)"buffer error", /* Z_BUF_ERROR (-5) */ + (z_const char *)"incompatible version",/* Z_VERSION_ERROR (-6) */ + (z_const char *)"" +}; const char * ZEXPORT zlibVersion() @@ -58,7 +58,7 @@ uLong ZEXPORT zlibCompileFlags() case 8: flags += 2 << 6; break; default: flags += 3 << 6; } -#ifdef DEBUG +#ifdef ZLIB_DEBUG flags += 1 << 8; #endif #if defined(ASMV) || defined(ASMINF) @@ -85,35 +85,35 @@ uLong ZEXPORT zlibCompileFlags() #ifdef FASTEST flags += 1L << 21; #endif -#ifdef STDC +#if defined(STDC) || defined(Z_HAVE_STDARG_H) # ifdef NO_vsnprintf - flags += 1L << 25; + flags += 1L << 25; # ifdef HAS_vsprintf_void - flags += 1L << 26; + flags += 1L << 26; # endif # else # ifdef HAS_vsnprintf_void - flags += 1L << 26; + flags += 1L << 26; # endif # endif #else - flags += 1L << 24; + flags += 1L << 24; # ifdef NO_snprintf - flags += 1L << 25; + flags += 1L << 25; # ifdef HAS_sprintf_void - flags += 1L << 26; + flags += 1L << 26; # endif # else # ifdef HAS_snprintf_void - flags += 1L << 26; + flags += 1L << 26; # endif # endif #endif return flags; } -#ifdef DEBUG - +#ifdef ZLIB_DEBUG +#include # ifndef verbose # define verbose 0 # endif @@ -181,6 +181,7 @@ void ZLIB_INTERNAL zmemzero(dest, len) } #endif +#ifndef Z_SOLO #ifdef SYS16BIT @@ -215,9 +216,11 @@ local ptr_table table[MAX_PTR]; voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, unsigned items, unsigned size) { - voidpf buf = opaque; /* just to make some compilers happy */ + voidpf buf; ulg bsize = (ulg)items*size; + (void)opaque; + /* If we allocate less than 65520 bytes, we assume that farmalloc * will return a usable pointer which doesn't have to be normalized. */ @@ -240,6 +243,9 @@ voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, unsigned items, unsigned size) void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) { int n; + + (void)opaque; + if (*(ush*)&ptr != 0) { /* object < 64K */ farfree(ptr); return; @@ -255,7 +261,6 @@ void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) next_ptr--; return; } - ptr = opaque; /* just to make some compilers happy */ Assert(0, "zcfree: ptr not found"); } @@ -274,13 +279,13 @@ void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, uInt items, uInt size) { - if (opaque) opaque = 0; /* to make compiler happy */ + (void)opaque; return _halloc((long)items, size); } void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) { - if (opaque) opaque = 0; /* to make compiler happy */ + (void)opaque; _hfree(ptr); } @@ -302,7 +307,7 @@ voidpf ZLIB_INTERNAL zcalloc (opaque, items, size) unsigned items; unsigned size; { - if (opaque) items += size - size; /* make compiler happy */ + (void)opaque; return sizeof(uInt) > 2 ? (voidpf)malloc(items * size) : (voidpf)calloc(items, size); } @@ -311,8 +316,10 @@ void ZLIB_INTERNAL zcfree (opaque, ptr) voidpf opaque; voidpf ptr; { + (void)opaque; free(ptr); - if (opaque) return; /* make compiler happy */ } #endif /* MY_ZCALLOC */ + +#endif /* !Z_SOLO */ diff --git a/third_party/zlib/zutil.h b/third_party/zlib/zutil.h index 3c8326f5ec..4425bcf75e 100644 --- a/third_party/zlib/zutil.h +++ b/third_party/zlib/zutil.h @@ -1,5 +1,5 @@ /* zutil.h -- internal interface and configuration of the compression library - * Copyright (C) 1995-2010 Jean-loup Gailly. + * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -13,7 +13,7 @@ #ifndef ZUTIL_H #define ZUTIL_H -#if ((__GNUC__-0) * 10 + __GNUC_MINOR__-0 >= 33) && !defined(NO_VIZ) +#ifdef HAVE_HIDDEN # define ZLIB_INTERNAL __attribute__((visibility ("hidden"))) #else # define ZLIB_INTERNAL @@ -21,7 +21,7 @@ #include "zlib.h" -#ifdef STDC +#if defined(STDC) && !defined(Z_SOLO) # if !(defined(_WIN32_WCE) && defined(_MSC_VER)) # include # endif @@ -44,10 +44,16 @@ # endif #endif +#ifdef Z_SOLO + typedef long ptrdiff_t; /* guess -- will be caught if guess is wrong */ +#endif + #ifndef local # define local static #endif -/* compile with -Dlocal if your debugger can't find static symbols */ +/* since "static" is used to mean two completely different things in C, we + define "local" for the non-static meaning of "static", for readability + (compile with -Dlocal if your debugger can't find static symbols) */ typedef unsigned char uch; typedef uch FAR uchf; @@ -55,13 +61,13 @@ typedef unsigned short ush; typedef ush FAR ushf; typedef unsigned long ulg; -extern const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ +extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ /* (size given to avoid silly warnings with Visual C++) */ #define ERR_MSG(err) z_errmsg[Z_NEED_DICT-(err)] #define ERR_RETURN(strm,err) \ - return (strm->msg = (char*)ERR_MSG(err), (err)) + return (strm->msg = ERR_MSG(err), (err)) /* To be used only when the state is known to be valid */ /* common constants */ @@ -93,69 +99,83 @@ extern const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ #if defined(MSDOS) || (defined(WINDOWS) && !defined(WIN32)) # define OS_CODE 0x00 -# if defined(__TURBOC__) || defined(__BORLANDC__) -# if (__STDC__ == 1) && (defined(__LARGE__) || defined(__COMPACT__)) - /* Allow compilation with ANSI keywords only enabled */ - void _Cdecl farfree( void *block ); - void *_Cdecl farmalloc( unsigned long nbytes ); -# else -# include +# ifndef Z_SOLO +# if defined(__TURBOC__) || defined(__BORLANDC__) +# if (__STDC__ == 1) && (defined(__LARGE__) || defined(__COMPACT__)) + /* Allow compilation with ANSI keywords only enabled */ + void _Cdecl farfree( void *block ); + void *_Cdecl farmalloc( unsigned long nbytes ); +# else +# include +# endif +# else /* MSC or DJGPP */ +# include # endif -# else /* MSC or DJGPP */ -# include # endif #endif #ifdef AMIGA -# define OS_CODE 0x01 +# define OS_CODE 1 #endif #if defined(VAXC) || defined(VMS) -# define OS_CODE 0x02 +# define OS_CODE 2 # define F_OPEN(name, mode) \ fopen((name), (mode), "mbc=60", "ctx=stm", "rfm=fix", "mrs=512") #endif +#ifdef __370__ +# if __TARGET_LIB__ < 0x20000000 +# define OS_CODE 4 +# elif __TARGET_LIB__ < 0x40000000 +# define OS_CODE 11 +# else +# define OS_CODE 8 +# endif +#endif + #if defined(ATARI) || defined(atarist) -# define OS_CODE 0x05 +# define OS_CODE 5 #endif #ifdef OS2 -# define OS_CODE 0x06 -# ifdef M_I86 +# define OS_CODE 6 +# if defined(M_I86) && !defined(Z_SOLO) # include # endif #endif #if defined(MACOS) || defined(TARGET_OS_MAC) -# define OS_CODE 0x07 -# if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os -# include /* for fdopen */ -# else -# ifndef fdopen -# define fdopen(fd,mode) NULL /* No fdopen() */ +# define OS_CODE 7 +# ifndef Z_SOLO +# if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os +# include /* for fdopen */ +# else +# ifndef fdopen +# define fdopen(fd,mode) NULL /* No fdopen() */ +# endif # endif # endif #endif -#ifdef TOPS20 -# define OS_CODE 0x0a +#ifdef __acorn +# define OS_CODE 13 #endif -#ifdef _MSC_VER -#define zalign(x) __declspec(align(x)) -#else -#define zalign(x) __attribute__((aligned((x)))) +#if defined(WIN32) && !defined(__CYGWIN__) +# define OS_CODE 10 #endif -#ifdef WIN32 -# ifndef __CYGWIN__ /* Cygwin is Unix, not Win32 */ -# define OS_CODE 0x0b -# endif +#ifdef _BEOS_ +# define OS_CODE 16 +#endif + +#ifdef __TOS_OS400__ +# define OS_CODE 18 #endif -#ifdef __50SERIES /* Prime/PRIMOS */ -# define OS_CODE 0x0f +#ifdef __APPLE__ +# define OS_CODE 19 #endif #if defined(_BEOS_) || defined(RISCOS) @@ -174,14 +194,15 @@ extern const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ # endif #endif -#if defined(__BORLANDC__) +#if defined(__BORLANDC__) && !defined(MSDOS) #pragma warn -8004 #pragma warn -8008 #pragma warn -8066 #endif /* provide prototypes for these when building zlib without LFS */ -#if !defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0 +#if !defined(_WIN32) && \ + (!defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0) ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t)); ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t)); #endif @@ -189,7 +210,7 @@ extern const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ /* common defaults */ #ifndef OS_CODE -# define OS_CODE 0x03 /* assume Unix */ +# define OS_CODE 3 /* assume Unix */ #endif #ifndef F_OPEN @@ -198,42 +219,7 @@ extern const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ /* functions */ -#if defined(STDC99) || (defined(__TURBOC__) && __TURBOC__ >= 0x550) -# ifndef HAVE_VSNPRINTF -# define HAVE_VSNPRINTF -# endif -#endif -#if defined(__CYGWIN__) -# ifndef HAVE_VSNPRINTF -# define HAVE_VSNPRINTF -# endif -#endif -#ifndef HAVE_VSNPRINTF -# ifdef MSDOS - /* vsnprintf may exist on some MS-DOS compilers (DJGPP?), - but for now we just assume it doesn't. */ -# define NO_vsnprintf -# endif -# ifdef __TURBOC__ -# define NO_vsnprintf -# endif -# ifdef WIN32 - /* In Win32, vsnprintf is available as the "non-ANSI" _vsnprintf. */ -# if !defined(vsnprintf) && !defined(NO_vsnprintf) -# if !defined(_MSC_VER) || ( defined(_MSC_VER) && _MSC_VER < 1500 ) -# define vsnprintf _vsnprintf -# endif -# endif -# endif -# ifdef __SASC -# define NO_vsnprintf -# endif -#endif -#ifdef VMS -# define NO_vsnprintf -#endif - -#if defined(pyr) +#if defined(pyr) || defined(Z_SOLO) # define NO_MEMCPY #endif #if defined(SMALL_MEDIUM) && !defined(_MSC_VER) && !defined(__SC__) @@ -263,7 +249,7 @@ extern const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ #endif /* Diagnostic functions */ -#ifdef DEBUG +#ifdef ZLIB_DEBUG # include extern int ZLIB_INTERNAL z_verbose; extern void ZLIB_INTERNAL z_error OF((char *m)); @@ -282,14 +268,25 @@ extern const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ # define Tracecv(c,x) #endif - -voidpf ZLIB_INTERNAL zcalloc OF((voidpf opaque, unsigned items, - unsigned size)); -void ZLIB_INTERNAL zcfree OF((voidpf opaque, voidpf ptr)); +#ifndef Z_SOLO + voidpf ZLIB_INTERNAL zcalloc OF((voidpf opaque, unsigned items, + unsigned size)); + void ZLIB_INTERNAL zcfree OF((voidpf opaque, voidpf ptr)); +#endif #define ZALLOC(strm, items, size) \ (*((strm)->zalloc))((strm)->opaque, (items), (size)) #define ZFREE(strm, addr) (*((strm)->zfree))((strm)->opaque, (voidpf)(addr)) #define TRY_FREE(s, p) {if (p) ZFREE(s, p);} +/* Reverse the bytes in a 32-bit value */ +#define ZSWAP32(q) ((((q) >> 24) & 0xff) + (((q) >> 8) & 0xff00) + \ + (((q) & 0xff00) << 8) + (((q) & 0xff) << 24)) + +#ifdef _MSC_VER +#define zalign(x) __declspec(align(x)) +#else +#define zalign(x) __attribute__((aligned((x)))) +#endif + #endif /* ZUTIL_H */